Skip to content

Commit

Permalink
[Enhancement] enhance sql digest (#52578)
Browse files Browse the repository at this point in the history 
Signed-off-by: Murphy <[email protected]>
(cherry picked from commit a5373f9)
Signed-off-by: Murphy <[email protected]>

# Conflicts:
#	fe/fe-core/src/main/java/com/starrocks/qe/ConnectProcessor.java
  • Loading branch information
@murphyatwork
murphyatwork committed Nov 5, 2024
1 parent 2065515 commit 66704c1
Show file tree
Hide file tree
Showing 7 changed files with 251 additions and 87 deletions.
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/common/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2424,6 +2424,10 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean enable_collect_query_detail_info = false;

@ConfField(mutable = true,
comment = "Enable the sql digest feature, building a parameterized digest for each sql in the query detail")
public static boolean enable_sql_digest = false;

@ConfField(mutable = true, comment = "explain level of query plan in this detail")
public static String query_detail_explain_level = "COSTS";

Expand Down
7 changes: 4 additions & 3 deletions fe/fe-core/src/main/java/com/starrocks/qe/ConnectProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,6 @@ public void auditAfterExec(String origStmt, StatementBase parsedStmt, PQueryStat
// err query
MetricRepo.COUNTER_QUERY_ERR.increase(1L);
ResourceGroupMetricMgr.increaseQueryErr(ctx, 1L);
ctx.getAuditEventBuilder().setDigest(computeStatementDigest(parsedStmt));
//represent analysis err
if (ctx.getState().getErrType() == QueryState.ErrType.ANALYSIS_ERR) {
MetricRepo.COUNTER_QUERY_ANALYSIS_ERR.increase(1L);
Expand All @@ -226,11 +225,13 @@ public void auditAfterExec(String origStmt, StatementBase parsedStmt, PQueryStat
MetricRepo.COUNTER_QUERY_SUCCESS.increase(1L);
MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
ResourceGroupMetricMgr.updateQueryLatency(ctx, elapseMs);
if (elapseMs > Config.qe_slow_log_ms || ctx.getSessionVariable().isEnableSQLDigest()) {
if (elapseMs > Config.qe_slow_log_ms) {
MetricRepo.COUNTER_SLOW_QUERY.increase(1L);
ctx.getAuditEventBuilder().setDigest(computeStatementDigest(parsedStmt));
}
}
if (Config.enable_sql_digest || ctx.getSessionVariable().isEnableSQLDigest()) {
ctx.getAuditEventBuilder().setDigest(computeStatementDigest(parsedStmt));
}
ctx.getAuditEventBuilder().setIsQuery(true);
if (ctx.getSessionVariable().isEnableBigQueryLog()) {
ctx.getAuditEventBuilder().setBigQueryLogCPUSecondThreshold(
Expand Down
83 changes: 1 addition & 82 deletions fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AstToSQLBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,9 @@
import com.starrocks.catalog.Table;
import com.starrocks.catalog.Type;
import com.starrocks.common.util.ParseUtil;
import com.starrocks.common.util.PrintableMap;
import com.starrocks.sql.ast.ArrayExpr;
import com.starrocks.sql.ast.CTERelation;
import com.starrocks.sql.ast.FieldReference;
import com.starrocks.sql.ast.FileTableFunctionRelation;
import com.starrocks.sql.ast.InsertStmt;
import com.starrocks.sql.ast.MapExpr;
import com.starrocks.sql.ast.NormalizedTableFunctionRelation;
import com.starrocks.sql.ast.SelectList;
Expand All @@ -38,8 +35,6 @@
import com.starrocks.sql.ast.TableFunctionRelation;
import com.starrocks.sql.ast.TableRelation;
import com.starrocks.sql.ast.ViewRelation;
import com.starrocks.sql.ast.pipe.CreatePipeStmt;
import com.starrocks.sql.parser.NodePosition;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -395,83 +390,6 @@ public String visitSlot(SlotRef expr, Void context) {
}
}

@Override
public String visitInsertStatement(InsertStmt insert, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("INSERT ");

// add hint
if (insert.getHintNodes() != null) {
sb.append(extractHintStr(insert.getHintNodes()));
}


if (insert.isOverwrite()) {
sb.append("OVERWRITE ");
} else {
sb.append("INTO ");
}

// target
if (insert.useTableFunctionAsTargetTable()) {
sb.append(visitFileTableFunction(
new FileTableFunctionRelation(insert.getTableFunctionProperties(), NodePosition.ZERO), context));
} else if (insert.useBlackHoleTableAsTargetTable()) {
sb.append("blackhole()");
} else {
sb.append(insert.getTableName().toSql());
}
sb.append(" ");

// target partition
if (insert.getTargetPartitionNames() != null &&
CollectionUtils.isNotEmpty(insert.getTargetPartitionNames().getPartitionNames())) {
List<String> names = insert.getTargetPartitionNames().getPartitionNames();
sb.append("PARTITION (").append(Joiner.on(",").join(names)).append(") ");
}

// label
if (StringUtils.isNotEmpty(insert.getLabel())) {
sb.append("WITH LABEL `").append(insert.getLabel()).append("` ");
}

// target column
if (CollectionUtils.isNotEmpty(insert.getTargetColumnNames())) {
String columns = insert.getTargetColumnNames().stream()
.map(x -> '`' + x + '`')
.collect(Collectors.joining(","));
sb.append("(").append(columns).append(") ");
}

// source
if (insert.getQueryStatement() != null) {
sb.append(visit(insert.getQueryStatement()));
}
return sb.toString();
}

@Override
public String visitCreatePipeStatement(CreatePipeStmt stmt, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("CREATE ");
if (stmt.isReplace()) {
sb.append("OR REPLACE ");
}
sb.append("PIPE ");
if (stmt.isIfNotExists()) {
sb.append("IF NOT EXISTS ");
}
sb.append(stmt.getPipeName()).append(" ");

Map<String, String> properties = stmt.getProperties();
if (properties != null && !properties.isEmpty()) {
sb.append("PROPERTIES(").append(new PrintableMap<>(properties, "=", true, false, hideCredential)).append(") ");
}

sb.append("AS ").append(visitInsertStatement(stmt.getInsertStmt(), context));
return sb.toString();
}

@Override
public String visitArrayExpr(ArrayExpr node, Void context) {
StringBuilder sb = new StringBuilder();
Expand All @@ -498,5 +416,6 @@ public String visitMapExpr(MapExpr node, Void context) {
sb.append("}");
return sb.toString();
}

}
}
106 changes: 105 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AstToStringBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
import com.starrocks.sql.ast.CreateUserStmt;
import com.starrocks.sql.ast.DataDescription;
import com.starrocks.sql.ast.DefaultValueExpr;
import com.starrocks.sql.ast.DeleteStmt;
import com.starrocks.sql.ast.DictionaryGetExpr;
import com.starrocks.sql.ast.DropMaterializedViewStmt;
import com.starrocks.sql.ast.ExceptRelation;
Expand All @@ -109,6 +110,7 @@
import com.starrocks.sql.ast.FileTableFunctionRelation;
import com.starrocks.sql.ast.GrantPrivilegeStmt;
import com.starrocks.sql.ast.GrantRoleStmt;
import com.starrocks.sql.ast.InsertStmt;
import com.starrocks.sql.ast.IntersectRelation;
import com.starrocks.sql.ast.JoinRelation;
import com.starrocks.sql.ast.LambdaFunctionExpr;
Expand Down Expand Up @@ -140,8 +142,11 @@
import com.starrocks.sql.ast.UserVariable;
import com.starrocks.sql.ast.ValuesRelation;
import com.starrocks.sql.ast.ViewRelation;
import com.starrocks.sql.ast.pipe.CreatePipeStmt;
import com.starrocks.sql.parser.NodePosition;
import com.starrocks.storagevolume.StorageVolume;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -926,6 +931,103 @@ public String visitAnalyticExpr(AnalyticExpr node, Void context) {
return sb.toString();
}

@Override
public String visitInsertStatement(InsertStmt insert, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("INSERT ");

// add hint
if (insert.getHintNodes() != null) {
sb.append(extractHintStr(insert.getHintNodes()));
}

if (insert.isOverwrite()) {
sb.append("OVERWRITE ");
} else {
sb.append("INTO ");
}

// target
if (insert.useTableFunctionAsTargetTable()) {
sb.append(visitFileTableFunction(
new FileTableFunctionRelation(insert.getTableFunctionProperties(), NodePosition.ZERO),
context));
} else if (insert.useBlackHoleTableAsTargetTable()) {
sb.append("blackhole()");
} else {
sb.append(insert.getTableName().toSql());
}
sb.append(" ");

// target partition
if (insert.getTargetPartitionNames() != null &&
org.apache.commons.collections4.CollectionUtils.isNotEmpty(
insert.getTargetPartitionNames().getPartitionNames())) {
List<String> names = insert.getTargetPartitionNames().getPartitionNames();
sb.append("PARTITION (").append(Joiner.on(",").join(names)).append(") ");
}

// label
visitInsertLabel(insert.getLabel(), sb);

// target column
if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(insert.getTargetColumnNames())) {
String columns = insert.getTargetColumnNames().stream()
.map(x -> '`' + x + '`')
.collect(Collectors.joining(","));
sb.append("(").append(columns).append(") ");
}

// source
if (insert.getQueryStatement() != null) {
sb.append(visit(insert.getQueryStatement()));
}
return sb.toString();
}

protected void visitInsertLabel(String label, StringBuilder sb) {
if (StringUtils.isNotEmpty(label)) {
sb.append("WITH LABEL `").append(label).append("` ");
}
}

@Override
public String visitCreatePipeStatement(CreatePipeStmt stmt, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("CREATE ");
if (stmt.isReplace()) {
sb.append("OR REPLACE ");
}
sb.append("PIPE ");
if (stmt.isIfNotExists()) {
sb.append("IF NOT EXISTS ");
}
sb.append(stmt.getPipeName()).append(" ");

Map<String, String> properties = stmt.getProperties();
if (properties != null && !properties.isEmpty()) {
sb.append("PROPERTIES(").append(new PrintableMap<>(properties, "=", true, false, hideCredential))
.append(") ");
}

sb.append("AS ").append(visitInsertStatement(stmt.getInsertStmt(), context));
return sb.toString();
}

@Override
public String visitDeleteStatement(DeleteStmt delete, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("DELETE FROM ");
sb.append(delete.getTableName().toSql());

if (delete.getWherePredicate() != null) {
sb.append(" WHERE ");
sb.append(visit(delete.getWherePredicate()));
}
return sb.toString();
}

@Override
public String visitArrayExpr(ArrayExpr node, Void context) {
StringBuilder sb = new StringBuilder();
sb.append('[');
Expand All @@ -934,6 +1036,7 @@ public String visitArrayExpr(ArrayExpr node, Void context) {
return sb.toString();
}

@Override
public String visitMapExpr(MapExpr node, Void context) {
StringBuilder sb = new StringBuilder();
sb.append("map{");
Expand Down Expand Up @@ -1369,7 +1472,7 @@ private String visitAstList(List<? extends ParseNode> contexts) {
return Joiner.on(", ").join(contexts.stream().map(this::visit).collect(toList()));
}

private String printWithParentheses(ParseNode node) {
protected String printWithParentheses(ParseNode node) {
if (node instanceof SlotRef || node instanceof LiteralExpr) {
return visit(node);
} else {
Expand Down Expand Up @@ -1449,6 +1552,7 @@ protected String extractHintStr(List<HintNode> hintNodes) {
}
return hintBuilder.toString();
}

}

public static void getDdlStmt(Table table, List<String> createTableStmt, List<String> addPartitionStmt,
Expand Down
52 changes: 51 additions & 1 deletion fe/fe-core/src/main/java/com/starrocks/sql/common/SqlDigestBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,68 @@

package com.starrocks.sql.common;

import com.google.common.base.Joiner;
import com.starrocks.analysis.InPredicate;
import com.starrocks.analysis.LimitElement;
import com.starrocks.analysis.LiteralExpr;
import com.starrocks.sql.analyzer.AstToStringBuilder;
import com.starrocks.sql.ast.StatementBase;
import com.starrocks.sql.ast.ValuesRelation;
import org.apache.commons.lang3.StringUtils;

//Used to build sql digests
import java.util.List;
import java.util.stream.Collectors;

/**
* Used to build sql digest(string without any dynamic parameters in it)
*/
public class SqlDigestBuilder {

public static String build(StatementBase statement) {
return new SqlDigestBuilderVisitor().visit(statement);
}

private static class SqlDigestBuilderVisitor extends AstToStringBuilder.AST2StringBuilderVisitor {

@Override
public String visitInPredicate(InPredicate node, Void context) {
if (!node.isLiteralChildren()) {
return super.visitInPredicate(node, context);
} else {
StringBuilder strBuilder = new StringBuilder();
String notStr = (node.isNotIn()) ? "NOT " : "";
strBuilder.append(printWithParentheses(node.getChild(0))).append(" ").append(notStr).append("IN ");
strBuilder.append("(?)");
return strBuilder.toString();
}
}

@Override
public String visitValues(ValuesRelation node, Void scope) {
if (node.isNullValues()) {
return "VALUES(NULL)";
}

StringBuilder sqlBuilder = new StringBuilder("VALUES");
if (!node.getRows().isEmpty()) {
StringBuilder rowBuilder = new StringBuilder();
rowBuilder.append("(");
List<String> rowStrings =
node.getRows().get(0).stream().map(this::visit).collect(Collectors.toList());
rowBuilder.append(Joiner.on(", ").join(rowStrings));
rowBuilder.append(")");
sqlBuilder.append(rowBuilder.toString());
}
return sqlBuilder.toString();
}

@Override
protected void visitInsertLabel(String label, StringBuilder sb) {
if (StringUtils.isNotEmpty(label)) {
sb.append("WITH LABEL ? ");
}
}

@Override
public String visitLiteral(LiteralExpr expr, Void context) {
return "?";
Expand Down
Loading

0 comments on commit 66704c1

Please sign in to comment.