From 8bebb11fcfc8ad3a165fa0e4f5598f39532d51d2 Mon Sep 17 00:00:00 2001 From: wtt <30461027+wtt40122@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:16:35 +0800 Subject: [PATCH] Add post processing service for log content (#153) --- .../mone/log/parse/AbstractLogParser.java | 44 ++++++++++++++----- .../mone/log/parse/CustomLogParser.java | 4 +- .../mone/log/parse/FieldInterceptor.java | 14 ++++++ .../xiaomi/mone/log/parse/JsonLogParser.java | 4 +- .../com/xiaomi/mone/log/parse/LogParser.java | 1 + .../xiaomi/mone/log/parse/NginxLogParser.java | 4 +- .../xiaomi/mone/log/parse/RawLogParser.java | 4 +- .../xiaomi/mone/log/parse/RegexLogParser.java | 4 +- .../mone/log/parse/SeparatorLogParser.java | 4 +- 9 files changed, 60 insertions(+), 23 deletions(-) create mode 100644 ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java index ff30d294a..8088b18f9 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java @@ -1,9 +1,16 @@ package com.xiaomi.mone.log.parse; +import cn.hutool.core.util.ClassUtil; +import cn.hutool.core.util.ReflectUtil; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** * @author wtt @@ -11,12 +18,30 @@ * @description * @date 2023/10/31 16:32 */ +@Slf4j public abstract class AbstractLogParser implements LogParser { protected LogParserData parserData; + private List fieldInterceptors = Lists.newArrayList(); + public AbstractLogParser(LogParserData parserData) { this.parserData = parserData; + createFieldInterceptors(); + } + + private void createFieldInterceptors() { + try { + Set> implementationClasses = ClassUtil.scanPackageBySuper(PACKAGE_NAME, FieldInterceptor.class); + + fieldInterceptors = implementationClasses.stream() + .map(ReflectUtil::newInstance) + .filter(obj -> obj instanceof FieldInterceptor) + .map(obj -> (FieldInterceptor) obj) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("createFieldInterceptors", e); + } } @Override @@ -26,20 +51,16 @@ public Map parse(String logData, String ip, Long lineNum, Long c wrapMap(parseData, parserData, ip, lineNum, fileName, collectStamp); checkMessageExist(parseData, logData); validRet(parseData, logData); - handleImproperTraceId(parseData); + + fieldInterceptors.stream().forEach(fieldInterceptor -> fieldInterceptor.postProcess(parseData)); return parseData; } - private void handleImproperTraceId(Map parseData) { - if (parseData.containsKey(TRACE_ID_KEY)) { - Object traceIdValue = parseData.get(TRACE_ID_KEY); - if (traceIdValue instanceof String) { - String traceId = (String) traceIdValue; - if (traceId.startsWith(TRACE_ID_KEY + "=")) { - parseData.put(TRACE_ID_KEY, traceId.substring(TRACE_ID_KEY.length() + 1)); - } - } - } + @Override + public Map parseSimple(String logData, Long collectStamp) { + Map parseData = doParseSimple(logData, collectStamp); + fieldInterceptors.stream().forEach(fieldInterceptor -> fieldInterceptor.postProcess(parseData)); + return parseData; } protected void validTimestamp(Map ret, Long collectStamp) { @@ -106,4 +127,5 @@ void validRet(Map ret, String logData) { public abstract Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName); + public abstract Map doParseSimple(String logData, Long collectStamp); } diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java index 24a44338b..23c5e7d10 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java @@ -52,11 +52,11 @@ public CustomLogParser(LogParserData parserData) { @Override public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return parseSimple(logData, collectStamp); + return doParseSimple(logData, collectStamp); } @Override - public Map parseSimple(String logData, Long collectStamp) { + public Map doParseSimple(String logData, Long collectStamp) { Map ret = new HashMap<>(); String originData = logData; if (logData == null) { diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java new file mode 100644 index 000000000..784055f23 --- /dev/null +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java @@ -0,0 +1,14 @@ +package com.xiaomi.mone.log.parse; + +import java.util.Map; + +/** + * @author wtt + * @version 1.0 + * @description + * @date 2023/11/17 11:17 + */ +public interface FieldInterceptor { + + void postProcess(Map data); +} diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java index 4990635e7..e65f3c7f9 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java @@ -41,11 +41,11 @@ public JsonLogParser(LogParserData parserData) { @Override public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return parseSimple(logData, collectStamp); + return doParseSimple(logData, collectStamp); } @Override - public Map parseSimple(String logData, Long collectStamp) { + public Map doParseSimple(String logData, Long collectStamp) { Map ret = new HashMap<>(); if (logData == null || logData.length() == 0) { return ret; diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java index 1a871865b..03dcb0fe1 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java @@ -58,6 +58,7 @@ public interface LogParser { String esKeyMap_lineNumber = "linenumber"; String esKyeMap_fileName = "filename"; String TRACE_ID_KEY = "traceId"; + String PACKAGE_NAME = "com.xiaomi.mone.log"; Map parse(String logData, String ip, Long lineNum, Long collectStamp, String fileName); diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java index 8b7184ceb..c9d2174a5 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java @@ -46,8 +46,8 @@ public Map doParse(String logData, String ip, Long lineNum, Long } @Override - public Map parseSimple(String logData, Long collectStamp) { - return this.regexLogParser.parseSimple(logData, collectStamp); + public Map doParseSimple(String logData, Long collectStamp) { + return this.regexLogParser.doParseSimple(logData, collectStamp); } public String generateRegexFromNginxScript(String nginxFormatStr) { diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java index b1fc219d6..331e250b7 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java @@ -22,13 +22,13 @@ public RawLogParser(LogParserData parserData) { @Override public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - Map ret = parseSimple(logData, collectStamp); + Map ret = doParseSimple(logData, collectStamp); checkMessageExist(ret, logData); return ret; } @Override - public Map parseSimple(String logData, Long collectStamp) { + public Map doParseSimple(String logData, Long collectStamp) { Map ret = new HashMap<>(); ret.put(ES_KEY_MAP_MESSAGE, logData); if (null != collectStamp) { diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java index 687ad4487..30e910dfe 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java @@ -40,11 +40,11 @@ public RegexLogParser(LogParserData parserData) { @Override public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return parseSimple(logData, collectStamp); + return doParseSimple(logData, collectStamp); } @Override - public Map parseSimple(String logData, Long collectStamp) { + public Map doParseSimple(String logData, Long collectStamp) { Map ret = new HashMap<>(); if (logData == null || logData.length() == 0) { return ret; diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java index f698add7d..973de71e5 100644 --- a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java +++ b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java @@ -35,11 +35,11 @@ public SeparatorLogParser(LogParserData parserData) { @Override public Map doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) { - return parseSimple(logData, collectStamp); + return doParseSimple(logData, collectStamp); } @Override - public Map parseSimple(String logData, Long collectStamp) { + public Map doParseSimple(String logData, Long collectStamp) { Map ret = new HashMap<>(); if (logData == null) { return ret;