Add post processing service for log content (apache#153)

shanwb · Nov 29, 2023 · 8bebb11 · 8bebb11
1 parent dac3ea1
commit 8bebb11
Show file tree

Hide file tree

Showing 9 changed files with 60 additions and 23 deletions.
diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/AbstractLogParser.java
@@ -1,22 +1,47 @@
 package com.xiaomi.mone.log.parse;
 
+import cn.hutool.core.util.ClassUtil;
+import cn.hutool.core.util.ReflectUtil;
+import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * @author wtt
  * @version 1.0
  * @description
  * @date 2023/10/31 16:32
  */
+@Slf4j
 public abstract class AbstractLogParser implements LogParser {
 
     protected LogParserData parserData;
 
+    private List<FieldInterceptor> fieldInterceptors = Lists.newArrayList();
+
     public AbstractLogParser(LogParserData parserData) {
         this.parserData = parserData;
+        createFieldInterceptors();
+    }
+
+    private void createFieldInterceptors() {
+        try {
+            Set<Class<?>> implementationClasses = ClassUtil.scanPackageBySuper(PACKAGE_NAME, FieldInterceptor.class);
+
+            fieldInterceptors = implementationClasses.stream()
+                    .map(ReflectUtil::newInstance)
+                    .filter(obj -> obj instanceof FieldInterceptor)
+                    .map(obj -> (FieldInterceptor) obj)
+                    .collect(Collectors.toList());
+        } catch (Exception e) {
+            log.error("createFieldInterceptors", e);
+        }
     }
 
     @Override
@@ -26,20 +51,16 @@ public Map<String, Object> parse(String logData, String ip, Long lineNum, Long c
         wrapMap(parseData, parserData, ip, lineNum, fileName, collectStamp);
         checkMessageExist(parseData, logData);
         validRet(parseData, logData);
-        handleImproperTraceId(parseData);
+
+        fieldInterceptors.stream().forEach(fieldInterceptor -> fieldInterceptor.postProcess(parseData));
         return parseData;
     }
 
-    private void handleImproperTraceId(Map<String, Object> parseData) {
-        if (parseData.containsKey(TRACE_ID_KEY)) {
-            Object traceIdValue = parseData.get(TRACE_ID_KEY);
-            if (traceIdValue instanceof String) {
-                String traceId = (String) traceIdValue;
-                if (traceId.startsWith(TRACE_ID_KEY + "=")) {
-                    parseData.put(TRACE_ID_KEY, traceId.substring(TRACE_ID_KEY.length() + 1));
-                }
-            }
-        }
+    @Override
+    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
+        Map<String, Object> parseData = doParseSimple(logData, collectStamp);
+        fieldInterceptors.stream().forEach(fieldInterceptor -> fieldInterceptor.postProcess(parseData));
+        return parseData;
     }
 
     protected void validTimestamp(Map<String, Object> ret, Long collectStamp) {
@@ -106,4 +127,5 @@ void validRet(Map<String, Object> ret, String logData) {
 
     public abstract Map<String, Object> doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName);
 
+    public abstract Map<String, Object> doParseSimple(String logData, Long collectStamp);
 }
diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java
@@ -52,11 +52,11 @@ public CustomLogParser(LogParserData parserData) {
 
     @Override
     public Map<String, Object> doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) {
-        return parseSimple(logData, collectStamp);
+        return doParseSimple(logData, collectStamp);
     }
 
     @Override
-    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
+    public Map<String, Object> doParseSimple(String logData, Long collectStamp) {
         Map<String, Object> ret = new HashMap<>();
         String originData = logData;
         if (logData == null) {

diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/FieldInterceptor.java
@@ -0,0 +1,14 @@
+package com.xiaomi.mone.log.parse;
+
+import java.util.Map;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/11/17 11:17
+ */
+public interface FieldInterceptor {
+
+    void postProcess(Map<String, Object> data);
+}
diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/JsonLogParser.java
@@ -41,11 +41,11 @@ public JsonLogParser(LogParserData parserData) {
 
     @Override
     public Map<String, Object> doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) {
-        return parseSimple(logData, collectStamp);
+        return doParseSimple(logData, collectStamp);
     }
 
     @Override
-    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
+    public Map<String, Object> doParseSimple(String logData, Long collectStamp) {
         Map<String, Object> ret = new HashMap<>();
         if (logData == null || logData.length() == 0) {
             return ret;

diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/LogParser.java
@@ -58,6 +58,7 @@ public interface LogParser {
     String esKeyMap_lineNumber = "linenumber";
     String esKyeMap_fileName = "filename";
     String TRACE_ID_KEY = "traceId";
+    String PACKAGE_NAME = "com.xiaomi.mone.log";
 
     Map<String, Object> parse(String logData, String ip, Long lineNum, Long collectStamp, String fileName);
 

diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/NginxLogParser.java
@@ -46,8 +46,8 @@ public Map<String, Object> doParse(String logData, String ip, Long lineNum, Long
     }
 
     @Override
-    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
-        return this.regexLogParser.parseSimple(logData, collectStamp);
+    public Map<String, Object> doParseSimple(String logData, Long collectStamp) {
+        return this.regexLogParser.doParseSimple(logData, collectStamp);
     }
 
     public String generateRegexFromNginxScript(String nginxFormatStr) {

diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RawLogParser.java
@@ -22,13 +22,13 @@ public RawLogParser(LogParserData parserData) {
 
     @Override
     public Map<String, Object> doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) {
-        Map<String, Object> ret = parseSimple(logData, collectStamp);
+        Map<String, Object> ret = doParseSimple(logData, collectStamp);
         checkMessageExist(ret, logData);
         return ret;
     }
 
     @Override
-    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
+    public Map<String, Object> doParseSimple(String logData, Long collectStamp) {
         Map<String, Object> ret = new HashMap<>();
         ret.put(ES_KEY_MAP_MESSAGE, logData);
         if (null != collectStamp) {

diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/RegexLogParser.java
@@ -40,11 +40,11 @@ public RegexLogParser(LogParserData parserData) {
 
     @Override
     public Map<String, Object> doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) {
-        return parseSimple(logData, collectStamp);
+        return doParseSimple(logData, collectStamp);
     }
 
     @Override
-    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
+    public Map<String, Object> doParseSimple(String logData, Long collectStamp) {
         Map<String, Object> ret = new HashMap<>();
         if (logData == null || logData.length() == 0) {
             return ret;

diff --git a/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java b/ozhera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/SeparatorLogParser.java
@@ -35,11 +35,11 @@ public SeparatorLogParser(LogParserData parserData) {
 
     @Override
     public Map<String, Object> doParse(String logData, String ip, Long lineNum, Long collectStamp, String fileName) {
-       return parseSimple(logData, collectStamp);
+        return doParseSimple(logData, collectStamp);
     }
 
     @Override
-    public Map<String, Object> parseSimple(String logData, Long collectStamp) {
+    public Map<String, Object> doParseSimple(String logData, Long collectStamp) {
         Map<String, Object> ret = new HashMap<>();
         if (logData == null) {
             return ret;