feat: parse http2 data in text mode (#580)

* h2 demo
* gzip
* http2 request test
* update test file
* update test
* update bin file

pkg/event_processor/http2_request.go

@@ -0,0 +1,158 @@
+// Copyright 2024 yuweizzz <[email protected]>. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event_processor
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/hpack"
+)
+
+const H2Magic = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
+const H2MagicLen = len(H2Magic)
+
+type HTTP2Request struct {
+	framer     *http2.Framer
+	packerType PacketType
+	isDone     bool
+	isInit     bool
+	reader     *bytes.Buffer
+	bufReader  *bufio.Reader
+}
+
+func (h2r *HTTP2Request) detect(payload []byte) error {
+	data := string(payload[0:H2MagicLen])
+	if data != H2Magic {
+		return errors.New("Not match http2 magic")
+	}
+	return nil
+}
+
+func (h2r *HTTP2Request) Init() {
+	h2r.reader = bytes.NewBuffer(nil)
+	h2r.bufReader = bufio.NewReader(h2r.reader)
+	h2r.framer = http2.NewFramer(nil, h2r.bufReader)
+	h2r.framer.ReadMetaHeaders = hpack.NewDecoder(0, nil)
+}
+
+func (h2r *HTTP2Request) Write(b []byte) (int, error) {
+	if !h2r.isInit {
+		h2r.Init()
+		h2r.isInit = true
+	}
+	length, err := h2r.reader.Write(b)
+	if err != nil {
+		return 0, err
+	}
+	return length, nil
+}
+
+func (h2r *HTTP2Request) ParserType() ParserType {
+	return ParserTypeHttp2Request
+}
+
+func (h2r *HTTP2Request) PacketType() PacketType {
+	return h2r.packerType
+}
+
+func (h2r *HTTP2Request) Name() string {
+	return "HTTP2Request"
+}
+
+func (h2r *HTTP2Request) IsDone() bool {
+	return h2r.isDone
+}
+
+func (h2r *HTTP2Request) Display() []byte {
+	_, err := h2r.bufReader.Discard(H2MagicLen)
+	if err != nil {
+		log.Println("[http2 request] Discard HTTP2 Magic error:", err)
+		return h2r.reader.Bytes()
+	}
+	var encoding string
+	dataBuf := bytes.NewBuffer(nil)
+	frameBuf := bytes.NewBufferString("")
+	for {
+		f, err := h2r.framer.ReadFrame()
+		if err != nil {
+			if err != io.EOF {
+				log.Println("[http2 request] Dump HTTP2 Frame error:", err)
+			}
+			break
+		}
+		switch f := f.(type) {
+		case *http2.MetaHeadersFrame:
+			frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tHEADERS\n"))
+			for _, header := range f.Fields {
+				frameBuf.WriteString(fmt.Sprintf("%s\n", header.String()))
+				if header.Name == "content-encoding" {
+					encoding = header.Value
+				}
+			}
+		case *http2.DataFrame:
+			_, err := dataBuf.Write(f.Data())
+			if err != nil {
+				log.Println("[http2 request] Write HTTP2 Data Frame buffuer error:", err)
+			}
+		default:
+			fh := f.Header()
+			frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\t%s\n", fh.Type.String()))
+		}
+	}
+	// merge data frame
+	if dataBuf.Len() > 0 {
+		frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tDATA\n"))
+		payload := dataBuf.Bytes()
+		switch encoding {
+		case "gzip":
+			reader, err := gzip.NewReader(bytes.NewReader(payload))
+			if err != nil {
+				log.Println("[http2 request] Create gzip reader error:", err)
+				break
+			}
+			payload, err = io.ReadAll(reader)
+			if err != nil {
+				log.Println("[http2 request] Uncompress gzip data error:", err)
+				break
+			}
+			h2r.packerType = PacketTypeGzip
+			defer reader.Close()
+		default:
+			h2r.packerType = PacketTypeNull
+		}
+		frameBuf.Write(payload)
+	}
+	return frameBuf.Bytes()
+}
+
+func init() {
+	h2r := &HTTP2Request{}
+	h2r.Init()
+	Register(h2r)
+}
+
+func (h2r *HTTP2Request) Reset() {
+	h2r.isDone = false
+	h2r.isInit = false
+	h2r.reader.Reset()
+	h2r.bufReader.Reset(h2r.reader)
+}

pkg/event_processor/http2_responese.go

@@ -0,0 +1,153 @@
+// Copyright 2024 yuweizzz <[email protected]>. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event_processor
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"log"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/hpack"
+)
+
+type HTTP2Response struct {
+	framer     *http2.Framer
+	packerType PacketType
+	isDone     bool
+	isInit     bool
+	reader     *bytes.Buffer
+	bufReader  *bufio.Reader
+}
+
+func (h2r *HTTP2Response) detect(payload []byte) error {
+	rd := bytes.NewReader(payload)
+	buf := bufio.NewReader(rd)
+	framer := http2.NewFramer(nil, buf)
+	framer.ReadMetaHeaders = hpack.NewDecoder(0, nil)
+	_, err := framer.ReadFrame()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (h2r *HTTP2Response) Init() {
+	h2r.reader = bytes.NewBuffer(nil)
+	h2r.bufReader = bufio.NewReader(h2r.reader)
+	h2r.framer = http2.NewFramer(nil, h2r.bufReader)
+	h2r.framer.ReadMetaHeaders = hpack.NewDecoder(0, nil)
+}
+
+func (h2r *HTTP2Response) Write(b []byte) (int, error) {
+	if !h2r.isInit {
+		h2r.Init()
+		h2r.isInit = true
+	}
+	length, err := h2r.reader.Write(b)
+	if err != nil {
+		return 0, err
+	}
+	return length, nil
+}
+
+func (h2r *HTTP2Response) ParserType() ParserType {
+	return ParserTypeHttp2Response
+}
+
+func (h2r *HTTP2Response) PacketType() PacketType {
+	return h2r.packerType
+}
+
+func (h2r *HTTP2Response) Name() string {
+	return "HTTP2Response"
+}
+
+func (h2r *HTTP2Response) IsDone() bool {
+	return h2r.isDone
+}
+
+func (h2r *HTTP2Response) Display() []byte {
+	var encoding string
+	dataBuf := bytes.NewBuffer(nil)
+	frameBuf := bytes.NewBufferString("")
+	for {
+		f, err := h2r.framer.ReadFrame()
+		if err != nil {
+			if err != io.EOF {
+				log.Println("[http2 response] Dump HTTP2 Frame error:", err)
+			}
+			break
+		}
+		switch f := f.(type) {
+		case *http2.MetaHeadersFrame:
+			frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tHEADERS\n"))
+			for _, header := range f.Fields {
+				frameBuf.WriteString(fmt.Sprintf("%s\n", header.String()))
+				if header.Name == "content-encoding" {
+					encoding = header.Value
+				}
+			}
+		case *http2.DataFrame:
+			_, err := dataBuf.Write(f.Data())
+			if err != nil {
+				log.Println("[http2 response] Write HTTP2 Data Frame buffuer error:", err)
+			}
+		default:
+			fh := f.Header()
+			frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\t%s\n", fh.Type.String()))
+		}
+	}
+	// merge data frame
+	if dataBuf.Len() > 0 {
+		frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tDATA\n"))
+		payload := dataBuf.Bytes()
+		switch encoding {
+		case "gzip":
+			reader, err := gzip.NewReader(bytes.NewReader(payload))
+			if err != nil {
+				log.Println("[http2 response] Create gzip reader error:", err)
+				break
+			}
+			payload, err = io.ReadAll(reader)
+			if err != nil {
+				log.Println("[http2 response] Uncompress gzip data error:", err)
+				break
+			}
+			h2r.packerType = PacketTypeGzip
+			defer reader.Close()
+		default:
+			h2r.packerType = PacketTypeNull
+		}
+		frameBuf.Write(payload)
+	}
+	return frameBuf.Bytes()
+}
+
+func init() {
+	h2r := &HTTP2Response{}
+	h2r.Init()
+	Register(h2r)
+}
+
+func (h2r *HTTP2Response) Reset() {
+	h2r.isDone = false
+	h2r.isInit = false
+	h2r.reader.Reset()
+	h2r.bufReader.Reset(h2r.reader)
+}

pkg/event_processor/iparser.go

@@ -94,11 +94,9 @@ func NewParser(payload []byte) IParser {
 				case ParserTypeHttpResponse:
 					newParser = new(HTTPResponse)
 				case ParserTypeHttp2Request:
-					// TODO support HTTP2 request
-					// via golang.org/x/net/http2
-					//hpack.NewEncoder(buf)
+					newParser = new(HTTP2Request)
 				case ParserTypeHttp2Response:
-					// TODO  support HTTP2 response
+					newParser = new(HTTP2Response)
 				default:
 					newParser = new(DefaultParser)
 				}

pkg/event_processor/processor_test.go

@@ -84,15 +84,35 @@ func TestEventProcessor_Serve(t *testing.T) {
 
 	lines = strings.Split(string(bufString), "\n")
 	ok := true
+	h2Req := 0
+	h2Resq := 0
 	for _, line := range lines {
 		if strings.Contains(strings.ToLower(line), "dump") {
 			t.Log(line)
 			ok = false
 		}
+		// http2 parse error log
+		if strings.Contains(line, "[http2 re") {
+			t.Log(line)
+			ok = false
+		}
+		// http2 parse count
+		if strings.Contains(line, "Name:HTTP2Response") {
+			h2Resq += 1
+		}
+		if strings.Contains(line, "Name:HTTP2Request") {
+			h2Req += 1
+		}
 	}
 	if err != nil {
 		t.Fatalf("close error: %s", err.Error())
 	}
+	if h2Resq != 3 {
+		t.Fatalf("some errors occurred: HTTP2Response lack")
+	}
+	if h2Req != 2 {
+		t.Fatalf("some errors occurred: HTTP2Request lack")
+	}
 	if !ok {
 		t.Fatalf("some errors occurred")
 	}

pkg/event_processor/testdata/all.json

@@ -13,3 +13,8 @@
 {"DataType":1,"Timestamp":952293599178141,"Pid":469960,"Tid":469960,"DataLen":77,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
 {"DataType":0,"Timestamp":952293616935735,"Pid":469960,"Tid":469960,"DataLen":1179,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
 {"DataType":0,"Timestamp":952293617095621,"Pid":469960,"Tid":469960,"DataLen":1664,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
+{"DataType":1,"Timestamp":952293616935736,"Pid":469961,"Tid":469961,"DataLen":110,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
+{"DataType":0,"Timestamp":952293616935737,"Pid":469963,"Tid":469963,"DataLen":406,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
+{"DataType":1,"Timestamp":952293616935738,"Pid":469962,"Tid":469962,"DataLen":193,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
+{"DataType":0,"Timestamp":952293616935739,"Pid":469964,"Tid":469964,"DataLen":545,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
+{"DataType":0,"Timestamp":952293616935740,"Pid":469965,"Tid":469965,"DataLen":384,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}