fix gzip recorder and optimize wiki image

README.md

@@ -22,7 +22,7 @@ Sharingan（写轮眼）是一个基于golang的流量录制回放工具，录
 * 录制线上服务真实请求流量（包括下游调用流量），在线下进行回放，解决构造测试数据难的问题。「复制能力」
 * 回放的时候匹配Mock下游调用，不再依赖具体的下游服务，解决维护测试环境成本高的问题。「幻术能力」
 
-### 1.3、优势
+### 1.3、特性
 
 * 支持下游流量录制。相比[tcpcopy](https://github.com/session-replay-tools/tcpcopy)、[goreplay](https://github.com/buger/goreplay)等方案，回放不依赖下游环境。
 * 支持并发流量录制和回放。录制对服务影响小，回放速度更快。

doc/replayer/replayer-theory.md

@@ -35,7 +35,7 @@ func Connect(fd int, sa Sockaddr) (err error) {
 
 [sharingan/replayer](../../replayer) 包拦截了SUT的Outbound请求，将其转发给Replayer-Agent的Mock Server。
 
-![replay-theory](http://img-hxy021.didistatic.com/static/sharingan/replay_theory_v2.png)
+![replay-theory](http://img-hxy021.didistatic.com/static/sharingan/replay_theory.png)
 
 如上图，回放剧本的传递过程如下：
   1. 用户浏览Web Server首页(:8998)，筛选一个流量，点击回放
@@ -66,7 +66,7 @@ Mock Server有个非常重要的工作，就是匹配Outbound请求，这直接
 
 下面简化下匹配算法核心步骤：
 
-![replay-match](http://img-hxy021.didistatic.com/static/sharingan/replay_match_v2.png)
+![replay-match](http://img-hxy021.didistatic.com/static/sharingan/replay_match.png)
 
 a) 匹配当前请求时，若上一次匹配成功，则从上一次匹配成功的请求（lastMatchedIndex）的下一个请求开始匹配，否则就从第一个请求开始匹配；
 

recorder/recording/action.go

@@ -1,11 +1,7 @@
 package recording
 
 import (
-	"bytes"
-	"compress/gzip"
-	"encoding/binary"
 	"encoding/json"
-	"io/ioutil"
 	"net"
 	"unicode/utf8"
 )
@@ -67,7 +63,7 @@ func (returnInbound *ReturnInbound) MarshalJSON() ([]byte, error) {
 		Response json.RawMessage
 	}{
 		ReturnInbound: *returnInbound,
-		Response:      EncodeAnyByteArray(UnzipHttpRepsonse(returnInbound.Response)),
+		Response:      EncodeAnyByteArray(returnInbound.Response),
 	})
 }
 
@@ -94,7 +90,7 @@ func (callOutbound *CallOutbound) MarshalJSON() ([]byte, error) {
 	}{
 		CallOutbound: *callOutbound,
 		Request:      EncodeAnyByteArray(callOutbound.Request),
-		Response:     EncodeAnyByteArray(UnzipHttpRepsonse(callOutbound.Response)),
+		Response:     EncodeAnyByteArray(callOutbound.Response),
 		CSpanId:      EncodeAnyByteArray(callOutbound.CSpanId),
 	})
 }
@@ -313,45 +309,3 @@ func EncodeAnyByteArray(s []byte) json.RawMessage {
 	encoded = append(encoded, '"')
 	return json.RawMessage(encoded)
 }
-
-// UnzipHttpRepsonse 解析http gzip返回
-func UnzipHttpRepsonse(data []byte) []byte {
-	var err error
-
-	// only gzip response
-	if !bytes.Contains(data, []byte("Content-Encoding: gzip")) {
-		return data
-	}
-
-	// 获取body，以$bodySplit分割
-	bodySplit := []byte("\r\n\r\n")
-	contents := bytes.Split(data, bodySplit)
-	if len(contents) != 2 {
-		return data
-	}
-	body := contents[1]
-
-	// 替换body
-	if contents[1], err = parseGzip(body); err == nil {
-		return bytes.Join(contents, bodySplit)
-	}
-
-	return data
-}
-
-func parseGzip(data []byte) ([]byte, error) {
-	b := new(bytes.Buffer)
-	binary.Write(b, binary.LittleEndian, data)
-
-	r, err := gzip.NewReader(b)
-	if err != nil {
-		return data, err
-	}
-
-	zipData, err := ioutil.ReadAll(r)
-	if err != nil {
-		return data, err
-	}
-
-	return zipData, nil
-}

replayer-agent/common/handlers/path/init.go

@@ -0,0 +1,17 @@
+package path
+
+import (
+	"log"
+	"os"
+)
+
+// Root current dir
+var Root string
+
+func init() {
+	var err error
+	Root, err = os.Getwd()
+	if err != nil {
+		log.Fatal("Initialize Root error: ", err)
+	}
+}

replayer-agent/common/handlers/tlog/init.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/didi/sharingan/recorder-agent/common/conf"
-	"github.com/didi/sharingan/recorder-agent/common/path"
+	"github.com/didi/sharingan/replayer-agent/common/handlers/conf"
+	"github.com/didi/sharingan/replayer-agent/common/handlers/path"
 
 	rotatelogs "github.com/lestrrat-go/file-rotatelogs"
 	"go.uber.org/zap"

replayer-agent/logic/outbound/connection.go

@@ -3,7 +3,6 @@ package outbound
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"net"
 	"regexp"
 	"strconv"
@@ -142,8 +141,6 @@ func (cs *ConnState) rmTrafixPrefix(ctx context.Context, request []byte) []byte
 
 		// 分段传输的场景，要把所有的前缀相关内容去掉
 		request = bytes.Replace(request, ss[0][0], []byte(""), -1)
-		// fmt.Printf("cs.traceID:%s, cs.proxyAddr:%s\n", cs.traceID, cs.proxyAddr)
-		// fmt.Printf("buf:%s\n", string(request))
 	}
 
 	return request
@@ -201,14 +198,12 @@ func (cs *ConnState) match(ctx context.Context, request []byte) error {
 		}
 		response := callOutbound.MatchedResponse
 		response = bytes.Replace(response, []byte("Connection: keep-alive\r\n"), []byte("Connection: close\r\n"), -1)
-		response = resetContentLength(ctx, response)
 		_, err := cs.conn.Write(response)
 		if err != nil {
 			tlog.Handler.Errorf(ctx, tlog.DebugTag, "errmsg=write back response failed||err=%s", err)
 			return err
 		}
 	}
-	//TODO: 为啥不直接放到上面if的elseif里
 	// set matched id as ActionIndex for simulateHttp|simulateMysql
 	if callOutbound.MatchedActionIndex < 0 && matchedTalk != nil {
 		callOutbound.MatchedActionIndex = matchedTalk.ActionIndex
@@ -234,27 +229,6 @@ func (cs *ConnState) match(ctx context.Context, request []byte) error {
 	return nil
 }
 
-// resetContentLength, 重新计算
-func resetContentLength(ctx context.Context, data []byte) []byte {
-	var contents [][]byte
-
-	if !bytes.Contains(data, []byte("Content-Encoding: gzip\r\n")) {
-		return data
-	}
-
-	bodySplit := []byte("\r\n\r\n")
-	if contents = bytes.Split(data, bodySplit); len(contents) != 2 {
-		return data
-	}
-
-	// 因为线上gzip的原因，Content-Length可能会减少
-	newLength := fmt.Sprintf("Content-Length: %d\r\n", len(contents[1])) // 计算body长度
-	data = contentLengthRegex.ReplaceAll(data, []byte(newLength))
-	data = bytes.Replace(data, []byte("Content-Encoding: gzip\r\n"), []byte(""), -1)
-
-	return data
-}
-
 // applySimulation
 func applySimulation(ctx context.Context, sim func(ctx context.Context, request []byte) []byte,
 	request []byte, conn net.Conn, callOutbound *replaying.CallOutbound) error {

replayer-agent/model/esmodel/session.go

@@ -2,11 +2,9 @@ package esmodel
 
 import (
 	"net"
-	"strconv"
 
 	"github.com/didi/sharingan/replayer-agent/utils/helper"
-
-	"github.com/json-iterator/go"
+	jsoniter "github.com/json-iterator/go"
 )
 
 func RetrieveSessions(data []byte) ([]Session, error) {
@@ -77,7 +75,8 @@ type CallFromInbound struct {
 
 func (r *Raw) UnmarshalJSON(data []byte) error {
 	// step1: unquote string
-	tmp, err := strconv.Unquote(helper.BytesToString(data))
+	// tmp, err := strconv.Unquote(helper.BytesToString(data))
+	tmp, err := Unquote(data)
 	if err != nil {
 		return err
 	}

replayer-agent/model/esmodel/unquote.go

@@ -0,0 +1,154 @@
+package esmodel
+
+import (
+	"errors"
+	"unicode"
+	"unicode/utf16"
+	"unicode/utf8"
+)
+
+// Unquote copy from json.unquote
+func Unquote(s []byte) (t string, err error) {
+	s, ok := unquoteBytes(s)
+	if !ok {
+		return "", errors.New("json.unquote false")
+	}
+
+	t = string(s)
+	return
+}
+
+func unquoteBytes(s []byte) (t []byte, ok bool) {
+	if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
+		return
+	}
+	s = s[1 : len(s)-1]
+
+	// Check for unusual characters. If there are none,
+	// then no unquoting is needed, so return a slice of the
+	// original bytes.
+	r := 0
+	for r < len(s) {
+		c := s[r]
+		if c == '\\' || c == '"' || c < ' ' {
+			break
+		}
+		if c < utf8.RuneSelf {
+			r++
+			continue
+		}
+		rr, size := utf8.DecodeRune(s[r:])
+		if rr == utf8.RuneError && size == 1 {
+			break
+		}
+		r += size
+	}
+	if r == len(s) {
+		return s, true
+	}
+
+	b := make([]byte, len(s)+2*utf8.UTFMax)
+	w := copy(b, s[0:r])
+	for r < len(s) {
+		// Out of room? Can only happen if s is full of
+		// malformed UTF-8 and we're replacing each
+		// byte with RuneError.
+		if w >= len(b)-2*utf8.UTFMax {
+			nb := make([]byte, (len(b)+utf8.UTFMax)*2)
+			copy(nb, b[0:w])
+			b = nb
+		}
+		switch c := s[r]; {
+		case c == '\\':
+			r++
+			if r >= len(s) {
+				return
+			}
+			switch s[r] {
+			default:
+				return
+			case '"', '\\', '/', '\'':
+				b[w] = s[r]
+				r++
+				w++
+			case 'b':
+				b[w] = '\b'
+				r++
+				w++
+			case 'f':
+				b[w] = '\f'
+				r++
+				w++
+			case 'n':
+				b[w] = '\n'
+				r++
+				w++
+			case 'r':
+				b[w] = '\r'
+				r++
+				w++
+			case 't':
+				b[w] = '\t'
+				r++
+				w++
+			case 'u':
+				r--
+				rr := getu4(s[r:])
+				if rr < 0 {
+					return
+				}
+				r += 6
+				if utf16.IsSurrogate(rr) {
+					rr1 := getu4(s[r:])
+					if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
+						// A valid pair; consume.
+						r += 6
+						w += utf8.EncodeRune(b[w:], dec)
+						break
+					}
+					// Invalid surrogate; fall back to replacement rune.
+					rr = unicode.ReplacementChar
+				}
+				w += utf8.EncodeRune(b[w:], rr)
+			}
+
+		// Quote, control characters are invalid.
+		case c == '"', c < ' ':
+			return
+
+		// ASCII
+		case c < utf8.RuneSelf:
+			b[w] = c
+			r++
+			w++
+
+		// Coerce to well-formed UTF-8.
+		default:
+			rr, size := utf8.DecodeRune(s[r:])
+			r += size
+			w += utf8.EncodeRune(b[w:], rr)
+		}
+	}
+	return b[0:w], true
+}
+
+func getu4(s []byte) rune {
+	if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
+		return -1
+	}
+	var r rune
+	for _, c := range s[2:6] {
+		switch {
+		case '0' <= c && c <= '9':
+			c = c - '0'
+		case 'a' <= c && c <= 'f':
+			c = c - 'a' + 10
+		case 'A' <= c && c <= 'F':
+			c = c - 'A' + 10
+		default:
+			return -1
+		}
+		r = r*16 + rune(c)
+	}
+	return r
+}

replayer-agent/model/esmodel/unquote_test.go

@@ -0,0 +1,27 @@
+package esmodel
+
+import "testing"
+
+func TestUnquote(t *testing.T) {
+	type args struct {
+		s []byte
+	}
+	tests := []struct {
+		name  string
+		args  args
+		wantT string
+	}{
+		{"1", args{s: []byte(`"\uD925\uDFA1"`)}, "񙞡"},
+		{"2", args{s: []byte(`"\u4e16\u754c"`)}, "世界"},
+		{"3", args{s: []byte(`"abc123"`)}, "abc123"},
+		{"4", args{s: []byte(`""`)}, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotT, _ := Unquote(tt.args.s)
+			if gotT != tt.wantT {
+				t.Errorf("Unquote() gotT = %v, want %v", gotT, tt.wantT)
+			}
+		})
+	}
+}