From 8cfdf8bc0d12d1fc44cf8cf3378f748cb2f0a6e7 Mon Sep 17 00:00:00 2001 From: yangjing Date: Fri, 17 Apr 2020 18:44:44 +0800 Subject: [PATCH] add go mod shell & update md --- .gitignore | 2 +- doc/replayer/README.md | 43 +-- doc/replayer/conf/http_api.md | 8 +- doc/replayer/conf/moduleinfo.md | 10 +- doc/replayer/guide/replay.md | 2 +- doc/replayer/guide/report.md | 12 +- doc/replayer/guide/troubleshoot.md | 6 +- doc/replayer/replayer-agent.md | 38 ++- doc/replayer/replayer-codecov.md | 10 +- doc/replayer/replayer-conf.md | 12 +- doc/replayer/replayer-local.md | 4 +- doc/replayer/replayer-parallel.md | 10 +- doc/replayer/replayer-sut.md | 38 ++- doc/replayer/replayer-theory.md | 6 +- example/replayer/sut_replayer.sh | 164 +++++++---- example/replayer/sut_replayer_glide.sh | 383 +++++++++++++++++++++++++ example/replayer/sut_replayer_gomod.sh | 367 +++++++++++++++++++++++ replayer-agent/control.sh | 114 ++++++-- replayer-agent/control_glide.sh | 225 +++++++++++++++ replayer-agent/control_gomod.sh | 212 ++++++++++++++ 20 files changed, 1502 insertions(+), 164 deletions(-) create mode 100755 example/replayer/sut_replayer_glide.sh create mode 100755 example/replayer/sut_replayer_gomod.sh create mode 100755 replayer-agent/control_glide.sh create mode 100755 replayer-agent/control_gomod.sh diff --git a/.gitignore b/.gitignore index 28d979d..98dd846 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ example/replayer/example.test example/replayer/example example/replayer/sut.pid example/replayer/sut.log -example/replayer/gocov +example/replayer/gocov* example/replayer/gocov-html example/replayer/nohup.out diff --git a/doc/replayer/README.md b/doc/replayer/README.md index 5239427..57c1d49 100644 --- a/doc/replayer/README.md +++ b/doc/replayer/README.md @@ -11,11 +11,11 @@ * **Inbound**: 指SUT对外提供的HTTP接口的请求和响应，即Inbound Request/Response。 * **Outbound**: 指SUT提供的http接口内调用的下游请求和响应，即Outboud Request/Response。目前支持的Outbound协议有 MYSQL、REDIS、HTTP、Binary Thrift、Compact Thrift。 * **流量**: 指在TCP层对SUT录制的Inbound请求/响应 和 对应的Outbound请求/响应。 -* **回放**: 基于录制的一条流量，Agent根据Inbound Request构造HTTP Request并对SUT发起请求；其中SUT的Outbound请求会发送到Mock Server进行流量匹配并返回Outbound响应；最后，Agent将收到的SUT HTTP Response与Inbound Response做对比，给出回放结果。 -* **Agent**: 包括回放过程的Web Server和Mock Server。其中Web Server默认监听8998端口，主要负责流量搜索，回放请求的构造和发起，以及回放结果的对比和展示。 +* **回放**: 基于录制的一条流量，Replayer-Agent根据Inbound Request构造HTTP Request并对SUT发起请求；其中SUT的Outbound请求会发送到Mock Server进行流量匹配并返回Outbound响应；最后，Replayer-Agent将收到的SUT HTTP Response与Inbound Response做对比，给出回放结果。 +* **Replayer-Agent**: 包括回放过程的Web Server和Mock Server。其中Web Server默认监听8998端口，主要负责流量搜索，回放请求的构造和发起，以及回放结果的对比和展示。 * **Mock Server**: 默认监听**3515**端口，主要负责接收SUT的下游请求，然后与回放流量的Outbound请求匹配，将最匹配的Outbound响应返回SUT。 -* **噪音**: 即在Inbound Response对比和Outbound Request匹配过程里，出现的不影响回放结果和匹配度的diff字段，如时间戳。Agent回放结果页支持上报噪音，方便再次回放时精确回放结果。 -* **DSL**: 特指ElasticSearch的查询DSL。Agent首页支持上报流量查询DSL，方便后面直接复用。 +* **噪音**: 即在Inbound Response对比和Outbound Request匹配过程里，出现的不影响回放结果和匹配度的diff字段，如时间戳。Replayer-Agent回放结果页支持上报噪音，方便再次回放时精确回放结果。 +* **DSL**: 特指ElasticSearch的查询DSL。Replayer-Agent首页支持上报流量查询DSL，方便后面直接复用。
@@ -24,25 +24,26 @@ 回放前提：已经完成流量录制。[录制接入文档](../recorder/README.md) 对于服务启动阶段有TCP请求的SUT，如初始化连接池等，推荐 服务启动顺序： -* 先启动Agent +* 先启动Replayer-Agent * 再启动SUT -#### 1. 配置并启动Agent +#### 1. 配置并启动Replayer-Agent -> 温馨提示：Agent默认配置的是 **[本地回放](#4本地回放)**，即 仅依赖本地配置文件的回放。 -* 如需 读取录制的线上流量，只需根据[回放Agent配置](./replayer-conf.md#5-es_url)修改es_url字段即可； -* 如需 读取上报到自有服务的噪音或DSL，只需根据[回放Agent配置](./replayer-conf.md#4-http_api)修改http_api字段即可； +> 温馨提示：Replayer-Agent默认配置的是 **[本地回放](#4本地回放)**，即 仅依赖本地配置文件的回放。 +* 如需 读取录制的线上流量，只需根据[Replayer-Agent配置](./replayer-conf.md#5-es_url)修改es_url字段即可； +* 如需 读取上报到自有服务的噪音或DSL，只需根据[Replayer-Agent配置](./replayer-conf.md#4-http_api)修改http_api字段即可； * 如需 新增本地回放模块，请参考[本地回放接入](./replayer-local.md)。 首先，确保本地存在go环境(官方或定制版均可，且版本无要求), 并根据本地go版本选择go mod或glide来安装sharingan的依赖。 ```shell script -glide install || go mod download +git clone https://github.com/didichuxing/sharingan.git +&& cd sharingan && go mod download // 低版本go执行 glide install ``` -然后编译并启动Agent： +然后编译并启动Replayer-Agent： ```shell script cd ./replayer-agent && go build && nohup ./replayer-agent >> run.log 2>&1 & ``` -> Agent一键安装和启动 [脚本](../../replayer-agent/control.sh) 及其 [使用方法](./replayer-agent.md) +> Replayer-Agent一键安装和启动 [脚本](../../replayer-agent/control.sh) 及其 [使用方法](./replayer-agent.md)
@@ -50,10 +51,10 @@ cd ./replayer-agent && go build && nohup ./replayer-agent >> run.log 2>&1 & > 需要使用定制的golang，并通过指定tag来编译 引入回放包的SUT代码。 -首先，配置定制版golang环境，目前支持go1.10、go1.11、go1.12、go1.13。亦可参考：[golang安装](https://github.com/didichuxing/sharingan-go) +首先，配置定制版golang环境，目前支持go1.10、go1.11、go1.12、go1.13。亦可参考：[golang安装](https://github.com/didichuxing/sharingan-go/tree/recorder) ```shell script -curl https://github.com/didichuxing/sharingan-go/raw/recorder/install/go1.10 | sh -&& export GOROOT=/tmp/recorder-go1.10 +curl https://github.com/didichuxing/sharingan-go/raw/recorder/install/go1.13 | sh +&& export GOROOT=/tmp/recorder-go1.13 && export PATH=$GOROOT/bin:$PATH ``` @@ -75,7 +76,7 @@ go build -tags="replayer" -gcflags="all=-N -l" ## 三、使用手册 -按照上面的接入流程逐步操作 并 成功启动SUT和Agent后，即可开始回放之旅~ +按照上面的接入流程逐步操作 并 成功启动SUT和Replayer-Agent后，即可开始回放之旅~ 首页点击"流量查询"搜索流量后，即可点击"运行"开始回放。 ![web_index](../images/web_index.png) @@ -108,14 +109,14 @@ go build -tags="replayer" -gcflags="all=-N -l" [本地回放接入](./replayer-local.md) -Agent默认配置的本地回放，如需修改，请参考：[回放Agent配置](./replayer-conf.md#4-http_api) +Replayer-Agent默认配置的本地回放，如需修改，请参考：[Replayer-Agent配置](./replayer-conf.md#4-http_api) -#### 5.Agent和SUT分开部署 +#### 5.Replayer-Agent和SUT分开部署 -如上 二-回放接入，默认将Agent和SUT部署在一台机器，其实，Agent和SUT是可以分开部署的。 +如上 二-回放接入，默认将Replayer-Agent和SUT部署在一台机器，其实，Replayer-Agent和SUT是可以分开部署的。 -* Agent的启动如上所示，无需改动 -* 启动SUT时，需设置环境变量 REPLAYER_MOCK_IP 为Agent的ip地址即可。详见: [SUT启动脚本-3. 与Agent分开部署](./replayer-sut.md#3-与agent分开部署) +* Replayer-Agent的启动如上所示，无需改动 +* 启动SUT时，需设置环境变量 REPLAYER_MOCK_IP 为Replayer-Agent的ip地址即可。详见: [SUT启动脚本-3. 与Replayer-Agent分开部署](./replayer-sut.md#3-与replayer-agent分开部署)
diff --git a/doc/replayer/conf/http_api.md b/doc/replayer/conf/http_api.md index 16d4c6e..0f96f0e 100644 --- a/doc/replayer/conf/http_api.md +++ b/doc/replayer/conf/http_api.md @@ -9,7 +9,7 @@
-回放Agent默认配置的是本地回放；对于想将 噪音、DSL、模块信息存入数据库的同学，请仔细阅读下面接口说明。 +Replayer-Agent默认配置的是本地回放；对于想将 噪音、DSL、模块信息存入数据库的同学，请仔细阅读下面接口说明。 > 温馨提示： > > 接口实现 只要字段名和字段类型符合接口说明即可，**至于字段的具体值无需担心，因为所有的值都会通过前端js处理后传给后端，比如dsl字段、noise字段等**。 @@ -72,7 +72,7 @@ 接口: http://{{your_domain}}/noise -定义 噪音上报接口。在回放结果页，可以上报回放结果里的diff噪音。对于上报过的噪音，回放Agent会在对比回放结果时过滤对应噪音字段，提升回放成功率。若noise_push为空，则 存入本地 [conf/noise](../../../replayer-agent/conf/noise) 下的配置文件。 +定义 噪音上报接口。在回放结果页，可以上报回放结果里的diff噪音。对于上报过的噪音，Replayer-Agent会在对比回放结果时过滤对应噪音字段，提升回放成功率。若noise_push为空，则 存入本地 [conf/noise](../../../replayer-agent/conf/noise) 下的配置文件。 ![push_noise](../../images/push_noise.png) @@ -189,6 +189,6 @@ * listen-addr：必选，SUT的监听地址，一般为127.0.0.1:xxxx。 -* department：可选，模块所属部门，默认空(则为default部门)。非空时，同时流量配置的读取自ES，则 会按部门字段读取es_url地址。es_url配置可详见：[回放Agent配置](../replayer-conf.md#5-es_url) +* department：可选，模块所属部门，默认空(则为default部门)。非空时，同时流量配置的读取自ES，则 会按部门字段读取es_url地址。es_url配置可详见：[Replayer-Agent配置](../replayer-conf.md#5-es_url) -非本地回放时，Agent只用到上面三个key值，如果业务方想在该接口顺便存储模块其他信息，只需扩充[]KV即可。 +非本地回放时，Replayer-Agent只用到上面三个key值，如果业务方想在该接口顺便存储模块其他信息，只需扩充[]KV即可。 diff --git a/doc/replayer/conf/moduleinfo.md b/doc/replayer/conf/moduleinfo.md index 6272db8..bac5961 100644 --- a/doc/replayer/conf/moduleinfo.md +++ b/doc/replayer/conf/moduleinfo.md @@ -18,6 +18,10 @@ { "name":"example", "data":"[{\"key\":\"listen-addr\",\"value\":\"127.0.0.1:9999\"}]" + }, + { + "name": "example2", + "data": "[{\"key\":\"listen-addr\",\"value\":\"10.179.84.1:8888\"},{\"key\":\"department\",\"value\":\"Biz\"}]" } ] } @@ -35,12 +39,16 @@ | name | string | 模块名 | | data | string | []KV类型的json串，存储模块详细信息 | +
+ > 温馨提示： > > name尽量与编译后的$binName保持一致。如果存在冲突，可以增加前缀'\*-'，即\*-$binName。 > > 如果name形如'\*a-b-c'，则尽量保证c具有可识别性，因为，[覆盖率统计回放](../replayer-codecov.md#1-覆盖率报告)会通过 *c* 来获取SUT进程信息并重启SUT。 +
+ | KV类型说明 | 类型 | 说明 | | :-----| :----- | :----- | | key | string | 与模块相关的一些信息，如监听地址listen-addr等 | @@ -49,4 +57,4 @@ 本地回放时，KV类型中key只需支持一个"listen-addr"即可： * listen-addr：必选，SUT的监听地址，一般为127.0.0.1:xxxx。 -* department：可选，模块所属部门，默认空(则为default部门)。非空时，同时流量配置的读取自ES，则 会按部门字段读取es_url地址。es_url配置可详见：[回放Agent配置](../replayer-conf.md#5-es_url) +* department：可选，模块所属部门，默认空(则为default部门)。非空时，同时流量配置的读取自ES，则 会按部门字段读取es_url地址。es_url配置可详见：[Replayer-Agent配置](../replayer-conf.md#5-es_url) diff --git a/doc/replayer/guide/replay.md b/doc/replayer/guide/replay.md index 8e1bb11..7302127 100644 --- a/doc/replayer/guide/replay.md +++ b/doc/replayer/guide/replay.md @@ -26,7 +26,7 @@ ##### 2. 定位失败原因 -回放Agent采用了非常严格的测试标准，只要有任何diff或者异常都会显示回放失败，目的是为了让大家能够发现隐藏问题 +Replayer-Agent采用了非常严格的测试标准，只要有任何diff或者异常都会显示回放失败，目的是为了让大家能够发现隐藏问题 对于回放失败的结果，还可以细分如下： diff --git a/doc/replayer/guide/report.md b/doc/replayer/guide/report.md index bc89b45..67bf38a 100644 --- a/doc/replayer/guide/report.md +++ b/doc/replayer/guide/report.md @@ -2,14 +2,14 @@
-回放过程中，为了提升回放体验、回放效率及回放成功率，回放Agent特意提供了噪音/DSL上报功能。 +回放过程中，为了提升回放体验、回放效率及回放成功率，Replayer-Agent特意提供了噪音/DSL上报功能。
下面详细介绍 DSL和噪音 上报的方法及注意事项： ##### 1. DSL上报 -根据 [流量搜索指南-1. 搜索界面介绍](./search.md#1-搜索界面介绍) 可知，Agent提供的流量搜索条件分的很细。为了节省流量搜索时间，Agent提供了DSL上报功能。 +根据 [流量搜索指南-1. 搜索界面介绍](./search.md#1-搜索界面介绍) 可知，Replayer-Agent提供的流量搜索条件分的很细。为了节省流量搜索时间，Replayer-Agent提供了DSL上报功能。 ![save_dsl](../../images/save_dsl.png) @@ -23,8 +23,8 @@ > 使用上报DSL： - 1. 点击"快速查询"，Agent会列出所有上报的tag。 - 2. 选中一个合适的tag名，Agent会自动填充红框字段，然后直接点击"流量查询"，即可查询流量。 + 1. 点击"快速查询"，Replayer-Agent会列出所有上报的tag。 + 2. 选中一个合适的tag名，Replayer-Agent会自动填充红框字段，然后直接点击"流量查询"，即可查询流量。 > tag命名规则 @@ -34,9 +34,9 @@ tag命名尽量具有可读性，且具有实际意义。比如按 '接口名_ ##### 2. 噪音上报 -流量回放结果里，部分失败的case是由噪音导致。为了解决这个问题，Agent提供了噪音上报功能。 +流量回放结果里，部分失败的case是由噪音导致。为了解决这个问题，Replayer-Agent提供了噪音上报功能。 -对于上报的噪音字段，下次回放时，Agent会过滤由噪音字段导致的diff，给出流量回放成功结果。 +对于上报的噪音字段，下次回放时，Replayer-Agent会过滤由噪音字段导致的diff，给出流量回放成功结果。 ![push_noise](../../images/push_noise.png) > 上报噪音： diff --git a/doc/replayer/guide/troubleshoot.md b/doc/replayer/guide/troubleshoot.md index 6397811..fc0f7b2 100644 --- a/doc/replayer/guide/troubleshoot.md +++ b/doc/replayer/guide/troubleshoot.md @@ -22,9 +22,9 @@ sharingan已经在滴滴内部服务了一批go业务模块，现将各业务模 ##### 2. SUT启动失败 * **现象1**：SUT启动时，提示TCP连接:3515拒绝 -* **原因**：回放Agent服务未启动 +* **原因**：Replayer-Agent服务未启动 -* **解决**：请先启动回放Agent，再启动SUT服务 +* **解决**：请先启动Replayer-Agent，再启动SUT服务 *** @@ -114,7 +114,7 @@ sharingan已经在滴滴内部服务了一批go业务模块，现将各业务模
-* **可能原因2**：连续点击"覆盖率报告"链接。每次点击"覆盖率报告"链接，Agent会自动重启SUT服务，重新统计覆盖率。 +* **可能原因2**：连续点击"覆盖率报告"链接。每次点击"覆盖率报告"链接，Replayer-Agent会自动重启SUT服务，重新统计覆盖率。 * **解决**：请在两次点击"覆盖率报告"链接之间，回放一个或多个流量。[覆盖率统计回放-1. 覆盖率报告](../replayer-codecov.md#1-覆盖率报告) diff --git a/doc/replayer/replayer-agent.md b/doc/replayer/replayer-agent.md index 90a3a02..b16a894 100644 --- a/doc/replayer/replayer-agent.md +++ b/doc/replayer/replayer-agent.md @@ -1,8 +1,13 @@ -### Agent一键安装和启动脚本 +### Replayer-Agent一键安装和启动脚本
-[./replayer-agent/control.sh](../../replayer-agent/control.sh) 脚本是 [流量回放-1. 配置并启动Agent](./README.md#1-配置并启动agent) 的一键安装和启动脚本。 +[./replayer-agent/control.sh](../../replayer-agent/control.sh) 脚本是 [流量回放-1. 配置并启动Replayer-Agent](./README.md#1-配置并启动replayer-agent) 的一键安装和启动脚本。 + +* go mod方式脚本：[./replayer-agent/control_gomod.sh](../../replayer-agent/control_gomod.sh) 。对于没有GO环境的机器，会自动安装golang1.13，并设置GOROOT、GOPATH。 +* glide方式脚本：[./replayer-agent/control_glide.sh](../../replayer-agent/control_glide.sh) 。对于没有GO环境的机器，会自动安装golang1.10、glide，并设置GOROOT、GOPATH。 + +control.sh 默认使用 go mod方式，即control_gomod.sh 脚本。
@@ -12,43 +17,52 @@ * GO 「任意版本官方golang，或者定制版本golang都可以」 * Glide 「低版本GO包管理工具」 -* Go mod 「高版本GO包管理工具，GO>=1.13原生自带」 +* Go mod 「高版本GO包管理工具，go原生自带」 + +GO>=1.13推荐go mod, 低版本推荐glide
##### 2. 使用 -下面的命令都是基于 **sharingan/replayer-agent** 操作的。 +下面的命令都是基于 **sharingan/replayer-agent** 目录操作的。 > 构建（build） -检测GO环境并编译Agent服务。 - -此脚本默认使用glide包管理工具，对于没有GO环境的机器，会自动安装golang1.10、glide，并设置GOROOT、GOPATH。 [golang安装](https://github.com/didichuxing/sharingan-go)。 - -使用gomod包管理的脚本即将提供~ +检测GO环境并编译Replayer-Agent服务。 ```shell sh control.sh build ``` +如果提示如下错误： +```text +curl: (7) Failed to connect to raw.githubusercontent.com port 443: Connection refused +``` +可能是raw.githubusercontent.com域名不通，建议配置个代理；或者 配置hosts (151.101.108.133 raw.githubusercontent.com)；再或者 自己安装go后，重新执行脚本构建。可参考: [go安装](https://github.com/golang/go#download-and-install) + +
> 启动（start） -启动Agent服务。 +启动Replayer-Agent服务。 ```shell sh control.sh start ``` +
+ > 停止（stop） -停止Agent服务。 +停止Replayer-Agent服务。 ```shell sh control.sh stop ``` +
+ > 重启（reload） -重启Agent服务。 +重启Replayer-Agent服务。 ```shell sh control.sh reload ``` diff --git a/doc/replayer/replayer-codecov.md b/doc/replayer/replayer-codecov.md index b820125..6c9bdcc 100644 --- a/doc/replayer/replayer-codecov.md +++ b/doc/replayer/replayer-codecov.md @@ -11,12 +11,12 @@ 回放前提：已经完成流量录制。[录制接入文档](../recorder/README.md) 对于服务启动阶段有TCP请求的SUT，如初始化连接池等，推荐 服务启动顺序： -* 先启动Agent +* 先启动Replayer-Agent * 再启动SUT -##### 1. 配置并启动Agent +##### 1. 配置并启动Replayer-Agent -同 [回放接入-1. 配置并启动Agent](./README.md#1-配置并启动agent) +同 [回放接入-1. 配置并启动Replayer-Agent](./README.md#1-配置并启动replayer-agent) ##### 2. 配置并启动SUT @@ -67,7 +67,7 @@ B. [批量回放](./replayer-parallel.md)结果页 > 注意： > -> a.点击 "覆盖率报告" 链接后，Agent会 **自动重启SUT服务**。所以，覆盖率数据会重新统计!!! +> a.点击 "覆盖率报告" 链接后，Replayer-Agent会 **自动重启SUT服务**。所以，覆盖率数据会重新统计!!! > > b.上面两处 "覆盖率报告" 链接 点击效果一样。覆盖率统计不区分单流量回放和批量回放，会一起统计。 @@ -99,7 +99,7 @@ b. 按 [main_flag.go](../../replayer-agent/install/codeCov/main_with_flag/main_f 历史覆盖率报告存放在本机 /tmp/ShaRinGan/ 目录下。 a. 覆盖率原始报告, 命名格式 coverage.$binName.$timestamp -> $timestamp 一般包括起始和终止两个时间戳。对于初始手动启动SUT的情况，只有终止时间戳。对于Agent自动启动SUT的情况，都有起始和终止两个时间戳。 +> $timestamp 一般包括起始和终止两个时间戳。对于初始手动启动SUT的情况，只有终止时间戳。对于Replayer-Agent自动启动SUT的情况，都有起始和终止两个时间戳。 b. 格式化后的 *.html 报告，命名格式 coverage.$binName.$timestamp.html > 意外惊喜：历史 *.html 报告，可通过接口 http://127.0.0.1:8998/coverage/report/coverage.$binName.$timestamp.html 查看。 diff --git a/doc/replayer/replayer-conf.md b/doc/replayer/replayer-conf.md index a3d3b3c..25ca9cf 100644 --- a/doc/replayer/replayer-conf.md +++ b/doc/replayer/replayer-conf.md @@ -1,13 +1,13 @@ -### 回放Agent 配置 +### Replayer-Agent 配置
-Agent的配置文件都在 **[./replayer-agent/conf](../../replayer-agent/conf)** 目录下。 +Replayer-Agent的配置文件都在 **[./replayer-agent/conf](../../replayer-agent/conf)** 目录下。 * **conf/dsl目录**: 存放回放首页里 上报到本机的dsl数据，每个模块一个文件，以模块名为文件名。 * **conf/noise目录**: 存放回放结果页里 上报到本机的噪音数据，每个模块一个文件，以模块名为文件名。 * **conf/traffic目录**: 存放本机录制的测试流量，每个模块一个文件，以模块名为文件名。 -* **conf/app.toml**: Agent的核心配置文件，可以配置Web Server端口及超时时间，Mock Server端口、噪音及DSL上报到自有服务的http接口地址、流量查询的ES地址等。 +* **conf/app.toml**: Replayer-Agent的核心配置文件，可以配置Web Server端口及超时时间，Mock Server端口、噪音及DSL上报到自有服务的http接口地址、流量查询的ES地址等。 * **conf/moduleinfo.json**: 模块配置。本地回放时，用来存放模块基本信息。
@@ -18,7 +18,7 @@ Agent的配置文件都在 **[./replayer-agent/conf](../../replayer-agent/conf)* 定义 Web Server 的http相关配置信息。只需要注意端口字段，其他字段使用默认值即可。 -* addr = ":8998" # Agent Web Server默认监听端口，可以修改 +* addr = ":8998" # Replayer-Agent Web Server默认监听端口，可以修改 * handlerTimeout = 60000 # Handler timeout(ms), default 60000 * readHeaderTimeout = 2000 # Request header timeout(ms), default 2000 * readTimeout = 5000 # Recieve http request timeout(ms), including the body, default 5000 @@ -82,9 +82,9 @@ Agent的配置文件都在 **[./replayer-agent/conf](../../replayer-agent/conf)* default字段非空时，支持不同业务部门的模块读取自不同的es地址。 -> 如果 模块信息未配置department字段，即 未定义所属部门信息，则统一使用default配置的es地址。[回放模块配置](./conf/moduleinfo.md) +> 如果 模块信息未配置department字段，即 未定义所属部门信息，则统一使用default配置的es地址。 > ->如果定义了所属部门信息，如department=Biz，则新增下面配置即可。 +> 如果定义了所属部门信息，如department=Biz (参见 [回放模块配置](./conf/moduleinfo.md) example2。)，则新增下面配置即可。 * Biz = "http://{{es_domain}}/xxx" diff --git a/doc/replayer/replayer-local.md b/doc/replayer/replayer-local.md index 92cbc6b..d22bf3f 100644 --- a/doc/replayer/replayer-local.md +++ b/doc/replayer/replayer-local.md @@ -2,7 +2,7 @@
-回放Agent默认的接入方式是[本地回放](./README.md#4本地回放)。如需 非本地回放，只需根据 [回放Agent配置](./replayer-conf.md#4-http_api) 设置[http_api]和[es_url]接口地址即可。 +Replayer-Agent默认的接入方式是[本地回放](./README.md#4本地回放)。如需 非本地回放，只需根据 [Replayer-Agent配置](./replayer-conf.md#4-http_api) 设置[http_api]和[es_url]接口地址即可。
@@ -21,7 +21,7 @@ 1. 注释掉http_api下的所有字段 2. 注释掉es_url下的所有字段 -各配置字段含义详见: [回放Agent配置](./replayer-conf.md) +各配置字段含义详见: [Replayer-Agent配置](./replayer-conf.md)
diff --git a/doc/replayer/replayer-parallel.md b/doc/replayer/replayer-parallel.md index 8927f3c..fd8d375 100644 --- a/doc/replayer/replayer-parallel.md +++ b/doc/replayer/replayer-parallel.md @@ -2,18 +2,18 @@
-为了提高测试效率，Agent还支持了 批量并发 回放。 +为了提高测试效率，Replayer-Agent还支持了 批量并发 回放。
针对 并发 及 使用 说明如下： ##### 1. 并发度 -Agent启动时，默认的并发度是 parallel=10。 +Replayer-Agent启动时，默认的并发度是 parallel=10。 > 温馨提示： > 当批量回放的流量总数小于parallel时，并发度自动调整为流量总数大小。 -如需调整并发度，只需在启动Agent时增加 -parallel 参数即可。 +如需调整并发度，只需在启动Replayer-Agent时增加 -parallel 参数即可。 ```shell script cd ./replayer-agent && go build && nohup ./replayer-agent -parallel=x 2>&1 & ``` @@ -24,7 +24,7 @@ cd ./replayer-agent && go build && nohup ./replayer-agent -parallel=x 2>&1 & ###### A. 普通使用 -a. 在Agent首页，输入流量筛选条件。 +a. 在Replayer-Agent首页，输入流量筛选条件。 ![web_index_parallel](../images/web_index_parallel.png) @@ -49,7 +49,7 @@ b. 点击 "批量回放"，即可看到下面的弹出框。 对于 已经根据 [噪音/DSL 上报指南](./guide/report.md#1-dsl上报) 上报了一些DSL的用户，可以批量回放 **所有DSL** 对应的流量。 -a. 操作非常简单，只需保证Agent首页没有输入任何筛选条件，包括 **"时间段""** 参数。 +a. 操作非常简单，只需保证Replayer-Agent首页没有输入任何筛选条件，包括 **"时间段""** 参数。 ![parallel_caidan](../images/parallel_caidan.png) diff --git a/doc/replayer/replayer-sut.md b/doc/replayer/replayer-sut.md index 5b5fd2c..2b9ba20 100644 --- a/doc/replayer/replayer-sut.md +++ b/doc/replayer/replayer-sut.md @@ -2,7 +2,13 @@
-[./example/replayer/sut_replayer.sh](../../example/replayer/sut_replayer.sh) 脚本是 [流量回放-2. 配置并启动SUT](./README.md#2-配置并启动sut) 示例 example 的一键安装和启动脚本。 +[./example/replayer/sut_replayer.sh](../../example/replayer/sut_replayer.sh) 脚本是 [流量回放-2. 配置并启动SUT](./README.md#2-配置并启动sut) 示例 example 的一键安装和启动脚本。 + +* go mod方式脚本：[./example/replayer/sut_replayer_gomod.sh](../../example/replayer/sut_replayer_gomod.sh) 。对于没有GO环境的机器，会自动安装golang1.13，并设置GOROOT、GOPATH。 + +* glide方式脚本： [./example/replayer/sut_replayer_glide.sh](../../example/replayer/sut_replayer_glide.sh) 。对于没有GO环境的机器，会自动安装golang1.10、glide，并设置GOROOT、GOPATH。 + +sut_replayer.sh 默认使用 go mod方式，即sut_replayer_gomod.sh 脚本。
@@ -17,20 +23,23 @@ ##### 2. 使用 -下面的命令都是基于 **sharingan/example/replayer** 操作的，相当于SUT的根目录。 +下面的命令都是基于 **sharingan/example/replayer** 名录操作的，相当于SUT的根目录。 > 构建（build） 检测GO环境并编译SUT服务。 -此脚本默认使用glide包管理工具，对于没有GO环境的机器，会自动安装golang1.10、glide，并设置GOROOT、GOPATH。 [golang安装](https://github.com/didichuxing/sharingan-go)。 - -使用gomod包管理的脚本即将提供~ - ```shell sh sut_replayer.sh build //普通回放 sh sut_replayer.sh build cov //覆盖率回放 ``` +如果提示如下错误： +```text +curl: (7) Failed to connect to raw.githubusercontent.com port 443: Connection refused +``` +可能是raw.githubusercontent.com域名不通，建议配置个代理；或者 配置hosts (151.101.108.133 raw.githubusercontent.com)；再或者 自己安装go后，重新执行脚本构建。可参考: [sharingan-go安装](https://github.com/didichuxing/sharingan-go/tree/recorder) + +
> 启动（start） @@ -39,6 +48,7 @@ sh sut_replayer.sh build cov //覆盖率回放 sh sut_replayer.sh start //普通回放 sh sut_replayer.sh start cov //覆盖率回放 ``` +
> 停止（stop） @@ -50,6 +60,8 @@ sh sut_replayer.sh stop cov //覆盖率回放 停止SUT时，覆盖率回放方式 会给出覆盖率报告 及 可以直接查看的html链接。覆盖率报告详细说明：[覆盖率报告](./replayer-codecov.md#1-覆盖率报告) ![shell_sut_cov_stop](../images/shell_sut_cov_stop.png) +
+ > 重启（reload） 重启SUT服务。 @@ -60,18 +72,18 @@ sh sut_replayer.sh reload cov //覆盖率回放
-##### 3. 与Agent分开部署 +##### 3. 与Replayer-Agent分开部署 -SUT与Agent服务可以在不同的机器分开部署，其中Agent的Mock Server监听端口3515也可以修改。 +SUT与Replayer-Agent服务可以在不同的机器分开部署，其中Replayer-Agent的Mock Server监听端口3515也可以修改。 > 分开部署 - 1. 启动Agent服务。参见: [Agent启动脚本](./replayer-agent.md) - 2. 修改脚本 [./example/replayer/sut_replayer.sh](../../example/replayer/sut_replayer.sh) 里的 REPLAYER_MOCK_IP 环境变量，为Agent的ip地址。 + 1. 启动Replayer-Agent服务。参见: [Replayer-Agent启动脚本](./replayer-agent.md) + 2. 修改脚本 [./example/replayer/sut_replayer.sh](../../example/replayer/sut_replayer.sh) 里的 REPLAYER_MOCK_IP 环境变量，为Replayer-Agent的ip地址。 3. 重启SUT服务即可。 > 修改Mock Server 3515端口 - 1. 使用新端口 修改 [Agent配置-outbound](./replayer-conf.md#3-outbound) 字段 - 2. 重启Agent服务。参见: [Agent启动脚本](./replayer-agent.md) + 1. 使用新端口 修改 [Replayer-Agent配置-outbound](./replayer-conf.md#3-outbound) 字段 + 2. 重启Replayer-Agent服务。参见: [Replayer-Agent启动脚本](./replayer-agent.md) 3. 修改脚本 [./example/replayer/sut_replayer.sh](../../example/replayer/sut_replayer.sh) 里的 REPLAYER_MOCK_PORT 环境变量，为Mock Server新端口。 - 4. 重启SUT服务即可。 \ No newline at end of file + 4. 重启SUT服务即可。 diff --git a/doc/replayer/replayer-theory.md b/doc/replayer/replayer-theory.md index a09aac7..045b2f8 100644 --- a/doc/replayer/replayer-theory.md +++ b/doc/replayer/replayer-theory.md @@ -13,7 +13,7 @@ Golang的流量回放 主要基于 [sharingan/replayer](../../replayer) 包 及 但实际业务中，不仅会有Outbound请求，而且Outbound请求还很多，协议也各种各样。 因此，流量回放首要解决的问题有： - 1. 如何拦截SUT的Outbound请求，将其转发给Agent的Mock Server。 + 1. 如何拦截SUT的Outbound请求，将其转发给Replayer-Agent的Mock Server。 2. 如何在录制的流量里，选择最合适的Outbound返回给SUT。 ## 二、Outbound请求拦截点 @@ -33,7 +33,7 @@ func Connect(fd int, sa Sockaddr) (err error) { ## 三、回放剧本传递 -[sharingan/replayer](../../replayer) 包拦截了SUT的Outbound请求，将其转发给Agent的Mock Server。 +[sharingan/replayer](../../replayer) 包拦截了SUT的Outbound请求，将其转发给Replayer-Agent的Mock Server。 ![replay-theory](https://github.com/didichuxing/sharingan/raw/master/doc/wiki/images/replay_theory.png) @@ -174,4 +174,4 @@ func Now() Time { } ``` -[sharingan/replayer](../../replayer) 包mock上面的 [time.Now](https://github.com/golang/go/blob/release-branch.go1.10/src/time/time.go#L1043) 方法，以实现时间的回放。 \ No newline at end of file +[sharingan/replayer](../../replayer) 包mock上面的 [time.Now](https://github.com/golang/go/blob/release-branch.go1.10/src/time/time.go#L1043) 方法，以实现时间的回放。 diff --git a/example/replayer/sut_replayer.sh b/example/replayer/sut_replayer.sh index 4cd1960..441c973 100755 --- a/example/replayer/sut_replayer.sh +++ b/example/replayer/sut_replayer.sh @@ -3,11 +3,13 @@ workspace=$(cd $(dirname $0) && pwd -P) cd ${workspace} -# TODO: Attention please!!! Is better that $app not contain '-'。 +# TODO: Attention please!!! Modify-1! Your project's binary name. Is better that $app not contain '-'。 app=example appcov="$app.test" -# TODO: Attenton please!!! Here is for sharingan/example/replayer, you can change to your project's real root. +# TODO: Attenton please!!! Modify-2! Your project's root path. root=$(dirname $(dirname $workspace)) +# TODO: Attenton please!!! Modify-3! The path where you git clone sharingan. It's used to copy main_test.go & gocov tool when failed by wgetting from github. +replayer_agent_root=$root/replayer-agent # set file sut_file=$workspace/$app @@ -30,23 +32,28 @@ cov_file_suffix='.cov' local_ip="127.0.0.1" local_port="8998" -# TODO: tmp for private git -#git_replayer_agent='https://github.com/didichuxing/sharingan/raw/master/replayer-agent' -git_replayer_agent='https://git.xiaojukeji.com/nuwa/fastdev2/raw/fastdev-pure' -#git_go='https://github.com/didichuxing/sharingan-go/raw/recorder' -git_go='https://git.xiaojukeji.com/nuwa/tools/go/raw/master' +# set github about go +git_replayer_agent='https://github.com/didichuxing/sharingan/raw/master/replayer-agent' +git_go='https://raw.githubusercontent.com/didichuxing/sharingan-go/recorder' +install_go_md='https://github.com/didichuxing/sharingan-go/tree/recorder' +VERSION="go1.13" -# build -function build() { - preGoroot="/tmp/recorder-go" - if [[ -z $GOROOT || $GOROOT != $preGoroot* ]];then - # install golang1.10 - curl "$git_go/install/go1.10" | sh +function install_go() { + curl "$git_go/install/$VERSION" >> /dev/null if [ $? -ne 0 ]; then - printf "${error_msg}build failed at installing golang1.10, please check!!!\n" + printf "${error_msg}build failed at installing sharingan-go $VERSION : 'curl $git_go/install/$VERSION' \n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" exit 1 fi - export GOROOT=/tmp/recorder-go1.10 + curl "$git_go/install/$VERSION" | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing sharingan-go $VERSION : 'curl $git_go/install/$VERSION | sh' \n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + export GOROOT="/tmp/recorder-$VERSION" if [ $? -ne 0 ]; then printf "${error_msg}build failed at setting GOROOT, please check!!!\n" exit 1 @@ -56,62 +63,113 @@ function build() { printf "${error_msg}build failed at updating PATH, please check!!!\n" exit 1 fi - fi - which glide &> /dev/null - if [ $? -ne 0 ]; then - # install glide - curl https://glide.sh/get | sh + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# install go +function install_go_tar() { + # get go url + releases_go='https://github.com/didichuxing/sharingan-go/releases/download/' + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.darwin-amd64.tar.gz" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.linux-amd64.tar.gz" + else + printf "${error_msg} Unknown system type! build failed at installing go! \n" + exit 1 + fi + + # install golang + recorder_dir=/tmp/recorder-${VERSION} + wget "$releases_go/$goSuffix" -O "$recorder_dir.tar.gz" if [ $? -ne 0 ]; then - printf "${error_msg}build failed at installing glide, please check!!!\n" + printf "${error_msg}build failed at downloading sharingan-go $VERSION : wget $releases_go/$goSuffix -O $recorder_dir.tar.gz !!!\n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" exit 1 fi - fi + rm -rf $recorder_dir && mkdir -p $recorder_dir + tar -xzf "$recorder_dir.tar.gz" -C ${recorder_dir} --strip-components=1 + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION, please check!!!\n" + exit 1 + fi + rm -rf "$recorder_dir.tar.gz" - if [ -z $GOPATH ];then - prePath="/src/github.com/didichuxing/sharingan" - if [[ $workspace == *$prePath* ]];then - export GOPATH=`echo ${workspace%/src/*}` - if [ $? -ne 0 ]; then - printf "${error_msg}build failed at setting GOPATH, please check!!!\n" - exit 1 - fi - else - printf "${error_msg}build failed at setting GOPATH, please check and clone sharingan to legal path!!!\n" + # export + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" exit 1 fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# build +function build() { + preGoroot="/tmp/recorder-go" + if [[ -z $GOROOT || $GOROOT != $preGoroot* ]];then + printf "${info_msg}Installing sharingan-go for you ~~~ \n" + install_go fi - cd $root - rm -rf vendor - # TODO: tmp for private - #glide get 'github.com/didichuxing/sharingan/replayer' + which go &> /dev/null if [ $? -ne 0 ]; then - printf "${error_msg}build failed at executing glide get sharingan/replayer, please check!!!\n" + printf "${error_msg}build failed at executing 'which go', please execute 'export PATH=\$GOROOT/bin:\$PATH' !!!\n" exit 1 fi - glide install - if [ $? -ne 0 ]; then - printf "${error_msg}build failed at executing glide install, please check!!!\n" - exit 1 + goVer=`go version` + if [[ $goVer < 'go version go1.11' ]];then + printf "${warn_msg}current is $goVer, go mod requires at least go1.11!!!\n" + install_go fi + if [ -z $GOPATH ];then + mkdir -p /tmp/replayer + export GOPATH=/tmp/replayer + printf "${warn_msg}No GOPATH, so setting GOPATH=/tmp/replayer \n" + fi + + cd $root + # TODO: tmp for private + #go get 'github.com/didichuxing/sharingan/replayer' +# if [ $? -ne 0 ]; then +# printf "${error_msg}build failed at executing go get sharingan/replayer, please check!!!\n" +# exit 1 +# fi + go clean -modcache + printf "${info_msg}go mod download, please wait~ \n" + go mod download + cd $workspace binName=$app if [[ $sub_cmd == $cov_cmd ]];then # wget main_test.go - # TODO: tmp for private -# wget -O main_test.go "$git_replayer_agent/install/codeCov/main_test.go" -# if [ $? -ne 0 ]; then -# printf "${error_msg}build failed at wgetting main_test.go, please check!!!\n" -# exit 1 -# fi + wget -O main_test.go "$git_replayer_agent/install/codeCov/main_test.go" + if [ $? -ne 0 ]; then + printf "${warn_msg}Failed to wget main_test.go. Now trying another way, copying from sharingan/replayer-agent!!!\n" + cp -rf $replayer_agent_root/install/codeCov/main_test.go . + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at copying main_test.go from $replayer_agent_root/install/codeCov/, please update var \$replayer_agent_root!!!\n" + exit 1 + fi + printf "${info_msg}Successed to copy main_test.go. \n" + fi # wget gocov* tool if [[ "$OSTYPE" =~ ^darwin.* ]]; then + sys_dir='darwin' wget -O gocov "$git_replayer_agent/install/codeCov/darwin/gocov" wget -O gocov-html "$git_replayer_agent/install/codeCov/darwin/gocov-html" elif [[ "$OSTYPE" =~ ^linux.* ]]; then + sys_dir='linux' wget -O gocov "$git_replayer_agent/install/codeCov/linux/gocov" wget -O gocov-html "$git_replayer_agent/install/codeCov/linux/gocov-html" else @@ -119,8 +177,13 @@ function build() { exit 1 fi if [ $? -ne 0 ]; then - printf "${error_msg}build failed at wgetting gocov* tool, please check!!!\n" - exit 1 + printf "${warn_msg}Failed to wget gocov* tool, Now trying another way, copying from sharingan/replayer-agent!!!\n" + cp -rf $replayer_agent_root/install/codeCov/$sys_dir/gocov* . + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at copying gocov* tool from $replayer_agent_root/install/codeCov/$sys_dir/, please update var \$replayer_agent_root!!!\n" + exit 1 + fi + printf "${info_msg}Successed to copy gocov* tool. \n" fi chmod 755 $workspace/gocov* @@ -129,7 +192,6 @@ function build() { exit 1 fi - cmdBuild='go test -o '$appcov' -gcflags="all=-N -l" -tags="replayer" -v -c -covermode=count -coverpkg ./...' binName=$appcov fi @@ -141,7 +203,7 @@ function build() { fi if [[ $sub_cmd == $cov_cmd ]];then - $cmdBuild + go test -o $appcov -gcflags="all=-N -l" -tags="replayer" -v -c -covermode=count -coverpkg ./... else go build -o $app -tags="replayer" -gcflags="all=-N -l" fi diff --git a/example/replayer/sut_replayer_glide.sh b/example/replayer/sut_replayer_glide.sh new file mode 100755 index 0000000..fec3232 --- /dev/null +++ b/example/replayer/sut_replayer_glide.sh @@ -0,0 +1,383 @@ +#!/bin/bash + +workspace=$(cd $(dirname $0) && pwd -P) +cd ${workspace} + +# TODO: Attention please!!! Modify-1! Your project's binary name. Is better that $app not contain '-'。 +app=example +appcov="$app.test" +# TODO: Attenton please!!! Modify-2! Your project's root path. +root=$(dirname $(dirname $workspace)) +# TODO: Attenton please!!! Modify-3! The path where you git clone sharingan. It's used to copy main_test.go & gocov tool when failed by wgetting from github. +replayer_agent_root=$root/replayer-agent + +# set file +sut_file=$workspace/$app +sut_file_cov=$workspace/$appcov +sut_pid_file=$workspace/sut.pid +sut_log_file=$workspace/sut.log + +# set msg +info_msg="\033[;32m[INFO]\033[0m\t" +warn_msg="\033[;33m[WARN]\033[0m\t" +error_msg="\033[;31m[ERROR]\033[0m\t" + +#cov +cov_cmd=cov +cov_file_path='/tmp/ShaRinGan/' +cov_file_prefix='coverage.' +cov_file_suffix='.cov' + +# set web server +local_ip="127.0.0.1" +local_port="8998" + +# set github about go +git_replayer_agent='https://github.com/didichuxing/sharingan/raw/master/replayer-agent' +git_go='https://raw.githubusercontent.com/didichuxing/sharingan-go/recorder' +install_go_md='https://github.com/didichuxing/sharingan-go/tree/recorder' +VERSION="go1.10" + +function install_go() { + curl "$git_go/install/$VERSION" >> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing sharingan-go $VERSION : 'curl $git_go/install/$VERSION' \n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + curl "$git_go/install/$VERSION" | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing sharingan-go $VERSION : 'curl $git_go/install/$VERSION | sh' \n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# install go +function install_go_tar() { + # get go url + releases_go='https://github.com/didichuxing/sharingan-go/releases/download/' + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.darwin-amd64.tar.gz" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.linux-amd64.tar.gz" + else + printf "${error_msg} Unknown system type! build failed at installing go! \n" + exit 1 + fi + + # install golang + recorder_dir=/tmp/recorder-${VERSION} + wget "$releases_go/$goSuffix" -O "$recorder_dir.tar.gz" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at downloading sharingan-go $VERSION : wget $releases_go/$goSuffix -O $recorder_dir.tar.gz !!!\n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + rm -rf $recorder_dir && mkdir -p $recorder_dir + tar -xzf "$recorder_dir.tar.gz" -C ${recorder_dir} --strip-components=1 + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION, please check!!!\n" + exit 1 + fi + rm -rf "$recorder_dir.tar.gz" + + # export + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# build +function build() { + preGoroot="/tmp/recorder-go" + if [[ -z $GOROOT || $GOROOT != $preGoroot* ]];then + printf "${info_msg}Installing sharingan-go for you ~~~ \n" + install_go + fi + + which glide &> /dev/null + if [ $? -ne 0 ]; then + # install glide + curl https://glide.sh/get | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing glide, please check!!!\n" + exit 1 + fi + fi + + which go &> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing 'which go', please execute 'export PATH=\$GOROOT/bin:\$PATH' !!!\n" + exit 1 + fi + + if [ -z $GOPATH ];then + prePath="/src/github.com/didichuxing/sharingan" + if [[ $workspace == *$prePath* ]];then + export GOPATH=`echo ${workspace%/src/*}` + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOPATH, please check!!!\n" + exit 1 + fi + else + printf "${error_msg}build failed at setting GOPATH, please check and clone sharingan to legal path!!!\n" + exit 1 + fi + fi + + cd $root + rm -rf vendor + # TODO: tmp for private + #glide get 'github.com/didichuxing/sharingan/replayer' + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing glide get sharingan/replayer, please check!!!\n" + exit 1 + fi + glide install + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing glide install, please check!!!\n" + exit 1 + fi + + cd $workspace + binName=$app + if [[ $sub_cmd == $cov_cmd ]];then + # wget main_test.go + wget -O main_test.go "$git_replayer_agent/install/codeCov/main_test.go" + if [ $? -ne 0 ]; then + printf "${warn_msg}Failed to wget main_test.go. Now trying another way, copying from sharingan/replayer-agent!!!\n" + cp -rf $replayer_agent_root/install/codeCov/main_test.go . + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at copying main_test.go from $replayer_agent_root/install/codeCov/, please update var \$replayer_agent_root!!!\n" + exit 1 + fi + printf "${info_msg}Successed to copy main_test.go. \n" + fi + + # wget gocov* tool + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + sys_dir='darwin' + wget -O gocov "$git_replayer_agent/install/codeCov/darwin/gocov" + wget -O gocov-html "$git_replayer_agent/install/codeCov/darwin/gocov-html" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + sys_dir='linux' + wget -O gocov "$git_replayer_agent/install/codeCov/linux/gocov" + wget -O gocov-html "$git_replayer_agent/install/codeCov/linux/gocov-html" + else + printf "${error_msg} Unknown system type! build failed at wgetting gocov* tool! \n" + exit 1 + fi + if [ $? -ne 0 ]; then + printf "${warn_msg}Failed to wget gocov* tool, Now trying another way, copying from sharingan/replayer-agent!!!\n" + cp -rf $replayer_agent_root/install/codeCov/$sys_dir/gocov* . + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at copying gocov* tool from $replayer_agent_root/install/codeCov/$sys_dir/, please update var \$replayer_agent_root!!!\n" + exit 1 + fi + printf "${info_msg}Successed to copy gocov* tool. \n" + fi + + chmod 755 $workspace/gocov* + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at chmod $workspace/gocov* tool, please check!!!\n" + exit 1 + fi + + binName=$appcov + fi + + pkgReplay='_ "github.com/didichuxing/sharingan/replayer"' + find ./ -name "*\.go" -maxdepth 1 | xargs -n 1 grep $pkgReplay &> /dev/null + if [ $? -eq 0 ];then + printf "${error_msg} build failed for not importing package sharingan/replayer, please import it!!!\n" + exit 1 + fi + + if [[ $sub_cmd == $cov_cmd ]];then + go test -o $appcov -gcflags="all=-N -l" -tags="replayer" -v -c -covermode=count -coverpkg ./... + else + go build -o $app -tags="replayer" -gcflags="all=-N -l" + fi + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at building sut, please check!!!\n" + exit 1 + fi + + printf "${info_msg}SUT $binName builds success!!!\n" +} + +# start +function start() { + binName=$app + binFile=$sut_file + cmdStart="$sut_file >> $sut_log_file" + cmdBuild="sh sut_replayer.sh build" + if [[ $sub_cmd == $cov_cmd ]];then + binName=$appcov + binFile=$sut_file_cov + curTime=`date +%s` + cmdStart="$sut_file_cov -systemTest -test.coverprofile=$cov_file_path$cov_file_prefix$app.$curTime$cov_file_suffix >> $sut_log_file" + cmdBuild="sh sut_replayer.sh build $cov_cmd" + fi + + if [ ! -f "$binFile" ]; then + printf "${error_msg}please exesute $cmdBuild first!!!\n" + exit 1 + fi + + if [[ "$OSTYPE" =~ ^linux.* ]]; then + local_ip=`ip addr | grep "inet " | grep -v 127 | awk '{print $2}' | awk -F '/' '{print $1}'` + fi + + #ps -ef | grep $binName | grep -v grep $grepV &> /dev/null + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -eq 0 ];then + printf "${warn_msg}SUT $binName has already started!!! Have fun with http://${local_ip}:${local_port} !\n" + exit 0 + fi + + # 默认值，如有需要可以修改 + export REPLAYER_MOCK_IP="127.0.0.1" + export REPLAYER_MOCK_PORT="3515" + + nohup $cmdStart >> $sut_log_file 2>&1 & + sleep 2 + pid=$! + + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -ne 0 ];then + printf "${error_msg} Failed to start sut $binName, please check $sut_log_file and try again!!!\n" + exit 1 + fi + echo $pid > $sut_pid_file + printf "${info_msg}SUT $binName starts success!!! [pid=${pid}] Have fun with http://${local_ip}:${local_port} !\n" +} + +# stop +function stop() { + binName=$app + if [[ $sub_cmd == $cov_cmd ]];then + binName=$appcov + fi + + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -ne 0 ];then + printf "${warn_msg}SUT $binName is not running!!!\n" + exit 0 + fi + + pkill $binName + sleep 2 + + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -eq 0 ];then + printf "${error_msg}Failed to stop sut $binName, please check and try again!!!\n" + exit 1 + fi + printf "${info_msg}SUT $binName stops success!!!\n" + + # codeCov report + if [[ $sub_cmd == $cov_cmd ]];then + f="$cov_file_path$cov_file_prefix"'*'"$app"'*'"$cov_file_suffix" + covFile=`ls -trls $f | tail -n 1` + #echo $covFile + if [[ $covFile != *$cov_file_suffix ]];then + printf "${warn_msg}Failed to get *.cov file, please check $cov_file_path!!!\n" + exit 0 + fi + + # get the full path of covfile created by stop cmd + covFile=/`echo ${covFile#*/}` + curTime=`date +%s` + covFile_renamed=`echo ${covFile%cov*}`$curTime + mv $covFile $covFile_renamed + if [ $? -ne 0 ];then + printf "${warn_msg}Failed to rename $covFile to $covFile_renamed, please check !!!\n" + exit 0 + fi + + # convert cov file to html + covNameOnly_html=`echo ${covFile_renamed##*/}` + $workspace/gocov convert $covFile_renamed | $workspace/gocov-html > $workspace/$covNameOnly_html'.html' + mv $workspace/$covNameOnly_html'.html' $cov_file_path/$covNameOnly_html'.html' + if [ $? -ne 0 ];then + printf "${warn_msg}Failed to gocov convert $covFile_renamed to $covFile_renamed'.html', please check !!!\n" + exit 0 + fi + + if [[ "$OSTYPE" =~ ^linux.* ]]; then + local_ip=`ip addr | grep "inet " | grep -v 127 | awk '{print $2}' | awk -F '/' '{print $1}'` + fi + + printf "${info_msg}Have fun with http://${local_ip}:${local_port}/coverage/report/$covNameOnly_html.html !!!\n" + fi +} + +# cov or null +sub_cmd=$2 +# check sub cmd +if [[ ! -z $sub_cmd && $sub_cmd != $cov_cmd ]];then + printf "${warn_msg} cmd lists:\n1. build [$cov_cmd]\n2. start [$cov_cmd]\n3. stop [$cov_cmd]\n4. reload [$cov_cmd]\n[$cov_cmd] is optional for codeCov\n" + exit 1 +fi + +case $1 in + "build" ) + build + ;; + "start" ) + start + ;; + "stop" ) + stop + ;; + "reload" ) + stop + start + ;; + * ) + printf "${warn_msg} cmd lists:\n1. build [$cov_cmd]\n2. start [$cov_cmd]\n3. stop [$cov_cmd]\n4. reload [$cov_cmd]\n[$cov_cmd] is optional for codeCov\n" + exit 1 + ;; +esac diff --git a/example/replayer/sut_replayer_gomod.sh b/example/replayer/sut_replayer_gomod.sh new file mode 100755 index 0000000..441c973 --- /dev/null +++ b/example/replayer/sut_replayer_gomod.sh @@ -0,0 +1,367 @@ +#!/bin/bash + +workspace=$(cd $(dirname $0) && pwd -P) +cd ${workspace} + +# TODO: Attention please!!! Modify-1! Your project's binary name. Is better that $app not contain '-'。 +app=example +appcov="$app.test" +# TODO: Attenton please!!! Modify-2! Your project's root path. +root=$(dirname $(dirname $workspace)) +# TODO: Attenton please!!! Modify-3! The path where you git clone sharingan. It's used to copy main_test.go & gocov tool when failed by wgetting from github. +replayer_agent_root=$root/replayer-agent + +# set file +sut_file=$workspace/$app +sut_file_cov=$workspace/$appcov +sut_pid_file=$workspace/sut.pid +sut_log_file=$workspace/sut.log + +# set msg +info_msg="\033[;32m[INFO]\033[0m\t" +warn_msg="\033[;33m[WARN]\033[0m\t" +error_msg="\033[;31m[ERROR]\033[0m\t" + +#cov +cov_cmd=cov +cov_file_path='/tmp/ShaRinGan/' +cov_file_prefix='coverage.' +cov_file_suffix='.cov' + +# set web server +local_ip="127.0.0.1" +local_port="8998" + +# set github about go +git_replayer_agent='https://github.com/didichuxing/sharingan/raw/master/replayer-agent' +git_go='https://raw.githubusercontent.com/didichuxing/sharingan-go/recorder' +install_go_md='https://github.com/didichuxing/sharingan-go/tree/recorder' +VERSION="go1.13" + +function install_go() { + curl "$git_go/install/$VERSION" >> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing sharingan-go $VERSION : 'curl $git_go/install/$VERSION' \n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + curl "$git_go/install/$VERSION" | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing sharingan-go $VERSION : 'curl $git_go/install/$VERSION | sh' \n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# install go +function install_go_tar() { + # get go url + releases_go='https://github.com/didichuxing/sharingan-go/releases/download/' + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.darwin-amd64.tar.gz" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.linux-amd64.tar.gz" + else + printf "${error_msg} Unknown system type! build failed at installing go! \n" + exit 1 + fi + + # install golang + recorder_dir=/tmp/recorder-${VERSION} + wget "$releases_go/$goSuffix" -O "$recorder_dir.tar.gz" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at downloading sharingan-go $VERSION : wget $releases_go/$goSuffix -O $recorder_dir.tar.gz !!!\n" + printf "${error_msg}Please install sharingan-go $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + rm -rf $recorder_dir && mkdir -p $recorder_dir + tar -xzf "$recorder_dir.tar.gz" -C ${recorder_dir} --strip-components=1 + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION, please check!!!\n" + exit 1 + fi + rm -rf "$recorder_dir.tar.gz" + + # export + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# build +function build() { + preGoroot="/tmp/recorder-go" + if [[ -z $GOROOT || $GOROOT != $preGoroot* ]];then + printf "${info_msg}Installing sharingan-go for you ~~~ \n" + install_go + fi + + which go &> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing 'which go', please execute 'export PATH=\$GOROOT/bin:\$PATH' !!!\n" + exit 1 + fi + goVer=`go version` + if [[ $goVer < 'go version go1.11' ]];then + printf "${warn_msg}current is $goVer, go mod requires at least go1.11!!!\n" + install_go + fi + + if [ -z $GOPATH ];then + mkdir -p /tmp/replayer + export GOPATH=/tmp/replayer + printf "${warn_msg}No GOPATH, so setting GOPATH=/tmp/replayer \n" + fi + + cd $root + # TODO: tmp for private + #go get 'github.com/didichuxing/sharingan/replayer' +# if [ $? -ne 0 ]; then +# printf "${error_msg}build failed at executing go get sharingan/replayer, please check!!!\n" +# exit 1 +# fi + go clean -modcache + printf "${info_msg}go mod download, please wait~ \n" + go mod download + + cd $workspace + binName=$app + if [[ $sub_cmd == $cov_cmd ]];then + # wget main_test.go + wget -O main_test.go "$git_replayer_agent/install/codeCov/main_test.go" + if [ $? -ne 0 ]; then + printf "${warn_msg}Failed to wget main_test.go. Now trying another way, copying from sharingan/replayer-agent!!!\n" + cp -rf $replayer_agent_root/install/codeCov/main_test.go . + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at copying main_test.go from $replayer_agent_root/install/codeCov/, please update var \$replayer_agent_root!!!\n" + exit 1 + fi + printf "${info_msg}Successed to copy main_test.go. \n" + fi + + # wget gocov* tool + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + sys_dir='darwin' + wget -O gocov "$git_replayer_agent/install/codeCov/darwin/gocov" + wget -O gocov-html "$git_replayer_agent/install/codeCov/darwin/gocov-html" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + sys_dir='linux' + wget -O gocov "$git_replayer_agent/install/codeCov/linux/gocov" + wget -O gocov-html "$git_replayer_agent/install/codeCov/linux/gocov-html" + else + printf "${error_msg} Unknown system type! build failed at wgetting gocov* tool! \n" + exit 1 + fi + if [ $? -ne 0 ]; then + printf "${warn_msg}Failed to wget gocov* tool, Now trying another way, copying from sharingan/replayer-agent!!!\n" + cp -rf $replayer_agent_root/install/codeCov/$sys_dir/gocov* . + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at copying gocov* tool from $replayer_agent_root/install/codeCov/$sys_dir/, please update var \$replayer_agent_root!!!\n" + exit 1 + fi + printf "${info_msg}Successed to copy gocov* tool. \n" + fi + + chmod 755 $workspace/gocov* + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at chmod $workspace/gocov* tool, please check!!!\n" + exit 1 + fi + + binName=$appcov + fi + + pkgReplay='_ "github.com/didichuxing/sharingan/replayer"' + find ./ -name "*\.go" -maxdepth 1 | xargs -n 1 grep $pkgReplay &> /dev/null + if [ $? -eq 0 ];then + printf "${error_msg} build failed for not importing package sharingan/replayer, please import it!!!\n" + exit 1 + fi + + if [[ $sub_cmd == $cov_cmd ]];then + go test -o $appcov -gcflags="all=-N -l" -tags="replayer" -v -c -covermode=count -coverpkg ./... + else + go build -o $app -tags="replayer" -gcflags="all=-N -l" + fi + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at building sut, please check!!!\n" + exit 1 + fi + + printf "${info_msg}SUT $binName builds success!!!\n" +} + +# start +function start() { + binName=$app + binFile=$sut_file + cmdStart="$sut_file >> $sut_log_file" + cmdBuild="sh sut_replayer.sh build" + if [[ $sub_cmd == $cov_cmd ]];then + binName=$appcov + binFile=$sut_file_cov + curTime=`date +%s` + cmdStart="$sut_file_cov -systemTest -test.coverprofile=$cov_file_path$cov_file_prefix$app.$curTime$cov_file_suffix >> $sut_log_file" + cmdBuild="sh sut_replayer.sh build $cov_cmd" + fi + + if [ ! -f "$binFile" ]; then + printf "${error_msg}please exesute $cmdBuild first!!!\n" + exit 1 + fi + + if [[ "$OSTYPE" =~ ^linux.* ]]; then + local_ip=`ip addr | grep "inet " | grep -v 127 | awk '{print $2}' | awk -F '/' '{print $1}'` + fi + + #ps -ef | grep $binName | grep -v grep $grepV &> /dev/null + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -eq 0 ];then + printf "${warn_msg}SUT $binName has already started!!! Have fun with http://${local_ip}:${local_port} !\n" + exit 0 + fi + + # 默认值，如有需要可以修改 + export REPLAYER_MOCK_IP="127.0.0.1" + export REPLAYER_MOCK_PORT="3515" + + nohup $cmdStart >> $sut_log_file 2>&1 & + sleep 2 + pid=$! + + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -ne 0 ];then + printf "${error_msg} Failed to start sut $binName, please check $sut_log_file and try again!!!\n" + exit 1 + fi + echo $pid > $sut_pid_file + printf "${info_msg}SUT $binName starts success!!! [pid=${pid}] Have fun with http://${local_ip}:${local_port} !\n" +} + +# stop +function stop() { + binName=$app + if [[ $sub_cmd == $cov_cmd ]];then + binName=$appcov + fi + + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -ne 0 ];then + printf "${warn_msg}SUT $binName is not running!!!\n" + exit 0 + fi + + pkill $binName + sleep 2 + + if [[ $sub_cmd == $cov_cmd ]];then + ps -ef | grep $binName | grep -v grep &> /dev/null + else + ps -ef | grep $binName | grep -v grep | grep -v $appcov &> /dev/null + fi + if [ $? -eq 0 ];then + printf "${error_msg}Failed to stop sut $binName, please check and try again!!!\n" + exit 1 + fi + printf "${info_msg}SUT $binName stops success!!!\n" + + # codeCov report + if [[ $sub_cmd == $cov_cmd ]];then + f="$cov_file_path$cov_file_prefix"'*'"$app"'*'"$cov_file_suffix" + covFile=`ls -trls $f | tail -n 1` + #echo $covFile + if [[ $covFile != *$cov_file_suffix ]];then + printf "${warn_msg}Failed to get *.cov file, please check $cov_file_path!!!\n" + exit 0 + fi + + # get the full path of covfile created by stop cmd + covFile=/`echo ${covFile#*/}` + curTime=`date +%s` + covFile_renamed=`echo ${covFile%cov*}`$curTime + mv $covFile $covFile_renamed + if [ $? -ne 0 ];then + printf "${warn_msg}Failed to rename $covFile to $covFile_renamed, please check !!!\n" + exit 0 + fi + + # convert cov file to html + covNameOnly_html=`echo ${covFile_renamed##*/}` + $workspace/gocov convert $covFile_renamed | $workspace/gocov-html > $workspace/$covNameOnly_html'.html' + mv $workspace/$covNameOnly_html'.html' $cov_file_path/$covNameOnly_html'.html' + if [ $? -ne 0 ];then + printf "${warn_msg}Failed to gocov convert $covFile_renamed to $covFile_renamed'.html', please check !!!\n" + exit 0 + fi + + if [[ "$OSTYPE" =~ ^linux.* ]]; then + local_ip=`ip addr | grep "inet " | grep -v 127 | awk '{print $2}' | awk -F '/' '{print $1}'` + fi + + printf "${info_msg}Have fun with http://${local_ip}:${local_port}/coverage/report/$covNameOnly_html.html !!!\n" + fi +} + +# cov or null +sub_cmd=$2 +# check sub cmd +if [[ ! -z $sub_cmd && $sub_cmd != $cov_cmd ]];then + printf "${warn_msg} cmd lists:\n1. build [$cov_cmd]\n2. start [$cov_cmd]\n3. stop [$cov_cmd]\n4. reload [$cov_cmd]\n[$cov_cmd] is optional for codeCov\n" + exit 1 +fi + +case $1 in + "build" ) + build + ;; + "start" ) + start + ;; + "stop" ) + stop + ;; + "reload" ) + stop + start + ;; + * ) + printf "${warn_msg} cmd lists:\n1. build [$cov_cmd]\n2. start [$cov_cmd]\n3. stop [$cov_cmd]\n4. reload [$cov_cmd]\n[$cov_cmd] is optional for codeCov\n" + exit 1 + ;; +esac diff --git a/replayer-agent/control.sh b/replayer-agent/control.sh index fbe1aef..1457fe9 100755 --- a/replayer-agent/control.sh +++ b/replayer-agent/control.sh @@ -20,20 +20,27 @@ error_msg="\033[;31m[ERROR]\033[0m\t" local_ip="127.0.0.1" local_port="8998" -# TODO: tmp for private git -#git_go='https://github.com/didichuxing/sharingan-go/raw/recorder' -git_go='https://git.xiaojukeji.com/nuwa/tools/go/raw/master' +# set github about go +git_go='https://raw.githubusercontent.com/didichuxing/sharingan-go/recorder' +install_go_md='https://github.com/golang/go#download-and-install' +VERSION="go1.13" -# build -function build() { - if [ -z $GOROOT ];then - # install golang1.10 - curl "$git_go/install/go1.10" | sh +function install_go() { + curl "$git_go/install/$VERSION" >> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION : 'curl $git_go/install/$VERSION' \n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + curl "$git_go/install/$VERSION" | sh if [ $? -ne 0 ]; then - printf "${error_msg}build failed at installing golang1.10, please check!!!\n" + printf "${error_msg}build failed at installing $VERSION : 'curl $git_go/install/$VERSION | sh' \n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" exit 1 fi - export GOROOT=/tmp/recorder-go1.10 + export GOROOT="/tmp/recorder-$VERSION" if [ $? -ne 0 ]; then printf "${error_msg}build failed at setting GOROOT, please check!!!\n" exit 1 @@ -43,37 +50,84 @@ function build() { printf "${error_msg}build failed at updating PATH, please check!!!\n" exit 1 fi - fi - which glide &> /dev/null - if [ $? -ne 0 ]; then - # install glide - curl https://glide.sh/get | sh + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# install go +function install_go_tar() { + # get go url + releases_go='https://github.com/didichuxing/sharingan-go/releases/download/' + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.darwin-amd64.tar.gz" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.linux-amd64.tar.gz" + else + printf "${error_msg} Unknown system type! build failed at installing go! \n" + exit 1 + fi + + # install golang + recorder_dir=/tmp/recorder-${VERSION} + wget "$releases_go/$goSuffix" -O "$recorder_dir.tar.gz" if [ $? -ne 0 ]; then - printf "${error_msg}build failed at installing glide, please check!!!\n" + printf "${error_msg}build failed at downloading $VERSION : wget $releases_go/$goSuffix -O $recorder_dir.tar.gz !!!\n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" exit 1 fi - fi + rm -rf $recorder_dir && mkdir -p $recorder_dir + tar -xzf "$recorder_dir.tar.gz" -C ${recorder_dir} --strip-components=1 + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION, please check!!!\n" + exit 1 + fi + rm -rf "$recorder_dir.tar.gz" - if [ -z $GOPATH ];then - prePath="/src/github.com/didichuxing/sharingan" - if [[ $workspace == *$prePath* ]];then - export GOPATH=`echo ${workspace%/src/*}` - if [ $? -ne 0 ]; then - printf "${error_msg}build failed at setting GOPATH, please check!!!\n" - exit 1 - fi - else - printf "${error_msg}build failed at setting GOPATH, please check and clone sharingan to legal path!!!\n" + # export + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" exit 1 fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# build +function build() { + if [ -z $GOROOT ];then + install_go + fi + + which go &> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing 'which go', please update \$PATH!!!\n" + exit 1 + fi + goVer=`go version` + if [[ $goVer < 'go version go1.11' ]];then + printf "${warn_msg}current is $goVer, go mod requires at least go1.11!!!\n" + install_go + fi + + if [ -z $GOPATH ];then + mkdir -p /tmp/replayer + export GOPATH=/tmp/replayer + printf "${warn_msg}No GOPATH, so setting GOPATH=/tmp/replayer \n" fi cd $root - rm -rf vendor - glide install + go clean -modcache + printf "${info_msg}go mod download, please wait~ \n" + go mod download if [ $? -ne 0 ]; then - printf "${error_msg}build failed at executing glide install, please check!!!\n" + printf "${error_msg}build failed at executing go mod download, please check \$GOPROXY or try agian!!!\n" exit 1 fi diff --git a/replayer-agent/control_glide.sh b/replayer-agent/control_glide.sh new file mode 100755 index 0000000..12f5424 --- /dev/null +++ b/replayer-agent/control_glide.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +workspace=$(cd $(dirname $0) && pwd -P) +cd ${workspace} + +app=replayer-agent +root=$(dirname $workspace) + +# set file +agent_file=$workspace/$app +agent_pid_file=$workspace/agent.pid +agent_log_file=$workspace/agent.log + +# set msg +info_msg="\033[;32m[INFO]\033[0m\t" +warn_msg="\033[;33m[WARN]\033[0m\t" +error_msg="\033[;31m[ERROR]\033[0m\t" + +# set web server +local_ip="127.0.0.1" +local_port="8998" + +# set github about go +git_go='https://raw.githubusercontent.com/didichuxing/sharingan-go/recorder' +install_go_md='https://github.com/golang/go#download-and-install' +VERSION="go1.10" + +function install_go() { + curl "$git_go/install/$VERSION" >> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION : 'curl $git_go/install/$VERSION' \n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + curl "$git_go/install/$VERSION" | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION : 'curl $git_go/install/$VERSION | sh' \n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# install go +function install_go_tar() { + # get go url + releases_go='https://github.com/didichuxing/sharingan-go/releases/download/' + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.darwin-amd64.tar.gz" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.linux-amd64.tar.gz" + else + printf "${error_msg} Unknown system type! build failed at installing go! \n" + exit 1 + fi + + # install golang + recorder_dir=/tmp/recorder-${VERSION} + wget "$releases_go/$goSuffix" -O "$recorder_dir.tar.gz" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at downloading $VERSION : wget $releases_go/$goSuffix -O $recorder_dir.tar.gz !!!\n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + rm -rf $recorder_dir && mkdir -p $recorder_dir + tar -xzf "$recorder_dir.tar.gz" -C ${recorder_dir} --strip-components=1 + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION, please check!!!\n" + exit 1 + fi + rm -rf "$recorder_dir.tar.gz" + + # export + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# build +function build() { + if [ -z $GOROOT ];then + printf "${info_msg}Installing go for you ~~~ \n" + install_go + fi + + which glide &> /dev/null + if [ $? -ne 0 ]; then + # install glide + curl https://glide.sh/get | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing glide, please check!!!\n" + exit 1 + fi + fi + + which go &> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing 'which go', please execute 'export PATH=\$GOROOT/bin:\$PATH' !!!\n" + exit 1 + fi + + if [ -z $GOPATH ];then + prePath="/src/github.com/didichuxing/sharingan" + if [[ $workspace == *$prePath* ]];then + export GOPATH=`echo ${workspace%/src/*}` + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOPATH, please check!!!\n" + exit 1 + fi + else + printf "${error_msg}build failed at setting GOPATH, please check and clone sharingan to legal path!!!\n" + exit 1 + fi + fi + + cd $root + rm -rf vendor + glide install + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing glide install, please check!!!\n" + exit 1 + fi + + cd $workspace + go build -o $app main.go + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at go build $app, please check!!!\n" + exit 1 + fi + + printf "${info_msg}$app builds success!!!\n" +} + +# start +function start() { + if [ ! -f "$agent_file" ]; then + printf "${error_msg}please exesute sh control.sh build first!!!\n" + exit 1 + fi + + if [[ "$OSTYPE" =~ ^linux.* ]]; then + local_ip=`ip addr | grep "inet " | grep -v 127 | awk '{print $2}' | awk -F '/' '{print $1}'` + fi + + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -eq 0 ];then + printf "${warn_msg} $app has already started!!! Have fun with http://${local_ip}:${local_port}\n" + exit 0 + fi + + # default -parallel=10 + nohup $agent_file -cursor >> $agent_log_file 2>&1 & + sleep 2 + pid=$! + + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -ne 0 ];then + printf "${error_msg} Failed to start $app, please check $agent_log_file and try again!!!\n" + exit 1 + fi + echo $pid > $agent_pid_file + printf "${info_msg}$app starts success!!! [pid=${pid}] Have fun with http://${local_ip}:${local_port} !\n" +} + +# stop +function stop() { + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -ne 0 ];then + printf "${warn_msg}$app is not running!!!\n" + exit 0 + fi + + pkill $app + sleep 2 + + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -eq 0 ];then + printf "${error_msg}Failed to stop $app, please check and try again!!!\n" + exit 1 + fi + printf "${info_msg}$app stops success!!!\n" +} + +case $1 in + "build" ) + build + ;; + "start" ) + start + ;; + "stop" ) + stop + ;; + "reload" ) + stop + start + ;; + * ) + printf "${warn_msg}unknow command!!! [usage: build、start、stop、reload]\n" + exit 1 + ;; +esac diff --git a/replayer-agent/control_gomod.sh b/replayer-agent/control_gomod.sh new file mode 100755 index 0000000..1457fe9 --- /dev/null +++ b/replayer-agent/control_gomod.sh @@ -0,0 +1,212 @@ +#!/bin/bash + +workspace=$(cd $(dirname $0) && pwd -P) +cd ${workspace} + +app=replayer-agent +root=$(dirname $workspace) + +# set file +agent_file=$workspace/$app +agent_pid_file=$workspace/agent.pid +agent_log_file=$workspace/agent.log + +# set msg +info_msg="\033[;32m[INFO]\033[0m\t" +warn_msg="\033[;33m[WARN]\033[0m\t" +error_msg="\033[;31m[ERROR]\033[0m\t" + +# set web server +local_ip="127.0.0.1" +local_port="8998" + +# set github about go +git_go='https://raw.githubusercontent.com/didichuxing/sharingan-go/recorder' +install_go_md='https://github.com/golang/go#download-and-install' +VERSION="go1.13" + +function install_go() { + curl "$git_go/install/$VERSION" >> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION : 'curl $git_go/install/$VERSION' \n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + curl "$git_go/install/$VERSION" | sh + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION : 'curl $git_go/install/$VERSION | sh' \n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# install go +function install_go_tar() { + # get go url + releases_go='https://github.com/didichuxing/sharingan-go/releases/download/' + if [[ "$OSTYPE" =~ ^darwin.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.darwin-amd64.tar.gz" + elif [[ "$OSTYPE" =~ ^linux.* ]]; then + goSuffix="$VERSION.recorder/$VERSION.linux-amd64.tar.gz" + else + printf "${error_msg} Unknown system type! build failed at installing go! \n" + exit 1 + fi + + # install golang + recorder_dir=/tmp/recorder-${VERSION} + wget "$releases_go/$goSuffix" -O "$recorder_dir.tar.gz" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at downloading $VERSION : wget $releases_go/$goSuffix -O $recorder_dir.tar.gz !!!\n" + printf "${error_msg}Please install $VERSION manually (refer to $install_go_md) and try again!!! \n" + printf "${warn_msg}Please execute 'export GOROOT=/tmp/recorder-$VERSION && export PATH=\$GOROOT/bin:\$PATH' after installing $VERSION!!!\n" + exit 1 + fi + rm -rf $recorder_dir && mkdir -p $recorder_dir + tar -xzf "$recorder_dir.tar.gz" -C ${recorder_dir} --strip-components=1 + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at installing $VERSION, please check!!!\n" + exit 1 + fi + rm -rf "$recorder_dir.tar.gz" + + # export + export GOROOT="/tmp/recorder-$VERSION" + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at setting GOROOT, please check!!!\n" + exit 1 + fi + export PATH=$GOROOT/bin:$PATH + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at updating PATH, please check!!!\n" + exit 1 + fi + + printf "${info_msg}GOROOT=$GOROOT \n" +} + +# build +function build() { + if [ -z $GOROOT ];then + install_go + fi + + which go &> /dev/null + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing 'which go', please update \$PATH!!!\n" + exit 1 + fi + goVer=`go version` + if [[ $goVer < 'go version go1.11' ]];then + printf "${warn_msg}current is $goVer, go mod requires at least go1.11!!!\n" + install_go + fi + + if [ -z $GOPATH ];then + mkdir -p /tmp/replayer + export GOPATH=/tmp/replayer + printf "${warn_msg}No GOPATH, so setting GOPATH=/tmp/replayer \n" + fi + + cd $root + go clean -modcache + printf "${info_msg}go mod download, please wait~ \n" + go mod download + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at executing go mod download, please check \$GOPROXY or try agian!!!\n" + exit 1 + fi + + cd $workspace + go build -o $app main.go + if [ $? -ne 0 ]; then + printf "${error_msg}build failed at go build $app, please check!!!\n" + exit 1 + fi + + printf "${info_msg}$app builds success!!!\n" +} + +# start +function start() { + if [ ! -f "$agent_file" ]; then + printf "${error_msg}please exesute sh control.sh build first!!!\n" + exit 1 + fi + + if [[ "$OSTYPE" =~ ^linux.* ]]; then + local_ip=`ip addr | grep "inet " | grep -v 127 | awk '{print $2}' | awk -F '/' '{print $1}'` + fi + + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -eq 0 ];then + printf "${warn_msg} $app has already started!!! Have fun with http://${local_ip}:${local_port}\n" + exit 0 + fi + + # default -parallel=10 + nohup $agent_file -cursor >> $agent_log_file 2>&1 & + sleep 2 + pid=$! + + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -ne 0 ];then + printf "${error_msg} Failed to start $app, please check $agent_log_file and try again!!!\n" + exit 1 + fi + echo $pid > $agent_pid_file + printf "${info_msg}$app starts success!!! [pid=${pid}] Have fun with http://${local_ip}:${local_port} !\n" +} + +# stop +function stop() { + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -ne 0 ];then + printf "${warn_msg}$app is not running!!!\n" + exit 0 + fi + + pkill $app + sleep 2 + + ps -ef | grep $app | grep -v grep &> /dev/null + if [ $? -eq 0 ];then + printf "${error_msg}Failed to stop $app, please check and try again!!!\n" + exit 1 + fi + printf "${info_msg}$app stops success!!!\n" +} + +case $1 in + "build" ) + build + ;; + "start" ) + start + ;; + "stop" ) + stop + ;; + "reload" ) + stop + start + ;; + * ) + printf "${warn_msg}unknow command!!! [usage: build、start、stop、reload]\n" + exit 1 + ;; +esac