Add prometheus metric for canary/bench (cadence-workflow#4625)

bench/README.md

@@ -71,6 +71,7 @@ bench:
 cadence:
   service: "cadence-frontend" # frontend service name
   host: "127.0.0.1:7933" # frontend address
+  #metrics: ... # optional detailed client side metrics like workflow latency  
 ```
 - **Metrics**: metrics configuration. Similar to server metric emitter, only M3/Statsd/Prometheus is supported. 
 - **Log**: logging configuration.  Similar to server logging configuration. 
@@ -207,4 +208,4 @@ When all tests complete, `Cron` will update the value of the `Passed` search att
 A sample cron configuration is in `config/bench/cron.json`, and it can be started with
 ```
 cadence --do <domain> wf start --tl cadence-bench-tl-0 --wt cron-test-workflow --dt 30 --et 7200 --if config/bench/cron.json
-```
+```

canary/README.md

@@ -84,6 +84,7 @@ cadence:
   service: "cadence-frontend" # frontend service name
   address: "127.0.0.1:7833" # frontend address
   #host: "127.0.0.1:7933" # replace address with host if using Thrift for compatibility
+  #metrics: ... # optional detailed client side metrics like workflow latency. But for monitoring, simply use server side metrics `workflow_success` is enough.
 ```
 - **Metrics**: metrics configuration. Similar to server metric emitter, only M3/Statsd/Prometheus is supported. 
 - **Log**: logging configuration.  Similar to server logging configuration. 
@@ -263,4 +264,4 @@ Batch workflow tests the batch job feature. Make sure advanced visibility featur
 To manually start one run of this test case:
 ```
 cadence --do <> workflow start --tl canary-task-queue --et 10 --wt workflow.batch -i 0
-```
+```

config/bench/development.yaml

@@ -6,3 +6,8 @@ bench:
 cadence:
   service: "cadence-frontend"
   host: "127.0.0.1:7933"
+  metrics:
+    statsd: ~
+    prometheus:
+      timerType: "histogram"
+      listenAddress: "127.0.0.1:8004"

config/canary/development.yaml

@@ -6,3 +6,8 @@ cadence:
   service: "cadence-frontend"
   address: "127.0.0.1:7833"
   #host: "127.0.0.1:7933" # replace address with host if using Thrift for compatibility
+  metrics:
+    statsd: ~
+    prometheus:
+      timerType: "histogram"
+      listenAddress: "127.0.0.1:8005"