Merge pull request LesnyRumcajs#171 from LesnyRumcajs/scenarios

introduce grpc scenarios instead of fixed proto and variable payloads

.gitignore

@@ -1,2 +1,4 @@
 results/
 *.tmp
+proto/
+payload/

README.md

@@ -52,7 +52,7 @@ The benchmark can be configured through the following environment variables:
 |--------|---------------|:---------------:|
 |GRPC_BENCHMARK_DURATION|Duration of the benchmark.|20s|
 |GRPC_BENCHMARK_WARMUP|Duration of the warmup. Stats won't be collected.|5s|
-|GRPC_REQUEST_PAYLOAD|File (from [payload/](payload/)) containing the data to be sent in the client request.|100B|
+|GRPC_REQUEST_SCENARIO|Scenario (from [scenarios/](scenarios/)) containing the protobuf and the data to be sent in the client request.|complex_proto|
 |GRPC_SERVER_CPUS|Maximum number of cpus used by the server.|1|
 |GRPC_SERVER_RAM|Maximum memory used by the server.|512m|
 |GRPC_CLIENT_CONNECTIONS|Number of connections to use.|50|
@@ -63,10 +63,11 @@ The benchmark can be configured through the following environment variables:
 ### Parameter recommendations
 * `GRPC_BENCHMARK_DURATION` should not be too small. Some implementations need a *warm-up* before achieving their optimal performance and most real-life gRPC services are expected to be long running processes. From what we measured, **300s** should be enough.
 * `GRPC_SERVER_CPUS` + `GRPC_CLIENT_CPUS` should not exceed total number of cores on the machine. The reason for this is that you don't want the `ghz` client to steal precious CPU cycles from the service under test. Keep in mind that having the `GRPC_CLIENT_CPUS` too low may not saturate the service in some of the more performant implementations. Also keep in mind limiting the number of `GRPC_SERVER_CPUS` to 1 will severely hamper the performance for some technologies - is running a service on 1 CPU your use case? It may be, but keep in mind eventual load balancer also incurs some costs.
+* `GRPC_REQUEST_SCENARIO` is a parameter to both `build.sh` and `bench.sh`. The images must be rebuilt each time you intend to use a scenario having a different `helloworld.proto` from the one ran previously.
 
 Other parameters will depend on your use-case. Choose wisely.
 
 # Results
-You can find our sample results in the [Wiki](https://github.com/LesnyRumcajs/grpc_bench/wiki). Be sure to run the benchmarks yourself if you have sufficient hardware, especially for multi-core scenarios. 
+You can find our sample results in the [Wiki](https://github.com/LesnyRumcajs/grpc_bench/wiki). Be sure to run the benchmarks yourself if you have sufficient hardware, especially for multi-core scenarios.
 
 

analyze.sh

@@ -21,5 +21,5 @@ $(git log -1 --pretty="%h %cD %cn %s")
 - GRPC_CLIENT_CONCURRENCY=${GRPC_CLIENT_CONCURRENCY}
 - GRPC_CLIENT_QPS=${GRPC_CLIENT_QPS}
 - GRPC_CLIENT_CPUS=${GRPC_CLIENT_CPUS}
-- GRPC_REQUEST_PAYLOAD=${GRPC_REQUEST_PAYLOAD}
+- GRPC_REQUEST_SCENARIO=${GRPC_REQUEST_SCENARIO}
 EOF

bench.sh

@@ -15,7 +15,7 @@ export GRPC_CLIENT_CONCURRENCY=${GRPC_CLIENT_CONCURRENCY:-"1000"}
 export GRPC_CLIENT_QPS=${GRPC_CLIENT_QPS:-"0"}
 export GRPC_CLIENT_QPS=$(( GRPC_CLIENT_QPS / GRPC_CLIENT_CONCURRENCY ))
 export GRPC_CLIENT_CPUS=${GRPC_CLIENT_CPUS:-"1"}
-export GRPC_REQUEST_PAYLOAD=${GRPC_REQUEST_PAYLOAD:-"100B"}
+export GRPC_REQUEST_SCENARIO=${GRPC_REQUEST_SCENARIO:-"complex_proto"}
 
 # Let containers know how many CPUs they will be running on
 # Additionally export other vars for further analysis script.
@@ -34,6 +34,12 @@ for benchmark in ${BENCHMARKS_TO_RUN}; do
 
 	mkdir -p "${RESULTS_DIR}"
 
+	# Setup the chosen scenario
+    if ! sh setup_scenario.sh $GRPC_REQUEST_SCENARIO true; then
+  		echo "Scenario setup fiascoed."
+  		exit 1
+	fi
+
 	# Start the gRPC Server container
 	docker run --name "${NAME}" --rm \
 		--cpus "${GRPC_SERVER_CPUS}" \
@@ -60,7 +66,7 @@ for benchmark in ${BENCHMARKS_TO_RUN}; do
             --connections="${GRPC_CLIENT_CONNECTIONS}" \
             --rps="${GRPC_CLIENT_QPS}" \
             --duration "${GRPC_BENCHMARK_WARMUP}" \
-            --data-file /payload/"${GRPC_REQUEST_PAYLOAD}" \
+            --data-file /payload/payload \
     		127.0.0.1:50051 > /dev/null
 
     	echo "done."
@@ -86,7 +92,7 @@ for benchmark in ${BENCHMARKS_TO_RUN}; do
         --connections="${GRPC_CLIENT_CONNECTIONS}" \
         --rps="${GRPC_CLIENT_QPS}" \
         --duration "${GRPC_BENCHMARK_DURATION}" \
-        --data-file /payload/"${GRPC_REQUEST_PAYLOAD}" \
+        --data-file /payload/payload \
 		127.0.0.1:50051 >"${RESULTS_DIR}/${NAME}".report
 
 	# Show quick summary (reqs/sec)

build.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+export GRPC_REQUEST_SCENARIO=${GRPC_REQUEST_SCENARIO:-"complex_proto"}
+
 # Build ghz Docker image.
 # See ghz-tool/Dockerfile for details/version
 docker build -t ghz_bench:latest ./ghz-tool/
@@ -9,6 +11,12 @@ BENCHMARKS_TO_BUILD="${@}"
 ##  ...or use all the *_bench dirs by default
 BENCHMARKS_TO_BUILD="${BENCHMARKS_TO_BUILD:-$(find . -maxdepth 1 -name '*_bench' -type d | sort)}"
 
+# Setup the chosen scenario
+if ! sh setup_scenario.sh $GRPC_REQUEST_SCENARIO false; then
+	echo "Scenario setup fiascoed."
+	exit 1
+fi
+
 builds=""
 for benchmark in ${BENCHMARKS_TO_BUILD}; do
 	echo "==> Building Docker image for ${benchmark}..."

cpp_asio_grpc_bench/main.cpp

@@ -49,7 +49,7 @@ void spawn_accept_loop(
         return;
       }
       helloworld::HelloReply response;
-      response.set_message(context->request.name());
+      *response.mutable_response() = std::move(*context->request.mutable_request());
       auto &writer = context->writer;
       agrpc::finish(writer, response, grpc::Status::OK,
                     boost::asio::bind_executor(
@@ -92,4 +92,4 @@ int main() {
   }
 
   server->Shutdown();
-}
+}

cpp_grpc_mt_bench/main.cpp

@@ -109,7 +109,7 @@ class ServerImpl final {
         new CallData(service_, cq_);
 
         // The actual processing.
-        reply_.set_message(request_.name());
+        *reply_.mutable_response() = std::move(*request_.mutable_request());
 
         // And we are done! Let the gRPC runtime know we've finished, using the
         // memory address of this instance as the uniquely identifying tag for

cpp_grpc_st_bench/main.cpp

@@ -101,7 +101,7 @@ class ServerImpl final {
         new CallData(service_, cq_);
 
         // The actual processing.
-        reply_.set_message(request_.name());
+        *reply_.mutable_response() = std::move(*request_.mutable_request());
 
         // And we are done! Let the gRPC runtime know we've finished, using the
         // memory address of this instance as the uniquely identifying tag for

crystal_grpc_bench/src/server.cr

@@ -6,7 +6,7 @@ require "./protobufs/helloworld.pb"
 
 class HelloWorldHandler < Helloworld::Greeter
   def say_hello(request : Helloworld::HelloRequest) : Helloworld::HelloReply
-    Helloworld::HelloReply.new(message: request.name)
+    Helloworld::HelloReply.new(response: request.request)
   end
 end
 

csharp_grpc_bench/GreeterServer/Program.cs

@@ -24,7 +24,7 @@ class GreeterImpl : Greeter.GreeterBase
         // Server side handler of the SayHello RPC
         public override Task<HelloReply> SayHello(HelloRequest request, ServerCallContext context)
         {
-            return Task.FromResult(new HelloReply { Message = request.Name });
+            return Task.FromResult(new HelloReply { Response = request.Request });
         }
     }
 

dotnet_grpc_bench/GreeterServer/Services/GreeterService.cs

@@ -23,7 +23,7 @@ public class GreeterService : Greeter.GreeterBase
         // Server side handler of the SayHello RPC
         public override Task<HelloReply> SayHello(HelloRequest request, ServerCallContext context)
         {
-            return Task.FromResult(new HelloReply { Message = request.Name });
+            return Task.FromResult(new HelloReply { Response = request.Request });
         }
     }
 }

elixir_grpc_bench/lib/server.ex

@@ -4,6 +4,6 @@ defmodule Helloworld.Greeter.Server do
   @spec say_hello(Helloworld.HelloRequest.t(), GRPC.Server.Stream.t()) ::
           Helloworld.HelloReply.t()
   def say_hello(request, _stream) do
-    Helloworld.HelloReply.new(message: "#{request.name}")
+    Helloworld.HelloReply.new(response: request.request)
   end
 end

erlang_grpcbox_bench/src/egb_handler.erl

@@ -9,6 +9,6 @@
 -spec say_hello(ctx:ctx(), helloworld_pb:hello_request()) ->
           {ok, helloworld_pb:hello_reply(), ctx:ctx()} | grpcbox_stream:grpc_error_response().
 
-say_hello(Ctx, #{name := Name}) ->
-    Rep = #{message => Name},
+say_hello(Ctx, #{request := Request}) ->
+    Rep = #{response => Request},
     {ok, Rep, Ctx}.

go_grpc_bench/example/main.go

@@ -39,7 +39,7 @@ type server struct {
 
 // SayHello implements helloworld.GreeterServer
 func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
-	return &pb.HelloReply{Message: in.GetName()}, nil
+	return &pb.HelloReply{Response: in.GetRequest()}, nil
 }
 
 func main() {

java_aot_bench/src/main/java/helloworld/GreetingEndpoint.java

@@ -22,8 +22,7 @@ public GreetingEndpoint(GreetingService greetingService) {
     @Override
     public void sayHello(HelloRequest request, StreamObserver<HelloReply> responseObserver) {
         // <3>
-        final String message = greetingService.sayHello(request.getName());
-        HelloReply reply = HelloReply.newBuilder().setMessage(message).build();
+        final var reply = HelloReply.newBuilder().setResponse(request.getRequest()).build();
         responseObserver.onNext(reply);
         responseObserver.onCompleted();
     }

java_hotspot_grpc_sgc_bench/src/main/java/io/grpc/examples/helloworld/HelloWorldServer.java

@@ -53,14 +53,14 @@ private void start() throws IOException {
 
   /**
    * Allow customization of the Executor with two environment variables:
-   * 
+   *
    * <p>
    * <ul>
    * <li>JVM_EXECUTOR_TYPE: direct, workStealing, single, fixed, cached</li>
    * <li>JVM_EXECUTOR_THREADS: integer value.</li>
    * </ul>
    * </p>
-   * 
+   *
    * The number of Executor Threads will default to the number of
    * availableProcessors(). Only the workStealing and fixed executors will use
    * this value.
@@ -107,7 +107,7 @@ static class GreeterImpl extends GreeterGrpc.GreeterImplBase {
 
     @Override
     public void sayHello(HelloRequest req, StreamObserver<HelloReply> responseObserver) {
-      var reply = HelloReply.newBuilder().setMessage(req.getName()).build();
+      final var reply = HelloReply.newBuilder().setResponse(req.getRequest()).build();
       responseObserver.onNext(reply);
       responseObserver.onCompleted();
     }

java_micronaut_bench/src/main/java/helloworld/GreetingEndpoint.java

@@ -22,8 +22,7 @@ public GreetingEndpoint(GreetingService greetingService) {
     @Override
     public void sayHello(HelloRequest request, StreamObserver<HelloReply> responseObserver) {
         // <3>
-        final String message = greetingService.sayHello(request.getName());
-        HelloReply reply = HelloReply.newBuilder().setMessage(message).build();
+        HelloReply reply = HelloReply.newBuilder().setResponse(request.getRequest()).build();
         responseObserver.onNext(reply);
         responseObserver.onCompleted();
     }

java_micronaut_workstealing_bench/src/main/java/helloworld/GreetingEndpoint.java

@@ -22,8 +22,7 @@ public GreetingEndpoint(GreetingService greetingService) {
     @Override
     public void sayHello(HelloRequest request, StreamObserver<HelloReply> responseObserver) {
         // <3>
-        final String message = greetingService.sayHello(request.getName());
-        HelloReply reply = HelloReply.newBuilder().setMessage(message).build();
+        final var reply = HelloReply.newBuilder().setResponse(request.getRequest()).build();
         responseObserver.onNext(reply);
         responseObserver.onCompleted();
     }

java_quarkus_bench/src/main/java/com/example/HelloService.java

@@ -12,7 +12,7 @@ public class HelloService extends MutinyGreeterGrpc.GreeterImplBase {
     @Override
     public Uni<HelloReply> sayHello(HelloRequest request) {
         return Uni.createFrom().item(
-                HelloReply.newBuilder().setMessage(request.getName()).build()
+                HelloReply.newBuilder().setResponse(request.getRequest()).build()
         );
     }
 }

kotlin_grpc_bench/src/main/kotlin/io/grpc/examples/helloworld/HelloWorldServer.kt

@@ -72,7 +72,7 @@ class HelloWorldServer(val port: Int) {
     private class HelloWorldService : GreeterGrpcKt.GreeterCoroutineImplBase() {
         override suspend fun sayHello(request: HelloRequest) = HelloReply
                 .newBuilder()
-                .setMessage("${request.name}")
+                .setResponse(request.getRequest())
                 .build()
     }
 }

node_grpcjs_st_bench/greeter_server.js

@@ -34,7 +34,7 @@ var hello_proto = grpc.loadPackageDefinition(packageDefinition).helloworld;
  * Implements the SayHello RPC method.
  */
 function sayHello(call, callback) {
-  callback(null, {message: call.request.name});
+  callback(null, {response: call.request.request});
 }
 
 /**