From d2e3365190a2a13b23ab90704960c059c060cbef Mon Sep 17 00:00:00 2001 From: costajob Date: Fri, 9 Sep 2016 15:20:28 +0200 Subject: [PATCH] Mike - added benchmarks --- Gemfile | 11 +++++ Gemfile.lock | 106 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 76 ++++++++++++++++++++++++++++++++ config.ru | 36 +++++++++++++++ lib/grape_app.rb | 9 ++++ lib/nyny_app.rb | 7 +++ lib/rack_app.rb | 7 +++ lib/ramaze_app.rb | 9 ++++ lib/roda_app.rb | 9 ++++ lib/sinatra_app.rb | 7 +++ 10 files changed, 277 insertions(+) create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 README.md create mode 100644 config.ru create mode 100644 lib/grape_app.rb create mode 100644 lib/nyny_app.rb create mode 100644 lib/rack_app.rb create mode 100644 lib/ramaze_app.rb create mode 100644 lib/roda_app.rb create mode 100644 lib/sinatra_app.rb diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..a3dff19 --- /dev/null +++ b/Gemfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true +# A sample Gemfile +source "https://rubygems.org" + +gem "sinatra" +gem "roda" +gem "rack-app" +gem "nyny" +gem "ramaze" +gem "grape" +gem "puma" diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..630889e --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,106 @@ +GEM + remote: https://rubygems.org/ + specs: + actionpack (4.1.16) + actionview (= 4.1.16) + activesupport (= 4.1.16) + rack (~> 1.5.2) + rack-test (~> 0.6.2) + actionview (4.1.16) + activesupport (= 4.1.16) + builder (~> 3.1) + erubis (~> 2.7.0) + activesupport (4.1.16) + i18n (~> 0.6, >= 0.6.9) + json (~> 1.7, >= 1.7.7) + minitest (~> 5.1) + thread_safe (~> 0.1) + tzinfo (~> 1.1) + axiom-types (0.1.1) + descendants_tracker (~> 0.0.4) + ice_nine (~> 0.11.0) + thread_safe (~> 0.3, >= 0.3.1) + better_errors (1.1.0) + coderay (>= 1.0.0) + erubis (>= 2.6.6) + builder (3.2.2) + coderay (1.1.1) + coercible (1.0.0) + descendants_tracker (~> 0.0.1) + descendants_tracker (0.0.4) + thread_safe (~> 0.3, >= 0.3.1) + enumerable-lazy (0.0.1) + equalizer (0.0.11) + erubis (2.7.0) + grape (0.17.0) + activesupport + builder + hashie (>= 2.1.0) + multi_json (>= 1.3.2) + multi_xml (>= 0.5.2) + mustermann19 (~> 0.4.3) + rack (>= 1.3.0) + rack-accept + virtus (>= 1.0.0) + hashie (3.4.4) + i18n (0.7.0) + ice_nine (0.11.2) + innate (2013.02.21) + rack (~> 1.5.2) + json (1.8.3) + minitest (5.9.0) + multi_json (1.12.1) + multi_xml (0.5.5) + mustermann19 (0.4.4) + enumerable-lazy + nyny (3.4.3) + actionpack (~> 4.1.6) + better_errors (~> 1.1.0) + rack-contrib (~> 1.1.0) + tilt (~> 1.4.1) + puma (3.5.2) + rack (1.5.5) + rack-accept (0.4.5) + rack (>= 0.4) + rack-app (5.2.0) + rack + rack-contrib (1.1.0) + rack (>= 0.9.1) + rack-protection (1.5.3) + rack + rack-test (0.6.3) + rack (>= 1.0) + rake (11.2.2) + ramaze (2012.12.08) + innate (>= 2012.12) + rake + roda (2.17.0) + rack + sinatra (1.4.7) + rack (~> 1.5) + rack-protection (~> 1.4) + tilt (>= 1.3, < 3) + thread_safe (0.3.5) + tilt (1.4.1) + tzinfo (1.2.2) + thread_safe (~> 0.1) + virtus (1.0.5) + axiom-types (~> 0.1) + coercible (~> 1.0) + descendants_tracker (~> 0.0, >= 0.0.3) + equalizer (~> 0.0, >= 0.0.9) + +PLATFORMS + ruby + +DEPENDENCIES + grape + nyny + puma + rack-app + ramaze + roda + sinatra + +BUNDLED WITH + 1.12.5 diff --git a/README.md b/README.md new file mode 100644 index 0000000..a315e2c --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +## Table of Contents + +* [Scope](#scope) +* [What's pico?](#what-s-pico) +* [Included frameworks](#included-frameworks) + * [Raw Rack](#raw-rack) +* [Application](#application) +* [Benchmarks](#benchmarks) + * [Platform](#platform) + * [Puma](#puma) + * [Wrk](#wrk) + * [Bootstrap](#bootstrap) + * [Results](#results) + +## Scope +This is an (un)fair benchmark of most mature (version >= 1) pico-framework available for the Ruby programming language. + +## What's pico? +With pico i intend very tiny routing Web framework, with almost no dependencies but for Rack. + +## Included frameworks +Here are the list of the pico-frameworks included in the benchmark: +* [Sinatra](http://www.sinatrarb.com/): is one of the first micro-frameworks for ruby, the most feature complete of the pack +* [Roda](http://roda.jeremyevans.net/): born form the ashes of [Cuba](http://cuba.is/) a performant tree-routing framework that can be extended via plug-ins +* [RackApp](http://www.rack-app.com/): a performant pico framework dependent on Rack only +* [NyNy](http://alisnic.github.io/nyny/): a tiny Web framework, dependent from ActionPack +* [Ramaze](http://ramaze.net/): a small, modular Web framework +* [Grape](https://github.com/ruby-grape/grape): an opinionated framework, with several dependencies + +### Raw Rack +I also included a plain rack application to see how much each solution diverge from the raw metal. + +## Application +The "application" i tested is barely minimal: it is the HTTP version of the "Hello World" example. + +## Benchmarks + +### Platform +I registered these benchmarks with a MacBook PRO 15 mid 2015 having these specs: +* OSX El Captain +* 2,2 GHz Intel Core i7 (4 cores) +* 16 GB 1600 MHz DDR3 +* Ruby 2.3.1p112 + +### Puma +All of the pico framework run over the mighty [Puma](http://puma.io/) application server. + +### RAM and CPU +I measured RAM and CPU consumption by using the Apple XCode's Instruments and recording max consumption peak. + +### Wrk +I used [wrk](https://github.com/wg/wrk) as the loading tool. +I measured each application server three times, picking the best lap (apart for JVM that demands warm-up). +Here is the common script i used: + +``` +wrk -t 4 -c 100 -d30s --timeout 2000 http://127.0.0.1:9292 +``` + +### Bootstrap +``` +bundle exec puma -w 7 --preload config.ru +``` + +### Results +Here are the benchmarks results ordered by increasing throughput. + +| App Server | Throughput (req/s) | Latency in ms (avg/stdev/max) | Delta from rack % | +| :------------| -----------------: | ----------------------------: | ----------------: | +| RackApp | !error! | !error! | 100.0 | +| Ramaze | 4127.44 | 20.17/19.42/299.00 | 90.7 | +| Grape | 15393.65 | 3.58/3.99/62.34 | 65.6 | +| Sinatra | 19130.20 | 4.47/4.33/128.41 | 57.2 | +| NyNy | 22124.41 | 4.22/3.36/87.65 | 50.6 | +| Roda | 40492.11 | 2.55/4.09/130.58 | 9.6 | +| Rack | 44813.41 | 2.19/2.80/96.79 | 0.0 | diff --git a/config.ru b/config.ru new file mode 100644 index 0000000..d22112a --- /dev/null +++ b/config.ru @@ -0,0 +1,36 @@ +MESSAGE = "Hello world!" + +$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__)) +require "sinatra_app" +require "roda_app" +require "rack_app" +require "nyny_app" +require "ramaze_app" +require "grape_app" + +run Proc.new { |env| ["200", {"Content-Type" => "text/html"}, [MESSAGE]] } + +map "/sinatra" do + run SinatraApp +end + +map "/roda" do + run RodaApp.freeze.app +end + +map "/rackapp" do + run RackApp +end + +map "/nyny" do + run NyNyApp.new +end + +map "/ramaze" do + Ramaze.start(:file => __FILE__, :started => true) + run Ramaze +end + +map "/grape" do + run GrapeApp +end diff --git a/lib/grape_app.rb b/lib/grape_app.rb new file mode 100644 index 0000000..813b66f --- /dev/null +++ b/lib/grape_app.rb @@ -0,0 +1,9 @@ +require "grape" + +class GrapeApp < Grape::API + format :txt + + get do + MESSAGE + end +end diff --git a/lib/nyny_app.rb b/lib/nyny_app.rb new file mode 100644 index 0000000..c8f3a5b --- /dev/null +++ b/lib/nyny_app.rb @@ -0,0 +1,7 @@ +require "nyny" + +class NyNyApp < NYNY::App + get "/" do + MESSAGE + end +end diff --git a/lib/rack_app.rb b/lib/rack_app.rb new file mode 100644 index 0000000..44196e0 --- /dev/null +++ b/lib/rack_app.rb @@ -0,0 +1,7 @@ +require "rack/app" + +class RackApp < Rack::App + get "/" do + MESSAGE + end +end diff --git a/lib/ramaze_app.rb b/lib/ramaze_app.rb new file mode 100644 index 0000000..6294628 --- /dev/null +++ b/lib/ramaze_app.rb @@ -0,0 +1,9 @@ +require "ramaze" + +class RamazeApp < Ramaze::Controller + def index + MESSAGE + end +end + +Ramaze::Log.loggers.clear diff --git a/lib/roda_app.rb b/lib/roda_app.rb new file mode 100644 index 0000000..54e0e2b --- /dev/null +++ b/lib/roda_app.rb @@ -0,0 +1,9 @@ +require 'roda' + +class RodaApp < Roda + route do |r| + r.get do + MESSAGE + end + end +end diff --git a/lib/sinatra_app.rb b/lib/sinatra_app.rb new file mode 100644 index 0000000..b5b28d0 --- /dev/null +++ b/lib/sinatra_app.rb @@ -0,0 +1,7 @@ +require 'sinatra/base' + +class SinatraApp < Sinatra::Base + get '/' do + MESSAGE + end +end