Merge branch 'backtesting' into develop

Conflicts:
	core/util.js
	gekko.js
	sample-config.js

core/backtestMarket.js

@@ -0,0 +1,90 @@
+var _ = require('lodash');
+var util = require('./util');
+var config = util.getConfig();
+var dirs = util.dirs();
+var log = require('./log');
+var moment = require('moment');
+
+var Reader = require(dirs.gekko + config.adapter.path);
+
+var daterange = config.backtest.daterange;
+
+var Market = function() {
+  _.bindAll(this);
+
+  this.pushing = false;
+  this.ended = false;
+
+  Readable.call(this, {objectMode: true});
+
+  console.log('');
+  log.info('\tWARNING: BACKTESTING FEATURE NEEDS PROPER TESTING');
+  log.info('\tWARNING: ACT ON THESE NUMBERS AT YOUR OWN RISK!');
+  console.log('');
+
+  this.reader = new Reader();
+  this.batchSize = config.backtest.batchSize;
+  this.iterator = {
+    from: daterange.from,
+    to: daterange.from.clone().add(this.batchSize, 'm').subtract(1, 's')
+  }
+}
+
+var Readable = require('stream').Readable;
+Market.prototype = Object.create(Readable.prototype, {
+  constructor: { value: Market }
+});
+
+Market.prototype._read = function noop() {
+  if(this.pushing)
+    return;
+
+  this.get();
+}
+
+// Market.prototype.start = function() {
+//   return this;
+// }
+
+Market.prototype.get = function() {
+  if(this.iterator.to >= daterange.to) {
+    this.iterator.to = daterange.to;
+    this.ended = true;
+  }
+
+  this.reader.get(
+    this.iterator.from.unix(),
+    this.iterator.to.unix(),
+    this.processCandles
+  )
+}
+
+Market.prototype.processCandles = function(candles) {
+  this.pushing = true;
+  var amount = _.size(candles);
+  _.each(candles, function(c, i) {
+    c.start = moment.unix(c.start);
+
+    if(++i === amount) {
+      // last one candle from batch
+      if(!this.ended)
+        this.pushing = false;
+      else {
+        _.defer(function() {
+          this.reader.close();
+          this.emit('end');
+        }.bind(this));
+      }
+    }
+
+    this.push(c);
+
+  }, this);
+
+  this.iterator = {
+    from: this.iterator.from.clone().add(this.batchSize, 'm'),
+    to: this.iterator.from.clone().add(this.batchSize * 2, 'm').subtract(1, 's')
+  }
+}
+
+module.exports = Market;

core/budfox/budfox.js

@@ -10,8 +10,6 @@ var Heart = require(dirs.budfox + 'heart');
 var MarketDataProvider =  require(dirs.budfox + 'marketDataProvider');
 var CandleManager = require(dirs.budfox + 'candleManager');
 
-var Readable = require('stream').Readable;
-
 var BudFox = function(config) {
   _.bindAll(this);
 
@@ -43,6 +41,8 @@ var BudFox = function(config) {
     this.pushCandles
   );
 
+  this.heart.pump();
+
   //    Budfox also reports:
 
   // Trades & last trade
@@ -57,18 +57,14 @@ var BudFox = function(config) {
   // );
 }
 
+var Readable = require('stream').Readable;
+
 BudFox.prototype = Object.create(Readable.prototype, {
   constructor: { value: BudFox }
 });
 
 BudFox.prototype._read = function noop() {}
 
-BudFox.prototype.start = function() {
-  this.heart.pump();
-
-  return this;
-}
-
 BudFox.prototype.pushCandles = function(candles) {
   _.each(candles, this.push);
 }

core/gekkoStream.js

@@ -4,22 +4,30 @@
 var Writable = require('stream').Writable;
 var _ = require('lodash');
 
+var mode = require('./util').gekkoMode();
+
 var Gekko = function(candleConsumers) {
   this.candleConsumers = candleConsumers;
   Writable.call(this, {objectMode: true});
+
+  this.finalize = _.bind(this.finalize, this);
 }
 
 Gekko.prototype = Object.create(Writable.prototype, {
   constructor: { value: Gekko }
 });
 
-Gekko.prototype._write = function(chunk, encoding, callback) {
-  // TODO: use different ticks and pause until all are done
+Gekko.prototype._write = function(chunk, encoding, _done) {
+  var done = _.after(this.candleConsumers.length, _done);
   _.each(this.candleConsumers, function(c) {
-    c.processCandle(chunk);
+    c.processCandle(chunk, done);
   });
+}
 
-  callback();
+Gekko.prototype.finalize = function() {
+  _.each(this.candleConsumers, function(c) {
+    c.finalize();
+  });
 }
 
 module.exports = Gekko;

core/pluginUtil.js

@@ -56,13 +56,24 @@ var pluginHelper = {
     if(!plugin.config)
       log.warn(
         'unable to find',
-        plugin.slug,
+        plugin.name,
         'in the config. Is your config up to date?'
       );
 
     if(!plugin.config || !plugin.config.enabled)
       return next();
 
+    if(!_.contains(plugin.modes, gekkoMode)) {
+      log.warn(
+        'The plugin',
+        plugin.name,
+        'does not support the mode',
+        gekkoMode + '.',
+        'It has been disabled.'
+      )
+      return next();
+    }
+
     log.info('Setting up:');
     log.info('\t', plugin.name);
     log.info('\t', plugin.description);

core/realtimeMarket.js

@@ -0,0 +1,22 @@
+var _ = require('lodash');
+
+var util = require('./util');
+var dirs = util.dirs();
+
+var config = util.getConfig();
+
+var exchanges = require(dirs.gekko + 'exchanges');
+var exchange = _.find(exchanges, function(e) {
+  return e.slug === config.watch.exchange.toLowerCase();
+});
+
+if(!exchange)
+  util.die(`Unsupported exchange: ${config.watch.exchange.toLowerCase()}`)
+
+var exchangeChecker = require(util.dirs().core + 'exchangeChecker');
+
+var error = exchangeChecker.cantMonitor(config.watch);
+if(error)
+  util.die(error, true);
+
+module.exports = require(dirs.budfox + 'budfox');

core/util.js

@@ -4,17 +4,20 @@ var path = require('path');
 var fs = require('fs');
 var semver = require('semver');
 
+var program = require('commander');
+
 var _config = false;
 var _package = false;
 var _nodeVersion = false;
 
+var _args = false;
+
 // helper functions
 var util = {
   getConfig: function() {
     if(_config)
       return _config;
-
-    var configFile = path.resolve(util.getArgument('config') || util.dirs().gekko + 'config');
+    var configFile = path.resolve(program.config || util.dirs().gekko + 'config');
 
     if(!fs.existsSync(configFile))
       util.die('Cannot find a config file.');
@@ -49,20 +52,6 @@ var util = {
     var required = util.getRequiredNodeVersion();
     return semver.satisfies(process.version, required);
   },
-  getArgument: function(argument) {
-    var ret;
-    _.each(process.argv, function(arg) {
-      // check if it's a configurable
-      var pos = arg.indexOf(argument + '=');
-      if(pos !== -1)
-        ret = arg.substr(argument.length + 1);
-      // check if it's a toggle
-      pos = arg.indexOf('-' + argument);
-      if(pos !== -1 && !ret)
-        ret = true;
-    });
-    return ret;
-  },
   // check if two moments are corresponding
   // to the same time
   equals: function(a, b) {
@@ -100,8 +89,8 @@ var util = {
     }
   },
   logVersion: function() {
-    console.log('Gekko version:', 'v' + util.getVersion());
-    console.log('Nodejs version:', process.version);
+    return  `Gekko version: v${util.getVersion()}`
+    + `\nNodejs version: ${process.version}`;
   },
   die: function(m, soft) {
     if(m) {
@@ -112,7 +101,7 @@ var util = {
         console.log('\nError:\n');
         console.log(m, '\n\n');
         console.log('\nMeta debug info:\n');
-        util.logVersion();
+        console.log(util.logVersion());
         console.log('');
       }
     }
@@ -140,10 +129,19 @@ var util = {
   },
   // TODO:
   gekkoMode: function() {
-    return 'realtime';
-  },
+    if(program.backtest)
+      return 'backtest'
+    else
+      return 'realtime';
+  }
 }
 
+program
+  .version(util.logVersion())
+  .option('-c, --config <file>', 'Config file')
+  .option('-b, --backtest', 'backtest')
+  .parse(process.argv);
+
 var config = util.getConfig();
 
 module.exports = util;

docs/Advanced_features.md

@@ -65,15 +65,15 @@ You can also use this feature to do a realtime study on what different EMA setti
 
 To specify a different config file, you can use the following command line argument:
 
-    node gekko config=config/user/alternative-config
+    node gekko --config config/user/alternative-config
 
 or a relative path:
 
-    node gekko config=../../alternative-config
+    node gekko --config ../../alternative-config
 
 or a static path:
 
-    node gekko config=home/gekko/config/user/alternative-config
+    node gekko --config home/gekko/config/user/alternative-config
 
 # Helper files
 

docs/Backtesting.md

@@ -1,52 +1,22 @@
-**These are old docs referrring to the old stable master branch on Gekko. The backtester is broken in this branch**
-
 # Backtesting with Gekko
 
-**Note that this functionality should only be used for testing purposes at this moment as it's in early development stage**
-
-After you configured and run the backtester Gekko will output the results like so:
-
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) start time:                  2013-04-24 07:00:00
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) end time:                    2013-05-23 16:00:00
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) timespan:                    29 days
-
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) start price:                 121.6
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) end price:                   125.44
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) Buy and Hold profit:         3.158%
-
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) amount of trades:            15
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) original simulated balance:  245.404 USD
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) current simulated balance:   281.819 USD
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) simulated profit:            36.415 USD (14.839%)
-    2013-06-30 13:25:30 (INFO): (PROFIT REPORT) simulated yearly profit:     447.030 USD (182.161%)
-
-## Preparing Gekko
-
-You can configure Gekko to test the current EMA strategy on historical data. To do this you need candle data in CSV format. On [this webpage](https://bitcointalk.org/index.php?topic=239815.0) you can downloaded precalculated candles from Mt. Gox or you can calculate your own using the script provided in the link. Alternatively you can supply your own candles, the only requirement is that the csv file has candles ordered like this: `timestamp,open,high,low,close`.
-
-## Configuring Gekko
+Gekko is able to backtest strategies against historical data.
 
-Once you have the csv file with candles you can configure Gekko for backtesting: in [config.js](https://github.com/askmike/gekko/blob/master/config.js) in the advanced zone you need to the backtesting part like so:
+## Setup
 
-    config.backtest = {
-      candleFile: 'candles.csv', // the candles file
-      from: 0, // optional start timestamp 
-      to: 0 // optional end timestamp
-    }
+For backtesting you should enable and configure the following plugins:
 
-Once configured Gekko will run the backtest instead of watching the live market. It wil use the following configuration items:
+ - trading advisor (to run your strategy)
+ - profit simulator (to calculate how succesfull the strategy would have been)
 
-* Everything under `backtest`.
-* Everything under `profitCalculator`.
-* Everything under `EMA` with the exception of interval, as this will be determined by the candles file.
+## Historical data
 
-## Running the backtester
+Gekko requires historical data to backtest strategies against. The easiest way to get this is to run Gekko on real markets with the plugin sqliteWriter enabled (this will cause Gekko to store realtime data on disk).
 
-Instead of running the paper / live trading Gekko using `node gekko`, you can start the backtester by running:
+## Configure
 
-    node gekko-backtest
+In your config set the `backtest.daterange` properties.
 
-## Notes
+## Run
 
-* Use the backtesting feature only for testing until the code is stable.
-* When there are missing candles Gekko will act as if the whole duration of the missing candle never happened.
+    node gekko --backtest