Crypto provider (coinbase#65)

* 0.1.19

* Add RobustRateCalculator

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gdax-trading-toolkit",
-  "version": "0.1.18",
+  "version": "0.1.19",
   "description": "A trading toolkit for building advanced trading bots on the GDAX platform",
   "main": "build/src/index.js",
   "types": "build/src/index.d.ts",
@@ -59,6 +59,7 @@
     "pushbullet": "2.0.0",
     "querystring": "0.2.0",
     "simple-mock": "0.8.0",
+    "simple-statistics": "5.0.1",
     "superagent": "3.0.0",
     "winston": "2.3.0",
     "ws": "1.1.1"

src/FXService/FXProvider.ts

@@ -12,8 +12,8 @@
  * License for the specific language governing permissions and limitations under the License.                              *
  ***************************************************************************************************************************/
 
-import { ONE, ZERO, Big } from '../lib/types';
-import { Logger, ConsoleLoggerFactory } from '../utils/Logger';
+import { Big, ONE, ZERO } from '../lib/types';
+import { Logger } from '../utils/Logger';
 import { BigNumber as BigJS } from 'bignumber.js';
 
 export interface CurrencyPair {
@@ -43,6 +43,7 @@ function makeFXObject(pair: CurrencyPair, value: string | number): FXObject {
 
 export class EFXRateUnavailable extends Error {
     readonly provider: string;
+
     constructor(msg: string, provider: string) {
         super(msg);
         this.provider = provider;
@@ -58,11 +59,18 @@ export abstract class FXProvider {
     private _pending: { [pair: string]: Promise<FXObject> } = {};
 
     constructor(config: FXProviderConfig) {
-        this.logger = config.logger || ConsoleLoggerFactory();
+        this.logger = config.logger;
     }
 
     abstract get name(): string;
 
+    log(level: string, message: string, meta?: any) {
+        if (!this.logger) {
+            return;
+        }
+        this.logger.log(level, message, meta);
+    }
+
     fetchCurrentRate(pair: CurrencyPair): Promise<FXObject> {
         // Special case immediately return 1.0
         if (pair.from === pair.to) {
@@ -109,7 +117,7 @@ export abstract class FXProvider {
         if (pending) {
             return pending;
         }
-        this.logger.log('debug', `Downloading current ${pair.from}-${pair.to} exchange rate from ${this.name}`);
+        this.log('debug', `Downloading current ${pair.from}-${pair.to} exchange rate from ${this.name}`);
         pending = this.downloadCurrentRate(pair);
         this._pending[index] = pending;
         return pending.then((result: FXObject) => {
@@ -124,5 +132,6 @@ export abstract class FXProvider {
      * @param pair
      */
     protected abstract downloadCurrentRate(pair: CurrencyPair): Promise<FXObject>;
+
     protected abstract supportsPair(pair: CurrencyPair): Promise<boolean>;
 }

src/FXService/FXService.ts

@@ -55,12 +55,12 @@ export class FXService extends EventEmitter {
 
     constructor(config: FXServiceConfig) {
         super();
+        this._rates = {};
         this.setLogger(config.logger || ConsoleLoggerFactory())
             .setCalculator(config.calculator)
             .setRefreshInterval(config.refreshInterval || 1000 * 60 * 5) // 5 minutes
             .setActivePairs(config.activePairs || []);
         this.errorState = false;
-        this._rates = {};
     }
 
     /**
@@ -194,7 +194,7 @@ export class FXService extends EventEmitter {
     /**
      * Returns the last set of exchange rate data that was returned by the RateCalculator
      */
-    get rates() {
+    get rates(): FXRates {
         return this._rates;
     }
 

src/FXService/calculators/RobustCalculator.ts

@@ -0,0 +1,177 @@
+/**********************************************************************************************************************
+ * @license                                                                                                           *
+ * Copyright 2017 Coinbase, Inc.                                                                                      *
+ *                                                                                                                    *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance     *
+ * with the License. You may obtain a copy of the License at                                                          *
+ *                                                                                                                    *
+ * http://www.apache.org/licenses/LICENSE-2.0                                                                         *
+ *                                                                                                                    *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on*
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the                 *
+ * License for the specific language governing permissions and limitations under the License.                         *
+ **********************************************************************************************************************/
+import { FXRateCalculator } from '../FXRateCalculator';
+import { CurrencyPair, FXObject, FXProvider, pairAsString } from '../FXProvider';
+import { Logger } from '../../utils/Logger';
+import { eachParallelAndFinish } from '../../utils/promises';
+import * as ss from 'simple-statistics';
+import { Big } from '../../lib/types';
+
+export interface RobustCalculatorConfig {
+    // An array of exchange rate services to use for source data
+    sources: FXProvider[];
+    // A value as a fraction (0 to 1), ∆ for which abs(P_i/P_M - 1) < ∆, where P_i is the current rate and P_M is the median rate,
+    // i.e. fraction of price deviation from the mean allowed
+    priceThreshold: number;
+    // A value as a fraction (0 to 1), ∆ for which abs(r_i - r_M)/P_M < ∆, where r_i is the r_i is the price change since the last
+    // tick and P_M is the median change in rate
+    deltaThreshold: number;
+    // The minimum number of reliable sources in a given request to consider the overall request valid
+    minNumberOfReliableSources: number;
+    logger?: Logger;
+}
+
+export const NO_CURRENT_PRICE_ERROR = 1;
+export const PRICE_DEVIATION_ERROR = 2;
+export const PRICE_CHANGE_DEVIATION_ERROR = 3;
+
+export interface RobustCalculatorReport {
+    data: { [pair: string]: QueryStatus };
+    sources: string[];
+}
+
+export interface QueryStatus {
+    time: Date;
+    prices: number[];
+    deltas: number[];
+    valid: boolean[];
+    errors: Error[];
+    rejectReason: number[];
+    lastPrice: number[];
+}
+
+/**
+ * Calculates exchange rates in a robust manner. The calculator queries exchange rates from multiple sources
+ * and then returns the average price for all the reliable sources.
+ *
+ * A source is deemed reliable if it is less than a given distance from the _median_ exchange rate
+ * AND
+ * the change is price is less than a given distance from the _median_ change in price.
+ *
+ * This should protect from erroneous rates being delivered in these cases:
+ *  - one or more source is down, or providing faulty data for an indefinite period of time
+ *  - A flash crash occurs on a source, providing poor data for a short amount of time.
+ *
+ *  To be reliable, one should supply at least 5 rate sources for this method.
+ */
+export class RobustCalculator extends FXRateCalculator {
+    deltaThreshold: number;
+    priceThreshold: number;
+    logger: Logger;
+    sources: FXProvider[];
+    minNumberOfReliableSources: number;
+    private report: RobustCalculatorReport;
+
+    constructor(config: RobustCalculatorConfig) {
+        super();
+        this.sources = config.sources;
+        this.deltaThreshold = config.deltaThreshold || 0.01;
+        this.priceThreshold = config.priceThreshold || 0.05;
+        this.minNumberOfReliableSources = config.minNumberOfReliableSources || 1;
+        this.logger = config.logger;
+        this.clearReport();
+        this.report.sources = this.sources.map((source: FXProvider) => source.name);
+    }
+
+    get lastReport(): RobustCalculatorReport {
+        return this.report;
+    }
+
+    log(level: string, message: string, meta?: any) {
+        if (!this.logger) {
+            return;
+        }
+        this.logger.log(level, message, meta);
+    }
+
+    calculateRatesFor(pairs: CurrencyPair[]): Promise<FXObject[]> {
+        return eachParallelAndFinish<CurrencyPair, FXObject>(pairs, (pair: CurrencyPair) => {
+            return this.determineRateForPair(pair);
+        }) as Promise<FXObject[]>;
+    }
+
+    private determineRateForPair(pair: CurrencyPair): Promise<FXObject> {
+        return eachParallelAndFinish(this.sources, (source: FXProvider) => {
+            return source.fetchCurrentRate(pair);
+        }).then((rates: (FXObject | Error)[]) => {
+            const rate: number = this.calculateRobustRate(pair, rates);
+            const result: FXObject = {
+                time: new Date(),
+                from: pair.from,
+                to: pair.to,
+                rate: rate && Big(rate),
+                change: null
+            };
+            return Promise.resolve(result);
+        });
+    }
+
+    private calculateRobustRate(pair: CurrencyPair, rates: (FXObject | Error)[]): number {
+        const delta: number[] = [];
+        const prices: number[] = [];
+        const _pair: string = pairAsString(pair);
+        const update: QueryStatus = { deltas: [], prices: [], rejectReason: [], errors: [], time: new Date(), valid: [], lastPrice: null };
+        update.lastPrice = this.report.data[_pair] ? this.report.data[_pair].prices : [];
+        rates.forEach((rate: FXObject | Error, i: number) => {
+            if (rate instanceof Error || !rate.rate.isFinite()) {
+                update.errors[i] = rate instanceof Error ? rate : null;
+                update.rejectReason[i] = NO_CURRENT_PRICE_ERROR;
+                this.report.data[_pair] = update;
+                return;
+            }
+            update.valid[i] = true;
+            update.errors[i] = null;
+            update.rejectReason[i] = 0;
+            update.prices[i] = rate.rate.toNumber();
+            prices.push(update.prices[i]);
+            const lastPrice: number = update.lastPrice[i];
+            update.deltas[i] = lastPrice ? update.prices[i] - lastPrice : 0;
+            delta.push(update.deltas[i]);
+        });
+        this.report.data[_pair] = update;
+        const numValid = prices.length;
+        if (numValid < this.minNumberOfReliableSources) {
+            return null;
+        }
+        const medianPrice = ss.median(prices);
+        const medianDelta = ss.median(delta);
+        let numReliablePrices = 0;
+        let sum = 0;
+        const threshold = this.priceThreshold * medianPrice;
+        const deltaThreshold = this.deltaThreshold * medianPrice;
+        for (let i = 0; i < numValid; i++) {
+            const priceDeviation = Math.abs(prices[i] - medianPrice);
+            const gradDeviation = Math.abs(delta[i] - medianDelta);
+            if (priceDeviation >= threshold) {
+                update.rejectReason[i] = PRICE_DEVIATION_ERROR;
+            }
+            if (gradDeviation >= deltaThreshold) {
+                update.rejectReason[i] = PRICE_CHANGE_DEVIATION_ERROR;
+            }
+            if (priceDeviation < threshold && gradDeviation < deltaThreshold) {
+                numReliablePrices++;
+                sum += prices[i];
+            }
+        }
+        if (numReliablePrices < this.minNumberOfReliableSources) {
+            return null;
+        }
+        return sum / numReliablePrices;
+    }
+
+    private clearReport() {
+        const sources = (this.report && this.report.sources) || [];
+        this.report = { sources: sources, data: {} };
+    }
+}

src/FXService/providers/CryptoProvider.ts

@@ -0,0 +1,64 @@
+/**********************************************************************************************************************
+ * @license                                                                                                           *
+ * Copyright 2017 Coinbase, Inc.                                                                                      *
+ *                                                                                                                    *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance     *
+ * with the License. You may obtain a copy of the License at                                                          *
+ *                                                                                                                    *
+ * http://www.apache.org/licenses/LICENSE-2.0                                                                         *
+ *                                                                                                                    *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on*
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the                 *
+ * License for the specific language governing permissions and limitations under the License.                         *
+ **********************************************************************************************************************/
+import { CurrencyPair, FXObject, FXProvider, FXProviderConfig, pairAsString } from '../FXProvider';
+import { Product, PublicExchangeAPI } from '../../exchanges/PublicExchangeAPI';
+import { BigJS } from '../../lib/types';
+
+export interface CryptoProviderConfig extends FXProviderConfig {
+    exchange: PublicExchangeAPI;
+}
+
+export class CryptoProvider extends FXProvider {
+    private exchange: PublicExchangeAPI;
+    private products: string[];
+
+    constructor(config: CryptoProviderConfig) {
+        super(config);
+        this.exchange = config.exchange;
+    }
+
+    get name(): string {
+        return `CryptoProvider (${this.exchange.owner})`;
+    }
+
+    protected downloadCurrentRate(pair: CurrencyPair): Promise<FXObject> {
+        const product: string = pairAsString(pair);
+        return this.exchange.loadMidMarketPrice(product).then((price: BigJS) => {
+            const result: FXObject = {
+                time: new Date(),
+                from: pair.from,
+                to: pair.to,
+                rate: price
+            };
+            return result;
+        });
+    }
+
+    protected supportsPair(pair: CurrencyPair): Promise<boolean> {
+        const getProducts = this.products ?
+            Promise.resolve(this.products) :
+            this.exchange.loadProducts().then((products: Product[]) => {
+                this.products = products.map((p) => p.id);
+                return this.products;
+            }).catch((err: Error) => {
+                this.log('warn', 'CryptoProvider could not load a list of products', err);
+                return [];
+            });
+        return getProducts.then((products: string[]) => {
+            const product = pairAsString(pair);
+            return products.includes(product);
+        });
+    }
+
+}

src/samples/RobustRateDemo.ts

@@ -0,0 +1,59 @@
+/**********************************************************************************************************************
+ * @license                                                                                                           *
+ * Copyright 2017 Coinbase, Inc.                                                                                      *
+ *                                                                                                                    *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance     *
+ * with the License. You may obtain a copy of the License at                                                          *
+ *                                                                                                                    *
+ * http://www.apache.org/licenses/LICENSE-2.0                                                                         *
+ *                                                                                                                    *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on*
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the                 *
+ * License for the specific language governing permissions and limitations under the License.                         *
+ **********************************************************************************************************************/
+import { CurrencyPair, FXProvider } from '../FXService/FXProvider';
+import { ConsoleLoggerFactory, Logger } from '../utils/Logger';
+import CCXTWrapper from '../exchanges/ccxt';
+import { GDAXExchangeAPI } from '../exchanges/gdax/GDAXExchangeAPI';
+import { RobustCalculator } from '../FXService/calculators/RobustCalculator';
+import { CryptoProvider } from '../FXService/providers/CryptoProvider';
+import { FXRates, FXService } from '../FXService/FXService';
+import { BitfinexExchangeAPI } from '../exchanges/bitfinex/BitfinexExchangeAPI';
+import { ExchangeAuthConfig } from '../exchanges/AuthConfig';
+
+const logger: Logger = ConsoleLoggerFactory();
+const noAuth: ExchangeAuthConfig = { key: null, secret: null };
+const providers: FXProvider[] = [
+    new CryptoProvider({ exchange: CCXTWrapper.createExchange('gemini', noAuth, logger), logger: logger }),
+    new CryptoProvider({ exchange: CCXTWrapper.createExchange('poloniex', noAuth, logger), logger: logger }),
+    new CryptoProvider({ exchange: CCXTWrapper.createExchange('bitmex', noAuth, logger), logger: logger }),
+    new CryptoProvider({ exchange: CCXTWrapper.createExchange('kraken', noAuth, logger), logger: logger }),
+    new CryptoProvider({ exchange: new BitfinexExchangeAPI({logger: logger, auth: noAuth }), logger: logger }),
+    new CryptoProvider({ exchange: new GDAXExchangeAPI({ auth: null, logger: logger }), logger: logger })
+];
+
+const robustCalculator = new RobustCalculator({
+    sources: providers,
+    logger: logger,
+    minNumberOfReliableSources: 2,
+    deltaThreshold: 0.03,
+    priceThreshold: 0.015
+});
+
+const pairs: CurrencyPair[] = [
+    { from: 'BTC', to: 'USD' },
+    { from: 'ETH', to: 'BTC' }
+];
+
+const fxService: FXService = new FXService({
+    logger: logger,
+    refreshInterval: 60 * 1000,
+    calculator: robustCalculator,
+    activePairs: pairs
+});
+
+fxService.on('FXRateUpdate', (rates: FXRates) => {
+    if (rates['BTC-USD'].rate) { logger.log('info', `BTC price: ${rates['BTC-USD'].rate.toFixed(5)}`); }
+    if (rates['ETH-BTC'].rate) { logger.log('info', `ETH price: ${rates['ETH-BTC'].rate.toFixed(5)}`); }
+    logger.log('debug', 'RobustRate report', robustCalculator.lastReport);
+});