Binance: implement get historic trades (thrasher-corp#588)

* Binance: implement get historic trades

* get binance trade data based on aggregate trade list
* fix small issue in rpc server: gctcli stops retrieving when there's
a gap in data

* update binance trade history availability in readme

* limit check batched aggregate requests

* add test for batched aggregated trades
* fix batch fromId query parameter
* update documentation

* send a serialised currency pair to GetAggregatedTrades

the rationale is that the API is kept generic so that callers can shoot
themselves in the foot if they want to

* allow requesting arbitrary limit of trades

* handle some error cases for batching GetAggregateTrades

* fix batch without end time

* don't return from batch too early if end time is not set
* additional check for supported limits

* don't use CheckLimits for GetAggregatedTrades

* the exchange doesn't use predefined valid limits for this request

CONTRIBUTORS

@@ -8,9 +8,9 @@ xtda | https://github.com/xtda
 ermalguni | https://github.com/ermalguni
 vadimzhukck | https://github.com/vadimzhukck
 MadCozBadd | https://github.com/MadCozBadd
+Rots | https://github.com/Rots
 140am | https://github.com/140am
 marcofranssen | https://github.com/marcofranssen
-Rots | https://github.com/Rots
 vazha | https://github.com/vazha
 dackroyd | https://github.com/dackroyd
 cranktakular | https://github.com/cranktakular

README.md

@@ -142,17 +142,17 @@ Binaries will be published once the codebase reaches a stable condition.
 
 |User|Contribution Amount|
 |--|--|
-| [thrasher-](https://github.com/thrasher-) | 643 |
-| [shazbert](https://github.com/shazbert) | 197 |
-| [gloriousCode](https://github.com/gloriousCode) | 171 |
+| [thrasher-](https://github.com/thrasher-) | 645 |
+| [shazbert](https://github.com/shazbert) | 199 |
+| [gloriousCode](https://github.com/gloriousCode) | 173 |
 | [dependabot-preview[bot]](https://github.com/apps/dependabot-preview) | 70 |
 | [xtda](https://github.com/xtda) | 47 |
 | [ermalguni](https://github.com/ermalguni) | 14 |
 | [vadimzhukck](https://github.com/vadimzhukck) | 10 |
 | [MadCozBadd](https://github.com/MadCozBadd) | 9 |
+| [Rots](https://github.com/Rots) | 9 |
 | [140am](https://github.com/140am) | 8 |
 | [marcofranssen](https://github.com/marcofranssen) | 8 |
-| [Rots](https://github.com/Rots) | 7 |
 | [vazha](https://github.com/vazha) | 7 |
 | [dackroyd](https://github.com/dackroyd) | 5 |
 | [cranktakular](https://github.com/cranktakular) | 5 |

cmd/documentation/exchanges_templates/exchanges_trade_readme.tmpl

@@ -42,7 +42,7 @@ _b in this context is an `IBotExchange` implemented struct_
 | Exchange | Recent Trades via REST | Live trade updates via Websocket | Trade history via REST |
 |----------|------|-----------|-----|
 | Alphapoint | No  | No        | No  |
-| Binance| Yes  | Yes        | No  |
+| Binance| Yes  | Yes        | Yes  |
 | Bitfinex | Yes  | Yes        | Yes  |
 | Bitflyer | Yes  | No      | No  |
 | Bithumb | Yes  | NA       | No  |

engine/rpcserver.go

@@ -2793,13 +2793,16 @@ func (s *RPCServer) GetHistoricTrades(r *gctrpc.GetSavedTradesRequest, stream gc
 		Asset:        r.AssetType,
 		Pair:         r.Pair,
 	}
-	iterateStartTime := UTCStartTime
-	iterateEndTime := iterateStartTime.Add(time.Hour)
-	for iterateStartTime.Before(UTCEndTime) {
+
+	for iterateStartTime := UTCStartTime; iterateStartTime.Before(UTCEndTime); iterateStartTime = iterateStartTime.Add(time.Hour) {
+		iterateEndTime := iterateStartTime.Add(time.Hour)
 		trades, err = exch.GetHistoricTrades(cp, asset.Item(r.AssetType), iterateStartTime, iterateEndTime)
 		if err != nil {
 			return err
 		}
+		if len(trades) == 0 {
+			continue
+		}
 		grpcTrades := &gctrpc.SavedTradesResponse{
 			ExchangeName: r.Exchange,
 			Asset:        r.AssetType,
@@ -2820,8 +2823,6 @@ func (s *RPCServer) GetHistoricTrades(r *gctrpc.GetSavedTradesRequest, stream gc
 		}
 
 		stream.Send(grpcTrades)
-		iterateStartTime = iterateStartTime.Add(time.Hour)
-		iterateEndTime = iterateEndTime.Add(time.Hour)
 	}
 	stream.Send(resp)
 

exchanges/binance/binance.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -161,27 +162,122 @@ func (b *Binance) GetHistoricalTrades(symbol string, limit int, fromID int64) ([
 	return nil, common.ErrFunctionNotSupported
 }
 
-// GetAggregatedTrades returns aggregated trade activity
-//
-// symbol: string of currency pair
-// limit: Optional. Default 500; max 1000.
-func (b *Binance) GetAggregatedTrades(symbol string, limit int) ([]AggregatedTrade, error) {
-	var resp []AggregatedTrade
-
-	if err := b.CheckLimit(limit); err != nil {
-		return resp, err
+// GetAggregatedTrades returns aggregated trade activity.
+// If more than one hour of data is requested or asked limit is not supported by exchange
+// then the trades are collected with multiple backend requests.
+// https://binance-docs.github.io/apidocs/spot/en/#compressed-aggregate-trades-list
+func (b *Binance) GetAggregatedTrades(arg *AggregatedTradeRequestParams) ([]AggregatedTrade, error) {
+	params := url.Values{}
+	params.Set("symbol", arg.Symbol)
+	// if the user request is directly not supported by the exchange, we might be able to fulfill it
+	// by merging results from multiple API requests
+	needBatch := false
+	if arg.Limit > 0 {
+		if arg.Limit > 1000 {
+			// remote call doesn't support higher limits
+			needBatch = true
+		} else {
+			params.Set("limit", strconv.Itoa(arg.Limit))
+		}
+	}
+	if arg.FromID != 0 {
+		params.Set("fromId", strconv.FormatInt(arg.FromID, 10))
+	}
+	if !arg.StartTime.IsZero() {
+		params.Set("startTime", strconv.FormatInt(convert.UnixMillis(arg.StartTime), 10))
+	}
+	if !arg.EndTime.IsZero() {
+		params.Set("endTime", strconv.FormatInt(convert.UnixMillis(arg.EndTime), 10))
 	}
 
-	params := url.Values{}
-	params.Set("symbol", strings.ToUpper(symbol))
-	if limit > 0 {
-		params.Set("limit", strconv.Itoa(limit))
+	// startTime and endTime are set and time between startTime and endTime is more than 1 hour
+	needBatch = needBatch || (!arg.StartTime.IsZero() && !arg.EndTime.IsZero() && arg.EndTime.Sub(arg.StartTime) > time.Hour)
+	// Fall back to batch requests, if possible and necessary
+	if needBatch {
+		// fromId xor start time must be set
+		canBatch := arg.FromID == 0 != arg.StartTime.IsZero()
+		if canBatch {
+			// Split the request into multiple
+			return b.batchAggregateTrades(arg, params)
+		}
+
+		// Can't handle this request locally or remotely
+		// We would receive {"code":-1128,"msg":"Combination of optional parameters invalid."}
+		return nil, errors.New("please set StartTime or FromId, but not both")
 	}
 
+	var resp []AggregatedTrade
 	path := b.API.Endpoints.URL + aggregatedTrades + "?" + params.Encode()
 	return resp, b.SendHTTPRequest(path, limitDefault, &resp)
 }
 
+// batchAggregateTrades fetches trades in multiple requests
+// first phase, hourly requests until the first trade (or end time) is reached
+// second phase, limit requests from previous trade until end time (or limit) is reached
+func (b *Binance) batchAggregateTrades(arg *AggregatedTradeRequestParams, params url.Values) ([]AggregatedTrade, error) {
+	var resp []AggregatedTrade
+	// prepare first request with only first hour and max limit
+	if arg.Limit == 0 || arg.Limit > 1000 {
+		// Extend from the default of 500
+		params.Set("limit", "1000")
+	}
+
+	var fromID int64
+	if arg.FromID > 0 {
+		fromID = arg.FromID
+	} else {
+		for start := arg.StartTime; len(resp) == 0; start = start.Add(time.Hour) {
+			if !arg.EndTime.IsZero() && !start.Before(arg.EndTime) {
+				// All requests returned empty
+				return nil, nil
+			}
+			params.Set("startTime", strconv.FormatInt(convert.UnixMillis(start), 10))
+			params.Set("endTime", strconv.FormatInt(convert.UnixMillis(start.Add(time.Hour)), 10))
+			path := b.API.Endpoints.URL + aggregatedTrades + "?" + params.Encode()
+			err := b.SendHTTPRequest(path, limitDefault, &resp)
+			if err != nil {
+				log.Warn(log.ExchangeSys, err.Error())
+				return resp, err
+			}
+		}
+		fromID = resp[len(resp)-1].ATradeID
+	}
+
+	// other requests follow from the last aggregate trade id and have no time window
+	params.Del("startTime")
+	params.Del("endTime")
+	// while we haven't reached the limit
+	for ; arg.Limit == 0 || len(resp) < arg.Limit; fromID = resp[len(resp)-1].ATradeID {
+		// Keep requesting new data after last retrieved trade
+		params.Set("fromId", strconv.FormatInt(fromID, 10))
+		path := b.API.Endpoints.URL + aggregatedTrades + "?" + params.Encode()
+		var additionalTrades []AggregatedTrade
+		err := b.SendHTTPRequest(path, limitDefault, &additionalTrades)
+		if err != nil {
+			return resp, err
+		}
+		lastIndex := len(additionalTrades)
+		if !arg.EndTime.IsZero() {
+			// get index for truncating to end time
+			lastIndex = sort.Search(len(additionalTrades), func(i int) bool {
+				return convert.UnixMillis(arg.EndTime) < additionalTrades[i].TimeStamp
+			})
+		}
+		// don't include the first as the request was inclusive from last ATradeID
+		resp = append(resp, additionalTrades[1:lastIndex]...)
+		// If only the starting trade is returned or if we received trades after end time
+		if len(additionalTrades) == 1 || lastIndex < len(additionalTrades) {
+			// We found the end
+			break
+		}
+	}
+	// Truncate if necessary
+	if arg.Limit > 0 && len(resp) > arg.Limit {
+		resp = resp[:arg.Limit]
+	}
+	return resp, nil
+}
+
 // GetSpotKline returns kline data
 //
 // KlinesRequestParams supports 5 parameters