Backtester: Fix dividing by zero when missing data (thrasher-corp#841)

* Fixes zero data issues on backtester & fixes drawdown calcs for zeros

* Fixes valuation of holdings + when simultaneous processing disabled

* Fixes an issue for when there is a lot of missing data

backtester/common/common_types.go

@@ -65,10 +65,10 @@ type EventHandler interface {
 // DataEventHandler interface used for loading and interacting with Data
 type DataEventHandler interface {
 	EventHandler
-	ClosePrice() decimal.Decimal
-	HighPrice() decimal.Decimal
-	LowPrice() decimal.Decimal
-	OpenPrice() decimal.Decimal
+	GetClosePrice() decimal.Decimal
+	GetHighPrice() decimal.Decimal
+	GetLowPrice() decimal.Decimal
+	GetOpenPrice() decimal.Decimal
 }
 
 // Directioner dictates the side of an order

backtester/data/data_test.go

@@ -74,10 +74,10 @@ func TestStream(t *testing.T) {
 	f.GetAssetType()
 	f.GetReason()
 	f.AppendReason("fake")
-	f.ClosePrice()
-	f.HighPrice()
-	f.LowPrice()
-	f.OpenPrice()
+	f.GetClosePrice()
+	f.GetHighPrice()
+	f.GetLowPrice()
+	f.GetOpenPrice()
 
 	d.AppendStream(fakeDataHandler{time: 1})
 	d.AppendStream(fakeDataHandler{time: 4})
@@ -208,18 +208,18 @@ func (t fakeDataHandler) GetReason() string {
 func (t fakeDataHandler) AppendReason(string) {
 }
 
-func (t fakeDataHandler) ClosePrice() decimal.Decimal {
+func (t fakeDataHandler) GetClosePrice() decimal.Decimal {
 	return decimal.Zero
 }
 
-func (t fakeDataHandler) HighPrice() decimal.Decimal {
+func (t fakeDataHandler) GetHighPrice() decimal.Decimal {
 	return decimal.Zero
 }
 
-func (t fakeDataHandler) LowPrice() decimal.Decimal {
+func (t fakeDataHandler) GetLowPrice() decimal.Decimal {
 	return decimal.Zero
 }
 
-func (t fakeDataHandler) OpenPrice() decimal.Decimal {
+func (t fakeDataHandler) GetOpenPrice() decimal.Decimal {
 	return decimal.Zero
 }

backtester/eventhandlers/exchange/exchange.go

@@ -40,7 +40,7 @@ func (e *Exchange) ExecuteOrder(o order.Event, data data.Handler, orderManager *
 		},
 		Direction:  o.GetDirection(),
 		Amount:     o.GetAmount(),
-		ClosePrice: data.Latest().ClosePrice(),
+		ClosePrice: data.Latest().GetClosePrice(),
 	}
 	eventFunds := o.GetAllocatedFunds()
 	cs, err := e.GetCurrencySettings(o.GetExchange(), o.GetAssetType(), o.Pair())

backtester/eventhandlers/portfolio/holdings/holdings.go

@@ -9,13 +9,14 @@ import (
 )
 
 // Create makes a Holding struct to track total values of strategy holdings over the course of a backtesting run
-func Create(ev common.EventHandler, funding funding.IPairReader) (Holding, error) {
+func Create(ev ClosePriceReader, funding funding.IPairReader) (Holding, error) {
 	if ev == nil {
 		return Holding{}, common.ErrNilEvent
 	}
 	if funding.QuoteInitialFunds().LessThan(decimal.Zero) {
 		return Holding{}, ErrInitialFundsZero
 	}
+
 	return Holding{
 		Offset:            ev.GetOffset(),
 		Pair:              ev.Pair(),
@@ -26,7 +27,7 @@ func Create(ev common.EventHandler, funding funding.IPairReader) (Holding, error
 		QuoteSize:         funding.QuoteInitialFunds(),
 		BaseInitialFunds:  funding.BaseInitialFunds(),
 		BaseSize:          funding.BaseInitialFunds(),
-		TotalInitialValue: funding.BaseInitialFunds().Mul(funding.QuoteInitialFunds()).Add(funding.QuoteInitialFunds()),
+		TotalInitialValue: funding.QuoteInitialFunds().Add(funding.BaseInitialFunds().Mul(ev.GetClosePrice())),
 	}, nil
 }
 
@@ -40,7 +41,7 @@ func (h *Holding) Update(e fill.Event, f funding.IPairReader) {
 // UpdateValue calculates the holding's value for a data event's time and price
 func (h *Holding) UpdateValue(d common.DataEventHandler) {
 	h.Timestamp = d.GetTime()
-	latest := d.ClosePrice()
+	latest := d.GetClosePrice()
 	h.Offset = d.GetOffset()
 	h.updateValue(latest)
 }

backtester/eventhandlers/portfolio/holdings/holdings_types.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/shopspring/decimal"
+	"github.com/thrasher-corp/gocryptotrader/backtester/common"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
 )
@@ -44,3 +45,10 @@ type Holding struct {
 	TotalValueLostToSlippage     decimal.Decimal `json:"total-value-lost-to-slippage"`
 	TotalValueLost               decimal.Decimal `json:"total-value-lost"`
 }
+
+// ClosePriceReader is used for holdings calculations
+// without needing to consider event types
+type ClosePriceReader interface {
+	common.EventHandler
+	GetClosePrice() decimal.Decimal
+}

backtester/eventhandlers/statistics/currencystatistics.go

@@ -24,9 +24,9 @@ func (c *CurrencyPairStatistic) CalculateResults(riskFreeRate decimal.Decimal) e
 	first := c.Events[0]
 	sep := fmt.Sprintf("%v %v %v |\t", first.DataEvent.GetExchange(), first.DataEvent.GetAssetType(), first.DataEvent.Pair())
 
-	firstPrice := first.DataEvent.ClosePrice()
+	firstPrice := first.DataEvent.GetClosePrice()
 	last := c.Events[len(c.Events)-1]
-	lastPrice := last.DataEvent.ClosePrice()
+	lastPrice := last.DataEvent.GetClosePrice()
 	for i := range last.Transactions.Orders {
 		if last.Transactions.Orders[i].Side == gctorder.Buy {
 			c.BuyOrders++
@@ -35,7 +35,7 @@ func (c *CurrencyPairStatistic) CalculateResults(riskFreeRate decimal.Decimal) e
 		}
 	}
 	for i := range c.Events {
-		price := c.Events[i].DataEvent.ClosePrice()
+		price := c.Events[i].DataEvent.GetClosePrice()
 		if c.LowestClosePrice.IsZero() || price.LessThan(c.LowestClosePrice) {
 			c.LowestClosePrice = price
 		}
@@ -45,7 +45,9 @@ func (c *CurrencyPairStatistic) CalculateResults(riskFreeRate decimal.Decimal) e
 	}
 
 	oneHundred := decimal.NewFromInt(100)
-	c.MarketMovement = lastPrice.Sub(firstPrice).Div(firstPrice).Mul(oneHundred)
+	if !firstPrice.IsZero() {
+		c.MarketMovement = lastPrice.Sub(firstPrice).Div(firstPrice).Mul(oneHundred)
+	}
 	if first.Holdings.TotalValue.GreaterThan(decimal.Zero) {
 		c.StrategyMovement = last.Holdings.TotalValue.Sub(first.Holdings.TotalValue).Div(first.Holdings.TotalValue).Mul(oneHundred)
 	}
@@ -63,9 +65,16 @@ func (c *CurrencyPairStatistic) CalculateResults(riskFreeRate decimal.Decimal) e
 		if c.Events[i].SignalEvent != nil && c.Events[i].SignalEvent.GetDirection() == common.MissingData {
 			c.ShowMissingDataWarning = true
 		}
-		benchmarkRates[i] = c.Events[i].DataEvent.ClosePrice().Sub(
-			c.Events[i-1].DataEvent.ClosePrice()).Div(
-			c.Events[i-1].DataEvent.ClosePrice())
+		if c.Events[i].DataEvent.GetClosePrice().IsZero() || c.Events[i-1].DataEvent.GetClosePrice().IsZero() {
+			// closing price for the current candle or previous candle is zero, use the previous
+			// benchmark rate to allow some consistency
+			c.ShowMissingDataWarning = true
+			benchmarkRates[i] = benchmarkRates[i-1]
+			continue
+		}
+		benchmarkRates[i] = c.Events[i].DataEvent.GetClosePrice().Sub(
+			c.Events[i-1].DataEvent.GetClosePrice()).Div(
+			c.Events[i-1].DataEvent.GetClosePrice())
 	}
 
 	// remove the first entry as its zero and impacts
@@ -121,8 +130,8 @@ func (c *CurrencyPairStatistic) PrintResults(e string, a asset.Item, p currency.
 	})
 	last := c.Events[len(c.Events)-1]
 	first := c.Events[0]
-	c.StartingClosePrice = first.DataEvent.ClosePrice()
-	c.EndingClosePrice = last.DataEvent.ClosePrice()
+	c.StartingClosePrice = first.DataEvent.GetClosePrice()
+	c.EndingClosePrice = last.DataEvent.GetClosePrice()
 	c.TotalOrders = c.BuyOrders + c.SellOrders
 	last.Holdings.TotalValueLost = last.Holdings.TotalValueLostToSlippage.Add(last.Holdings.TotalValueLostToVolumeSizing)
 	sep := fmt.Sprintf("%v %v %v |\t", e, a, p)
@@ -207,21 +216,21 @@ func CalculateBiggestEventDrawdown(closePrices []common.DataEventHandler) (Swing
 		return Swing{}, fmt.Errorf("%w to calculate drawdowns", errReceivedNoData)
 	}
 	var swings []Swing
-	lowestPrice := closePrices[0].LowPrice()
-	highestPrice := closePrices[0].HighPrice()
+	lowestPrice := closePrices[0].GetLowPrice()
+	highestPrice := closePrices[0].GetHighPrice()
 	lowestTime := closePrices[0].GetTime()
 	highestTime := closePrices[0].GetTime()
 	interval := closePrices[0].GetInterval()
 
 	for i := range closePrices {
-		currHigh := closePrices[i].HighPrice()
-		currLow := closePrices[i].LowPrice()
+		currHigh := closePrices[i].GetHighPrice()
+		currLow := closePrices[i].GetLowPrice()
 		currTime := closePrices[i].GetTime()
 		if lowestPrice.GreaterThan(currLow) && !currLow.IsZero() {
 			lowestPrice = currLow
 			lowestTime = currTime
 		}
-		if highestPrice.LessThan(currHigh) && highestPrice.IsPositive() {
+		if highestPrice.LessThan(currHigh) {
 			if lowestTime.Equal(highestTime) {
 				// create distinction if the greatest drawdown occurs within the same candle
 				lowestTime = lowestTime.Add(interval.Duration() - time.Nanosecond)
@@ -231,26 +240,28 @@ func CalculateBiggestEventDrawdown(closePrices []common.DataEventHandler) (Swing
 				log.Error(log.BackTester, err)
 				continue
 			}
-			swings = append(swings, Swing{
-				Highest: ValueAtTime{
-					Time:  highestTime,
-					Value: highestPrice,
-				},
-				Lowest: ValueAtTime{
-					Time:  lowestTime,
-					Value: lowestPrice,
-				},
-				DrawdownPercent:  lowestPrice.Sub(highestPrice).Div(highestPrice).Mul(decimal.NewFromInt(100)),
-				IntervalDuration: int64(len(intervals.Ranges[0].Intervals)),
-			})
+			if highestPrice.IsPositive() && lowestPrice.IsPositive() {
+				swings = append(swings, Swing{
+					Highest: ValueAtTime{
+						Time:  highestTime,
+						Value: highestPrice,
+					},
+					Lowest: ValueAtTime{
+						Time:  lowestTime,
+						Value: lowestPrice,
+					},
+					DrawdownPercent:  lowestPrice.Sub(highestPrice).Div(highestPrice).Mul(decimal.NewFromInt(100)),
+					IntervalDuration: int64(len(intervals.Ranges[0].Intervals)),
+				})
+			}
 			// reset the drawdown
 			highestPrice = currHigh
 			highestTime = currTime
 			lowestPrice = currLow
 			lowestTime = currTime
 		}
 	}
-	if (len(swings) > 0 && swings[len(swings)-1].Lowest.Value != closePrices[len(closePrices)-1].LowPrice()) || swings == nil {
+	if (len(swings) > 0 && swings[len(swings)-1].Lowest.Value != closePrices[len(closePrices)-1].GetLowPrice()) || swings == nil {
 		// need to close out the final drawdown
 		if lowestTime.Equal(highestTime) {
 			// create distinction if the greatest drawdown occurs within the same candle
@@ -297,8 +308,8 @@ func CalculateBiggestEventDrawdown(closePrices []common.DataEventHandler) (Swing
 
 func (c *CurrencyPairStatistic) calculateHighestCommittedFunds() {
 	for i := range c.Events {
-		if c.Events[i].Holdings.BaseSize.Mul(c.Events[i].DataEvent.ClosePrice()).GreaterThan(c.HighestCommittedFunds.Value) {
-			c.HighestCommittedFunds.Value = c.Events[i].Holdings.BaseSize.Mul(c.Events[i].DataEvent.ClosePrice())
+		if c.Events[i].Holdings.BaseSize.Mul(c.Events[i].DataEvent.GetClosePrice()).GreaterThan(c.HighestCommittedFunds.Value) {
+			c.HighestCommittedFunds.Value = c.Events[i].Holdings.BaseSize.Mul(c.Events[i].DataEvent.GetClosePrice())
 			c.HighestCommittedFunds.Time = c.Events[i].Holdings.Timestamp
 		}
 	}

backtester/eventhandlers/statistics/currencystatistics_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/shopspring/decimal"
+	"github.com/thrasher-corp/gocryptotrader/backtester/common"
 	"github.com/thrasher-corp/gocryptotrader/backtester/eventhandlers/portfolio/compliance"
 	"github.com/thrasher-corp/gocryptotrader/backtester/eventhandlers/portfolio/holdings"
 	"github.com/thrasher-corp/gocryptotrader/backtester/eventtypes/event"
@@ -105,6 +106,7 @@ func TestCalculateResults(t *testing.T) {
 		SignalEvent: &signal.Signal{
 			Base:       even2,
 			ClosePrice: decimal.NewFromInt(1337),
+			Direction:  common.MissingData,
 		},
 	}
 
@@ -116,6 +118,41 @@ func TestCalculateResults(t *testing.T) {
 	if !cs.MarketMovement.Equal(decimal.NewFromFloat(-33.15)) {
 		t.Error("expected -33.15")
 	}
+	ev3 := ev2
+	ev3.DataEvent = &kline.Kline{
+		Base:   even2,
+		Open:   decimal.NewFromInt(1339),
+		Close:  decimal.NewFromInt(1339),
+		Low:    decimal.NewFromInt(1339),
+		High:   decimal.NewFromInt(1339),
+		Volume: decimal.NewFromInt(1339),
+	}
+	cs.Events = append(cs.Events, ev, ev3)
+	cs.Events[0].DataEvent = &kline.Kline{
+		Base:   even2,
+		Open:   decimal.Zero,
+		Close:  decimal.Zero,
+		Low:    decimal.Zero,
+		High:   decimal.Zero,
+		Volume: decimal.Zero,
+	}
+	err = cs.CalculateResults(decimal.NewFromFloat(0.03))
+	if err != nil {
+		t.Error(err)
+	}
+
+	cs.Events[1].DataEvent = &kline.Kline{
+		Base:   even2,
+		Open:   decimal.Zero,
+		Close:  decimal.Zero,
+		Low:    decimal.Zero,
+		High:   decimal.Zero,
+		Volume: decimal.Zero,
+	}
+	err = cs.CalculateResults(decimal.NewFromFloat(0.03))
+	if err != nil {
+		t.Error(err)
+	}
 }
 
 func TestPrintResults(t *testing.T) {

backtester/eventhandlers/statistics/fundingstatistics.go

@@ -176,7 +176,6 @@ func CalculateIndividualFundingStatistics(disableUSDTracking bool, reportItem *f
 			item.HighestClosePrice.Time = closePrices[i].Time
 		}
 	}
-
 	for i := range relatedStats {
 		if relatedStats[i].stat == nil {
 			return nil, fmt.Errorf("%w related stats", common.ErrNilArguments)

backtester/eventhandlers/statistics/statistics.go

@@ -354,7 +354,7 @@ func (s *Statistic) PrintAllEventsChronologically() {
 								currencyStatistic.Events[i].DataEvent.GetExchange(),
 								currencyStatistic.Events[i].DataEvent.GetAssetType(),
 								currencyStatistic.Events[i].DataEvent.Pair(),
-								currencyStatistic.Events[i].DataEvent.ClosePrice().Round(8),
+								currencyStatistic.Events[i].DataEvent.GetClosePrice().Round(8),
 								currencyStatistic.Events[i].DataEvent.GetReason()))
 					default:
 						errs = append(errs, fmt.Errorf("%v %v %v unexpected data received %+v", exch, a, pair, currencyStatistic.Events[i]))

backtester/eventhandlers/strategies/base/base.go

@@ -32,10 +32,10 @@ func (s *Strategy) GetBaseData(d data.Handler) (signal.Signal, error) {
 			Interval:     latest.GetInterval(),
 			Reason:       latest.GetReason(),
 		},
-		ClosePrice: latest.ClosePrice(),
-		HighPrice:  latest.HighPrice(),
-		OpenPrice:  latest.OpenPrice(),
-		LowPrice:   latest.LowPrice(),
+		ClosePrice: latest.GetClosePrice(),
+		HighPrice:  latest.GetHighPrice(),
+		OpenPrice:  latest.GetOpenPrice(),
+		LowPrice:   latest.GetLowPrice(),
 	}, nil
 }
 

backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage.go

@@ -52,7 +52,7 @@ func (s *Strategy) OnSignal(d data.Handler, _ funding.IFundTransferer, _ portfol
 		return &es, nil
 	}
 
-	es.SetPrice(d.Latest().ClosePrice())
+	es.SetPrice(d.Latest().GetClosePrice())
 	es.SetDirection(order.Buy)
 	es.AppendReason("DCA purchases on every iteration")
 	return &es, nil

backtester/eventhandlers/strategies/rsi/rsi.go

@@ -55,7 +55,7 @@ func (s *Strategy) OnSignal(d data.Handler, _ funding.IFundTransferer, _ portfol
 	if err != nil {
 		return nil, err
 	}
-	es.SetPrice(d.Latest().ClosePrice())
+	es.SetPrice(d.Latest().GetClosePrice())
 
 	if offset := d.Offset(); offset <= int(s.rsiPeriod.IntPart()) {
 		es.AppendReason("Not enough data for signal generation")

backtester/eventhandlers/strategies/top2bottom2/top2bottom2.go

@@ -102,7 +102,7 @@ func (s *Strategy) OnSimultaneousSignals(d []data.Handler, f funding.IFundTransf
 		if err != nil {
 			return nil, err
 		}
-		es.SetPrice(d[i].Latest().ClosePrice())
+		es.SetPrice(d[i].Latest().GetClosePrice())
 		offset := d[i].Offset()
 
 		if offset <= int(s.mfiPeriod.IntPart()) {