Feature: Data history manager engine subsystem (thrasher-corp#693)

* Adds lovely initial concept for historical data doer

* Adds ability to save tasks. Adds config. Adds startStop to engine

* Has a database microservice without use of globals! Further infrastructure design. Adds readme

* Commentary to help design

* Adds migrations for database

* readme and adds database models

* Some modelling that doesn't work end of day

* Completes datahistoryjob sql.Begins datahistoryjobresult

* Adds datahistoryjob functions to retreive job results. Adapts subsystem

* Adds process for upserting jobs and job results to the database

* Broken end of day weird sqlboiler crap

* Fixes issue with SQL generation.

* RPC generation and addition of basic upsert command

* Renames types

* Adds rpc functions

* quick commit before context swithc. Exchanges aren't being populated

* Begin the tests!

* complete sql tests. stop failed jobs. CLI command creation

* Defines rpc commands

* Fleshes out RPC implementation

* Expands testing

* Expands testing, removes double remove

* Adds coverage of data history subsystem, expands errors and nil checks

* Minor logic improvement

* streamlines datahistory test setup

* End of day minor linting

* Lint, convert simplify, rpc expansion, type expansion, readme expansion

* Documentation update

* Renames for consistency

* Completes RPC server commands

* Fixes tests

* Speeds up testing by reducing unnecessary actions. Adds maxjobspercycle config

* Comments for everything

* Adds missing result string. checks interval supported. default start end cli

* Fixes ID problem. Improves binance trade fetch. job ranges are processed

* adds dbservice coverage. adds rpcserver coverage

* docs regen, uses dbcon interface, reverts binance, fixes races, toggle manager

* Speed up tests, remove bad global usage, fix uuid check

* Adds verbose. Updates docs. Fixes postgres

* Minor changes to logging and start stop

* Fixes postgres db tests, fixes postgres column typo

* Fixes old string typo,removes constraint,error parsing for nonreaders

* prevents dhm running when table doesn't exist. Adds prereq documentation

* Adds parallel, rmlines, err fix, comment fix, minor param fixes

* doc regen, common time range check and test updating

* Fixes job validation issues. Updates candle range checker.

* Ensures test cannot fail due to time.Now() shenanigans

* Fixes oopsie, adds documentation and a warn

* Fixes another time test, adjusts copy

* Drastically speeds up data history manager tests via function overrides

* Fixes summary bug and better logs

* Fixes local time test, fixes websocket tests

* removes defaults and comment,updates error messages,sets cli command args

* Fixes FTX trade processing

* Fixes issue where jobs got stuck if data wasn't returned but retrieval was successful

* Improves test speed. Simplifies trade verification SQL. Adds command help

* Fixes the oopsies

* Fixes use of query within transaction. Fixes trade err

* oopsie, not needed

* Adds missing data status. Properly ends job even when data is missing

* errors are more verbose and so have more words to describe them

* Doc regen for new status

* tiny test tinkering

* str := string("Removes .String()").String()

* Merge fixups

* Fixes a data race discovered during github actions

* Allows websocket test to pass consistently

* Fixes merge issue preventing datahistorymanager from starting via config

* Niterinos cmd defaults and explanations

* fixes default oopsie

* Fixes lack of nil protection

* Additional oopsie

* More detailed error for validating job exchange

CONTRIBUTORS

@@ -9,8 +9,8 @@ Rots | https://github.com/Rots
 vazha | https://github.com/vazha
 ermalguni | https://github.com/ermalguni
 MadCozBadd | https://github.com/MadCozBadd
-vadimzhukck | https://github.com/vadimzhukck
 dependabot[bot] | https://github.com/apps/dependabot
+vadimzhukck | https://github.com/vadimzhukck
 140am | https://github.com/140am
 marcofranssen | https://github.com/marcofranssen
 dackroyd | https://github.com/dackroyd
@@ -21,6 +21,7 @@ azhang | https://github.com/azhang
 andreygrehov | https://github.com/andreygrehov
 bretep | https://github.com/bretep
 Christian-Achilli | https://github.com/Christian-Achilli
+yangrq1018 | https://github.com/yangrq1018
 gam-phon | https://github.com/gam-phon
 cornelk | https://github.com/cornelk
 if1live | https://github.com/if1live
@@ -29,7 +30,7 @@ soxipy | https://github.com/soxipy
 mshogin | https://github.com/mshogin
 herenow | https://github.com/herenow
 blombard | https://github.com/blombard
-CodeLingoBot | https://github.com/CodeLingoBot
+tk42 | https://github.com/tk42
 daniel-cohen | https://github.com/daniel-cohen
 DirectX | https://github.com/DirectX
 frankzougc | https://github.com/frankzougc
@@ -45,3 +46,4 @@ lookfirst | https://github.com/lookfirst
 merkeld | https://github.com/merkeld
 CodeLingoTeam | https://github.com/CodeLingoTeam
 Daanikus | https://github.com/Daanikus
+CodeLingoBot | https://github.com/CodeLingoBot

README.md

@@ -143,17 +143,17 @@ Binaries will be published once the codebase reaches a stable condition.
 
 |User|Contribution Amount|
 |--|--|
-| [thrasher-](https://github.com/thrasher-) | 654 |
+| [thrasher-](https://github.com/thrasher-) | 655 |
 | [shazbert](https://github.com/shazbert) | 207 |
-| [gloriousCode](https://github.com/gloriousCode) | 179 |
+| [gloriousCode](https://github.com/gloriousCode) | 180 |
 | [dependabot-preview[bot]](https://github.com/apps/dependabot-preview) | 88 |
 | [xtda](https://github.com/xtda) | 47 |
 | [Rots](https://github.com/Rots) | 15 |
 | [vazha](https://github.com/vazha) | 15 |
 | [ermalguni](https://github.com/ermalguni) | 14 |
 | [MadCozBadd](https://github.com/MadCozBadd) | 12 |
+| [dependabot[bot]](https://github.com/apps/dependabot) | 12 |
 | [vadimzhukck](https://github.com/vadimzhukck) | 10 |
-| [dependabot[bot]](https://github.com/apps/dependabot) | 10 |
 | [140am](https://github.com/140am) | 8 |
 | [marcofranssen](https://github.com/marcofranssen) | 8 |
 | [dackroyd](https://github.com/dackroyd) | 5 |
@@ -164,6 +164,7 @@ Binaries will be published once the codebase reaches a stable condition.
 | [andreygrehov](https://github.com/andreygrehov) | 2 |
 | [bretep](https://github.com/bretep) | 2 |
 | [Christian-Achilli](https://github.com/Christian-Achilli) | 2 |
+| [yangrq1018](https://github.com/yangrq1018) | 2 |
 | [gam-phon](https://github.com/gam-phon) | 2 |
 | [cornelk](https://github.com/cornelk) | 2 |
 | [if1live](https://github.com/if1live) | 2 |
@@ -172,7 +173,7 @@ Binaries will be published once the codebase reaches a stable condition.
 | [mshogin](https://github.com/mshogin) | 2 |
 | [herenow](https://github.com/herenow) | 2 |
 | [blombard](https://github.com/blombard) | 1 |
-| [CodeLingoBot](https://github.com/CodeLingoBot) | 1 |
+| [tk42](https://github.com/tk42) | 2 |
 | [daniel-cohen](https://github.com/daniel-cohen) | 1 |
 | [DirectX](https://github.com/DirectX) | 1 |
 | [frankzougc](https://github.com/frankzougc) | 1 |
@@ -188,3 +189,4 @@ Binaries will be published once the codebase reaches a stable condition.
 | [merkeld](https://github.com/merkeld) | 1 |
 | [CodeLingoTeam](https://github.com/CodeLingoTeam) | 1 |
 | [Daanikus](https://github.com/Daanikus) | 1 |
+| [CodeLingoBot](https://github.com/CodeLingoBot) | 1 |

backtester/backtest/backtest.go

@@ -461,19 +461,19 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
 		}
 		resp.Item.RemoveDuplicates()
 		resp.Item.SortCandlesByTimestamp(false)
-		resp.Range = gctkline.CalculateCandleDateRanges(
+		resp.Range, err = gctkline.CalculateCandleDateRanges(
 			resp.Item.Candles[0].Time,
 			resp.Item.Candles[len(resp.Item.Candles)-1].Time.Add(cfg.DataSettings.Interval),
 			gctkline.Interval(cfg.DataSettings.Interval),
 			0,
 		)
-		err = resp.Range.VerifyResultsHaveData(resp.Item.Candles)
 		if err != nil {
-			if strings.Contains(err.Error(), "missing candles data between") {
-				log.Warn(log.BackTester, err.Error())
-			} else {
-				return nil, err
-			}
+			return nil, err
+		}
+		resp.Range.SetHasDataFromCandles(resp.Item.Candles)
+		summary := resp.Range.DataSummary(false)
+		if len(summary) > 0 {
+			log.Warnf(log.BackTester, "%v", summary)
 		}
 	case cfg.DataSettings.DatabaseData != nil:
 		if cfg.DataSettings.DatabaseData.InclusiveEndDate {
@@ -509,19 +509,19 @@ func (bt *BackTest) loadData(cfg *config.Config, exch gctexchange.IBotExchange,
 
 		resp.Item.RemoveDuplicates()
 		resp.Item.SortCandlesByTimestamp(false)
-		resp.Range = gctkline.CalculateCandleDateRanges(
+		resp.Range, err = gctkline.CalculateCandleDateRanges(
 			cfg.DataSettings.DatabaseData.StartDate,
 			cfg.DataSettings.DatabaseData.EndDate,
 			gctkline.Interval(cfg.DataSettings.Interval),
 			0,
 		)
-		err = resp.Range.VerifyResultsHaveData(resp.Item.Candles)
 		if err != nil {
-			if strings.Contains(err.Error(), "missing candles data between") {
-				log.Warn(log.BackTester, err.Error())
-			} else {
-				return nil, err
-			}
+			return nil, err
+		}
+		resp.Range.SetHasDataFromCandles(resp.Item.Candles)
+		summary := resp.Range.DataSummary(false)
+		if len(summary) > 0 {
+			log.Warnf(log.BackTester, "%v", summary)
 		}
 	case cfg.DataSettings.APIData != nil:
 		if cfg.DataSettings.APIData.InclusiveEndDate {
@@ -601,11 +601,14 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
 	if cfg.DataSettings.Interval <= 0 {
 		return nil, errIntervalUnset
 	}
-	dates := gctkline.CalculateCandleDateRanges(
+	dates, err := gctkline.CalculateCandleDateRanges(
 		cfg.DataSettings.APIData.StartDate,
 		cfg.DataSettings.APIData.EndDate,
 		gctkline.Interval(cfg.DataSettings.Interval),
 		resultLimit)
+	if err != nil {
+		return nil, err
+	}
 	candles, err := api.LoadData(
 		dataType,
 		cfg.DataSettings.APIData.StartDate,
@@ -617,14 +620,12 @@ func loadAPIData(cfg *config.Config, exch gctexchange.IBotExchange, fPair curren
 	if err != nil {
 		return nil, fmt.Errorf("%v. Please check your GoCryptoTrader configuration", err)
 	}
-	err = dates.VerifyResultsHaveData(candles.Candles)
-	if err != nil && errors.Is(err, gctkline.ErrMissingCandleData) {
-		log.Warn(log.BackTester, err.Error())
-	} else if err != nil {
-		return nil, err
+	dates.SetHasDataFromCandles(candles.Candles)
+	summary := dates.DataSummary(false)
+	if len(summary) > 0 {
+		log.Warnf(log.BackTester, "%v", summary)
 	}
-
-	candles.FillMissingDataWithEmptyEntries(&dates)
+	candles.FillMissingDataWithEmptyEntries(dates)
 	candles.RemoveOutsideRange(cfg.DataSettings.APIData.StartDate, cfg.DataSettings.APIData.EndDate)
 	return &kline.DataFromKline{
 		Item:  *candles,
@@ -983,13 +984,16 @@ func (bt *BackTest) loadLiveData(resp *kline.DataFromKline, cfg *config.Config,
 	}
 	endDate := candles.Candles[len(candles.Candles)-1].Time.Add(cfg.DataSettings.Interval)
 	if resp.Range.Ranges == nil {
-		dataRange := gctkline.CalculateCandleDateRanges(
+		dataRange, err := gctkline.CalculateCandleDateRanges(
 			startDate,
 			endDate,
 			gctkline.Interval(cfg.DataSettings.Interval),
 			0,
 		)
-		resp.Range = gctkline.IntervalRangeHolder{
+		if err != nil {
+			return err
+		}
+		resp.Range = &gctkline.IntervalRangeHolder{
 			Start:  gctkline.CreateIntervalTime(startDate),
 			End:    gctkline.CreateIntervalTime(endDate),
 			Ranges: dataRange.Ranges,

backtester/backtest/backtest_test.go

@@ -430,7 +430,7 @@ func TestFullCycle(t *testing.T) {
 			}},
 		},
 		Base: data.Base{},
-		Range: gctkline.IntervalRangeHolder{
+		Range: &gctkline.IntervalRangeHolder{
 			Start: gctkline.CreateIntervalTime(tt),
 			End:   gctkline.CreateIntervalTime(tt.Add(gctkline.FifteenMin.Duration())),
 			Ranges: []gctkline.IntervalRange{
@@ -528,7 +528,7 @@ func TestFullCycleMulti(t *testing.T) {
 			}},
 		},
 		Base: data.Base{},
-		Range: gctkline.IntervalRangeHolder{
+		Range: &gctkline.IntervalRangeHolder{
 			Start: gctkline.CreateIntervalTime(tt),
 			End:   gctkline.CreateIntervalTime(tt.Add(gctkline.FifteenMin.Duration())),
 			Ranges: []gctkline.IntervalRange{

backtester/config/configbuilder/main.go

@@ -400,13 +400,13 @@ func parseDatabase(reader *bufio.Reader, cfg *config.Config) error {
 			return fmt.Errorf("database failed to set config: %w", err)
 		}
 		if cfg.DataSettings.DatabaseData.ConfigOverride.Driver == database.DBPostgreSQL {
-			_, err = dbPSQL.Connect()
+			_, err = dbPSQL.Connect(cfg.DataSettings.DatabaseData.ConfigOverride)
 			if err != nil {
 				return fmt.Errorf("database failed to connect: %v", err)
 			}
 		} else if cfg.DataSettings.DatabaseData.ConfigOverride.Driver == database.DBSQLite ||
 			cfg.DataSettings.DatabaseData.ConfigOverride.Driver == database.DBSQLite3 {
-			_, err = dbsqlite3.Connect()
+			_, err = dbsqlite3.Connect(cfg.DataSettings.DatabaseData.ConfigOverride.Database)
 			if err != nil {
 				return fmt.Errorf("database failed to connect: %v", err)
 			}

backtester/data/kline/api/api_test.go

@@ -71,7 +71,7 @@ func TestLoadCandles(t *testing.T) {
 func TestLoadTrades(t *testing.T) {
 	t.Parallel()
 	interval := gctkline.FifteenMin
-	tt1 := time.Now().Add(-time.Minute * 15).Round(interval.Duration())
+	tt1 := time.Now().Add(-time.Minute * 30).Round(interval.Duration())
 	tt2 := time.Now().Round(interval.Duration())
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)

backtester/data/kline/kline.go

@@ -13,6 +13,9 @@ import (
 // HasDataAtTime verifies checks the underlying range data
 // To determine whether there is any candle data present at the time provided
 func (d *DataFromKline) HasDataAtTime(t time.Time) bool {
+	if d.Range == nil {
+		return false
+	}
 	return d.Range.HasDataAtDate(t)
 }
 

backtester/data/kline/kline_test.go

@@ -88,9 +88,12 @@ func TestHasDataAtTime(t *testing.T) {
 		t.Error("expected false")
 	}
 
-	ranger := gctkline.CalculateCandleDateRanges(dStart, dEnd, gctkline.OneDay, 100000)
+	ranger, err := gctkline.CalculateCandleDateRanges(dStart, dEnd, gctkline.OneDay, 100000)
+	if err != nil {
+		t.Error(err)
+	}
 	d.Range = ranger
-	_ = d.Range.VerifyResultsHaveData(d.Item.Candles)
+	d.Range.SetHasDataFromCandles(d.Item.Candles)
 	has = d.HasDataAtTime(dInsert)
 	if !has {
 		t.Error("expected true")

backtester/data/kline/kline_types.go

@@ -15,7 +15,7 @@ var errNoCandleData = errors.New("no candle data provided")
 type DataFromKline struct {
 	Item gctkline.Item
 	data.Base
-	Range gctkline.IntervalRangeHolder
+	Range *gctkline.IntervalRangeHolder
 
 	addedTimes map[time.Time]bool
 }

backtester/eventhandlers/statistics/currencystatistics/currencystatistics.go

@@ -259,11 +259,15 @@ func calculateMaxDrawdown(closePrices []common.DataEventHandler) Swing {
 			lowestTime = currTime
 		}
 		if highestPrice < currHigh && highestPrice > 0 {
-			intervals := gctkline.CalculateCandleDateRanges(highestTime, lowestTime, closePrices[i].GetInterval(), 0)
 			if lowestTime.Equal(highestTime) {
 				// create distinction if the greatest drawdown occurs within the same candle
 				lowestTime = lowestTime.Add((time.Hour * 23) + (time.Minute * 59) + (time.Second * 59))
 			}
+			intervals, err := gctkline.CalculateCandleDateRanges(highestTime, lowestTime, closePrices[i].GetInterval(), 0)
+			if err != nil {
+				log.Error(log.BackTester, err)
+				continue
+			}
 			swings = append(swings, Swing{
 				Highest: Iteration{
 					Time:  highestTime,
@@ -285,7 +289,14 @@ func calculateMaxDrawdown(closePrices []common.DataEventHandler) Swing {
 	}
 	if (len(swings) > 0 && swings[len(swings)-1].Lowest.Price != closePrices[len(closePrices)-1].LowPrice()) || swings == nil {
 		// need to close out the final drawdown
-		intervals := gctkline.CalculateCandleDateRanges(highestTime, lowestTime, closePrices[0].GetInterval(), 0)
+		if lowestTime.Equal(highestTime) {
+			// create distinction if the greatest drawdown occurs within the same candle
+			lowestTime = lowestTime.Add((time.Hour * 23) + (time.Minute * 59) + (time.Second * 59))
+		}
+		intervals, err := gctkline.CalculateCandleDateRanges(highestTime, lowestTime, closePrices[0].GetInterval(), 0)
+		if err != nil {
+			log.Error(log.BackTester, err)
+		}
 		drawdownPercent := 0.0
 		if highestPrice > 0 {
 			drawdownPercent = ((lowestPrice - highestPrice) / highestPrice) * 100

backtester/eventhandlers/statistics/currencystatistics/currencystatistics_test.go

@@ -219,7 +219,7 @@ func TestPrintResults(t *testing.T) {
 }
 
 func TestCalculateMaxDrawdown(t *testing.T) {
-	tt1 := time.Now().Round(gctkline.OneDay.Duration())
+	tt1 := time.Now().Add(-gctkline.OneDay.Duration() * 7).Round(gctkline.OneDay.Duration())
 	exch := testExchange
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)

backtester/eventhandlers/statistics/statistics_test.go

@@ -555,8 +555,8 @@ func TestCalculateTheResults(t *testing.T) {
 		t.Error(err)
 	}
 
-	tt := time.Now()
-	tt2 := time.Now().Add(time.Hour)
+	tt := time.Now().Add(-gctkline.OneDay.Duration() * 7)
+	tt2 := time.Now().Add(-gctkline.OneDay.Duration() * 6)
 	exch := testExchange
 	a := asset.Spot
 	p := currency.NewPair(currency.BTC, currency.USDT)

backtester/eventhandlers/strategies/base/base_test.go

@@ -50,7 +50,7 @@ func TestGetBase(t *testing.T) {
 	_, err = s.GetBaseData(&datakline.DataFromKline{
 		Item:  gctkline.Item{},
 		Base:  d,
-		Range: gctkline.IntervalRangeHolder{},
+		Range: &gctkline.IntervalRangeHolder{},
 	})
 	if err != nil {
 		t.Error(err)

backtester/eventhandlers/strategies/dollarcostaverage/dollarcostaverage_test.go

@@ -73,7 +73,7 @@ func TestOnSignal(t *testing.T) {
 	da := &kline.DataFromKline{
 		Item:  gctkline.Item{},
 		Base:  d,
-		Range: gctkline.IntervalRangeHolder{},
+		Range: &gctkline.IntervalRangeHolder{},
 	}
 	var resp signal.Event
 	resp, err = s.OnSignal(da, nil)
@@ -105,9 +105,12 @@ func TestOnSignal(t *testing.T) {
 		t.Error(err)
 	}
 
-	ranger := gctkline.CalculateCandleDateRanges(dStart, dEnd, gctkline.OneDay, 100000)
+	ranger, err := gctkline.CalculateCandleDateRanges(dStart, dEnd, gctkline.OneDay, 100000)
+	if err != nil {
+		t.Error(err)
+	}
 	da.Range = ranger
-	_ = da.Range.VerifyResultsHaveData(da.Item.Candles)
+	da.Range.SetHasDataFromCandles(da.Item.Candles)
 	resp, err = s.OnSignal(da, nil)
 	if err != nil {
 		t.Error(err)
@@ -149,7 +152,7 @@ func TestOnSignals(t *testing.T) {
 	da := &kline.DataFromKline{
 		Item:  gctkline.Item{},
 		Base:  d,
-		Range: gctkline.IntervalRangeHolder{},
+		Range: &gctkline.IntervalRangeHolder{},
 	}
 	var resp []signal.Event
 	resp, err = s.OnSimultaneousSignals([]data.Handler{da}, nil)
@@ -184,9 +187,12 @@ func TestOnSignals(t *testing.T) {
 		t.Error(err)
 	}
 
-	ranger := gctkline.CalculateCandleDateRanges(dStart, dEnd, gctkline.OneDay, 100000)
+	ranger, err := gctkline.CalculateCandleDateRanges(dStart, dEnd, gctkline.OneDay, 100000)
+	if err != nil {
+		t.Error(err)
+	}
 	da.Range = ranger
-	_ = da.Range.VerifyResultsHaveData(da.Item.Candles)
+	da.Range.SetHasDataFromCandles(da.Item.Candles)
 	resp, err = s.OnSimultaneousSignals([]data.Handler{da}, nil)
 	if err != nil {
 		t.Error(err)