Merge pull request freqtrade#8272 from paranoidandy/bot-loop-start-ev…

…ery-candle-bt

Make strategy.bot_loop_start run once per candle in backtest

docs/bot-basics.md

@@ -60,10 +60,10 @@ This loop will be repeated again and again until the bot is stopped.
 
 * Load historic data for configured pairlist.
 * Calls `bot_start()` once.
-* Calls `bot_loop_start()` once.
 * Calculate indicators (calls `populate_indicators()` once per pair).
 * Calculate entry / exit signals (calls `populate_entry_trend()` and `populate_exit_trend()` once per pair).
 * Loops per candle simulating entry and exit points.
+  * Calls `bot_loop_start()` strategy callback.
   * Check for Order timeouts, either via the `unfilledtimeout` configuration, or via `check_entry_timeout()` / `check_exit_timeout()` strategy callbacks.
   * Calls `adjust_entry_price()` strategy callback for open entry orders.
   * Check for trade entry signals (`enter_long` / `enter_short` columns).

docs/strategy-callbacks.md

@@ -51,7 +51,8 @@ During hyperopt, this runs only once at startup.
 
 ## Bot loop start
 
-A simple callback which is called once at the start of every bot throttling iteration (roughly every 5 seconds, unless configured differently).
+A simple callback which is called once at the start of every bot throttling iteration in dry/live mode (roughly every 5
+seconds, unless configured differently) or once per candle in backtest/hyperopt mode.
 This can be used to perform calculations which are pair independent (apply to all pairs), loading of external data, etc.
 
 ``` python
@@ -61,11 +62,12 @@ class AwesomeStrategy(IStrategy):
 
     # ... populate_* methods
 
-    def bot_loop_start(self, **kwargs) -> None:
+    def bot_loop_start(self, current_time: datetime, **kwargs) -> None:
         """
         Called at the start of the bot iteration (one loop).
         Might be used to perform pair-independent tasks
         (e.g. gather some remote resource for comparison)
+        :param current_time: datetime object, containing the current datetime
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         """
         if self.config['runmode'].value in ('live', 'dry_run'):

freqtrade/freqtradebot.py

@@ -212,7 +212,8 @@ def process(self) -> None:
         self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist),
                                   self.strategy.gather_informative_pairs())
 
-        strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
+        strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)(
+            current_time=datetime.now(timezone.utc))
 
         self.strategy.analyze(self.active_pair_whitelist)
 

freqtrade/optimize/backtesting.py

@@ -205,7 +205,6 @@ def _set_strategy(self, strategy: IStrategy):
         self.strategy.order_types['stoploss_on_exchange'] = False
 
         self.strategy.ft_bot_start()
-        strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
 
     def _load_protections(self, strategy: IStrategy):
         if self.config.get('enable_protections', False):
@@ -1155,6 +1154,8 @@ def backtest(self, processed: Dict,
         while current_time <= end_date:
             open_trade_count_start = LocalTrade.bt_open_open_trade_count
             self.check_abort()
+            strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)(
+                current_time=current_time)
             for i, pair in enumerate(data):
                 row_index = indexes[pair]
                 row = self.validate_row(data, pair, row_index, current_time)

freqtrade/plot/plotting.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -635,7 +636,7 @@ def load_and_plot_trades(config: Config):
     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
     IStrategy.dp = DataProvider(config, exchange)
     strategy.ft_bot_start()
-    strategy.bot_loop_start()
+    strategy.bot_loop_start(datetime.now(timezone.utc))
     plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
     timerange = plot_elements['timerange']
     trades = plot_elements['trades']

freqtrade/strategy/interface.py

@@ -251,11 +251,12 @@ def bot_start(self, **kwargs) -> None:
         """
         pass
 
-    def bot_loop_start(self, **kwargs) -> None:
+    def bot_loop_start(self, current_time: datetime, **kwargs) -> None:
         """
         Called at the start of the bot iteration (one loop).
         Might be used to perform pair-independent tasks
         (e.g. gather some remote resource for comparison)
+        :param current_time: datetime object, containing the current datetime
         :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
         """
         pass

freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2

@@ -1,5 +1,5 @@
 
-def bot_loop_start(self, **kwargs) -> None:
+def bot_loop_start(self, current_time: datetime, **kwargs) -> None:
     """
     Called at the start of the bot iteration (one loop).
     Might be used to perform pair-independent tasks
@@ -8,6 +8,7 @@ def bot_loop_start(self, **kwargs) -> None:
     For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
 
     When not implemented by a strategy, this simply does nothing.
+    :param current_time: datetime object, containing the current datetime
     :param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
     """
     pass

tests/optimize/test_backtesting.py

@@ -344,7 +344,7 @@ def test_backtest_abort(default_conf, mocker, testdatadir) -> None:
     assert backtesting.progress.progress == 0
 
 
-def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
+def test_backtesting_start(default_conf, mocker, caplog) -> None:
     def get_timerange(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
 
@@ -367,6 +367,7 @@ def get_timerange(input1):
     backtesting = Backtesting(default_conf)
     backtesting._set_strategy(backtesting.strategylist[0])
     backtesting.strategy.bot_loop_start = MagicMock()
+    backtesting.strategy.bot_start = MagicMock()
     backtesting.start()
     # check the logs, that will contain the backtest result
     exists = [
@@ -376,7 +377,8 @@ def get_timerange(input1):
     for line in exists:
         assert log_has(line, caplog)
     assert backtesting.strategy.dp._pairlists is not None
-    assert backtesting.strategy.bot_loop_start.call_count == 1
+    assert backtesting.strategy.bot_start.call_count == 1
+    assert backtesting.strategy.bot_loop_start.call_count == 0
     assert sbs.call_count == 1
     assert sbc.call_count == 1
 

tests/optimize/test_hyperopt.py

@@ -872,7 +872,8 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None:
     hyperopt.backtesting.exchange.get_max_leverage = MagicMock(return_value=1.0)
     assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto)
     assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter)
-    assert hyperopt.backtesting.strategy.bot_loop_started is True
+    assert hyperopt.backtesting.strategy.bot_started is True
+    assert hyperopt.backtesting.strategy.bot_loop_started is False
 
     assert hyperopt.backtesting.strategy.buy_rsi.in_space is True
     assert hyperopt.backtesting.strategy.buy_rsi.value == 35
@@ -922,7 +923,8 @@ def test_in_strategy_auto_hyperopt_with_parallel(mocker, hyperopt_conf, tmpdir,
 
     assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto)
     assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter)
-    assert hyperopt.backtesting.strategy.bot_loop_started is True
+    assert hyperopt.backtesting.strategy.bot_started is True
+    assert hyperopt.backtesting.strategy.bot_loop_started is False
 
     assert hyperopt.backtesting.strategy.buy_rsi.in_space is True
     assert hyperopt.backtesting.strategy.buy_rsi.value == 35
@@ -959,7 +961,8 @@ def test_in_strategy_auto_hyperopt_per_epoch(mocker, hyperopt_conf, tmpdir, fee)
     hyperopt.backtesting.exchange.get_max_leverage = MagicMock(return_value=1.0)
     assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto)
     assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter)
-    assert hyperopt.backtesting.strategy.bot_loop_started is True
+    assert hyperopt.backtesting.strategy.bot_loop_started is False
+    assert hyperopt.backtesting.strategy.bot_started is True
 
     assert hyperopt.backtesting.strategy.buy_rsi.in_space is True
     assert hyperopt.backtesting.strategy.buy_rsi.value == 35

tests/strategy/strats/hyperoptable_strategy.py

@@ -50,6 +50,7 @@ def protections(self):
         return prot
 
     bot_loop_started = False
+    bot_started = False
 
     def bot_loop_start(self):
         self.bot_loop_started = True
@@ -58,6 +59,7 @@ def bot_start(self, **kwargs) -> None:
         """
         Parameters can also be defined here ...
         """
+        self.bot_started = True
         self.buy_rsi = IntParameter([0, 50], default=30, space='buy')
 
     def informative_pairs(self):