Merge pull request freqtrade#4801 from freqtrade/pairlist_caching

Cache pairlist in pairlist, not globally

freqtrade/plugins/pairlist/IPairList.py

@@ -73,7 +73,7 @@ def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
         """
         raise NotImplementedError()
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist.
 
@@ -84,7 +84,6 @@ def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
         it will raise the exception if a Pairlist Handler is used at the first
         position in the chain.
 
-        :param cached_pairlist: Previously generated pairlist (cached)
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: List of pairs
         """

freqtrade/plugins/pairlist/StaticPairList.py

@@ -42,10 +42,9 @@ def short_desc(self) -> str:
         """
         return f"{self.name}"
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist
-        :param cached_pairlist: Previously generated pairlist (cached)
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: List of pairs
         """

freqtrade/plugins/pairlist/VolumePairList.py

@@ -4,9 +4,10 @@
 Provides dynamic pair list based on trade volumes
 """
 import logging
-from datetime import datetime
 from typing import Any, Dict, List
 
+from cachetools.ttl import TTLCache
+
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
@@ -33,7 +34,8 @@ def __init__(self, exchange, pairlistmanager,
         self._number_pairs = self._pairlistconfig['number_assets']
         self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
         self._min_value = self._pairlistconfig.get('min_value', 0)
-        self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -63,17 +65,19 @@ def short_desc(self) -> str:
         """
         return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist
-        :param cached_pairlist: Previously generated pairlist (cached)
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: List of pairs
         """
         # Generate dynamic whitelist
         # Must always run if this pairlist is not the first in the list.
-        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
-            self._last_refresh = int(datetime.now().timestamp())
+        pairlist = self._pair_cache.get('pairlist')
+        if pairlist:
+            # Item found - no refresh necessary
+            return pairlist
+        else:
 
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
@@ -82,9 +86,9 @@ def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
                     if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
                         and v[self._sort_key] is not None)]
             pairlist = [s['symbol'] for s in filtered_tickers]
-        else:
-            # Use the cached pairlist if it's not time yet to refresh
-            pairlist = cached_pairlist
+
+            pairlist = self.filter_pairlist(pairlist, tickers)
+            self._pair_cache['pairlist'] = pairlist
 
         return pairlist
 

freqtrade/plugins/pairlistmanager.py

@@ -3,7 +3,7 @@
 """
 import logging
 from copy import deepcopy
-from typing import Any, Dict, List
+from typing import Dict, List
 
 from cachetools import TTLCache, cached
 
@@ -79,11 +79,8 @@ def refresh_pairlist(self) -> None:
         if self._tickers_needed:
             tickers = self._get_cached_tickers()
 
-        # Adjust whitelist if filters are using tickers
-        pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
-
         # Generate the pairlist with first Pairlist Handler in the chain
-        pairlist = self._pairlist_handlers[0].gen_pairlist(pairlist, tickers)
+        pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
 
         # Process all Pairlist Handlers in the chain
         for pairlist_handler in self._pairlist_handlers:
@@ -95,19 +92,6 @@ def refresh_pairlist(self) -> None:
 
         self._whitelist = pairlist
 
-    def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
-        """
-        Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
-        pairs that do not have ticker available
-        """
-        if self._tickers_needed:
-            # Copy list since we're modifying this list
-            for p in deepcopy(pairlist):
-                if p not in tickers:
-                    pairlist.remove(p)
-
-        return pairlist
-
     def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
         """
         Verify and remove items from pairlist - returning a filtered pairlist.

tests/plugins/test_pairlist.py

@@ -1,5 +1,6 @@
 # pragma pylint: disable=missing-docstring,C0103,protected-access
 
+import time
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
@@ -260,6 +261,8 @@ def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_
     freqtrade.pairlists.refresh_pairlist()
     assert whitelist == freqtrade.pairlists.whitelist
 
+    # Delay to allow 0 TTL cache to expire...
+    time.sleep(1)
     whitelist = ['FUEL/BTC', 'ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']
     tickers_dict['FUEL/BTC']['quoteVolume'] = 10000.0
     freqtrade.pairlists.refresh_pairlist()
@@ -604,17 +607,14 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
                           get_tickers=tickers
                           )
     freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
-    assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == 0
+    assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 0
     assert tickers.call_count == 0
     freqtrade.pairlists.refresh_pairlist()
     assert tickers.call_count == 1
 
-    assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh != 0
-    lrf = freqtrade.pairlists._pairlist_handlers[0]._last_refresh
+    assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 1
     freqtrade.pairlists.refresh_pairlist()
     assert tickers.call_count == 1
-    # Time should not be updated.
-    assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf
 
 
 def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers):