Merge pull request #3358 from hroff-1902/refactor_generate_pairlist

Split the pairlist generation logic and filtering

freqtrade/pairlist/IPairList.py

@@ -8,6 +8,7 @@ from typing import Any, Dict, List
 
 from cachetools import TTLCache, cached
 
+from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active
 
 
@@ -90,6 +91,24 @@ class IPairList(ABC):
         """
         raise NotImplementedError()
 
+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist.
+
+        This method is called once by the pairlistmanager in the refresh_pairlist()
+        method to supply the starting pairlist for the chain of the Pairlist Handlers.
+        Pairlist Filters (those Pairlist Handlers that cannot be used at the first
+        position in the chain) shall not override this base implementation --
+        it will raise the exception if a Pairlist Handler is used at the first
+        position in the chain.
+
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        raise OperationalException("This Pairlist Handler should not be used "
+                                   "at the first position in the list of Pairlist Handlers.")
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.

freqtrade/pairlist/StaticPairList.py

@@ -4,8 +4,9 @@ Static Pair List provider
 Provides pair white list as it configured in config
 """
 import logging
-from typing import Dict, List
+from typing import Any, Dict, List
 
+from freqtrade.exceptions import OperationalException
 from freqtrade.pairlist.IPairList import IPairList
 
 
@@ -14,6 +15,15 @@ logger = logging.getLogger(__name__)
 
 class StaticPairList(IPairList):
 
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        if self._pairlist_pos != 0:
+            raise OperationalException(f"{self.name} can only be used in the first position "
+                                       "in the list of Pairlist Handlers.")
+
     @property
     def needstickers(self) -> bool:
         """
@@ -30,6 +40,15 @@ class StaticPairList(IPairList):
         """
         return f"{self.name}"
 
+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.
@@ -38,4 +57,4 @@ class StaticPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+        return pairlist

freqtrade/pairlist/VolumePairList.py

@@ -68,6 +68,31 @@ class VolumePairList(IPairList):
         """
         return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
 
+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        # Generate dynamic whitelist
+        # Must always run if this pairlist is not the first in the list.
+        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
+            self._last_refresh = int(datetime.now().timestamp())
+
+            # Use fresh pairlist
+            # Check if pair quote currency equals to the stake currency.
+            filtered_tickers = [
+                    v for k, v in tickers.items()
+                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
+                        and v[self._sort_key] is not None)]
+            pairlist = [s['symbol'] for s in filtered_tickers]
+        else:
+            # Use the cached pairlist if it's not time yet to refresh
+            pairlist = cached_pairlist
+
+        return pairlist
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.
@@ -76,37 +101,8 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        # Generate dynamic whitelist
-        # Must always run if this pairlist is not the first in the list.
-        if (self._pairlist_pos != 0 or
-                (self._last_refresh + self.refresh_period < datetime.now().timestamp())):
-
-            self._last_refresh = int(datetime.now().timestamp())
-            pairs = self._gen_pair_whitelist(pairlist, tickers)
-        else:
-            pairs = pairlist
-
-        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
-
-        return pairs
-
-    def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]:
-        """
-        Updates the whitelist with with a dynamically generated list
-        :param pairlist: pairlist to filter or sort
-        :param tickers: Tickers (from exchange.get_tickers()).
-        :return: List of pairs
-        """
-        if self._pairlist_pos == 0:
-            # If VolumePairList is the first in the list, use fresh pairlist
-            # Check if pair quote currency equals to the stake currency.
-            filtered_tickers = [
-                    v for k, v in tickers.items()
-                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
-                        and v[self._sort_key] is not None)]
-        else:
-            # If other pairlist is in front, use the incoming pairlist.
-            filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
+        # Use the incoming pairlist.
+        filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
         if self._min_value > 0:
             filtered_tickers = [
@@ -120,4 +116,6 @@ class VolumePairList(IPairList):
         # Limit pairlist to the requested number of pairs
         pairs = pairs[:self._number_pairs]
 
+        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
+
         return pairs

freqtrade/pairlist/pairlistmanager.py

@@ -87,6 +87,9 @@ class PairListManager():
         # Adjust whitelist if filters are using tickers
         pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
 
+        # Generate the pairlist with first Pairlist Handler in the chain
+        pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
+
         # Process all Pairlist Handlers in the chain
         for pairlist_handler in self._pairlist_handlers:
             pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)