Split the generation logic and filtering

freqtrade/pairlist/IPairList.py

@@ -8,6 +8,7 @@ from typing import Any, Dict, List
 
 from cachetools import TTLCache, cached
 
+from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active
 
 
@@ -90,6 +91,16 @@ class IPairList(ABC):
         """
         raise NotImplementedError()
 
+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        raise OperationalException("This Pairlist Handler should not be used "
+                                   "at the first position in the list of Pairlist Handlers.")
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.

freqtrade/pairlist/StaticPairList.py

@@ -30,6 +30,15 @@ class StaticPairList(IPairList):
         """
         return f"{self.name}"
 
+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.
@@ -38,4 +47,4 @@ class StaticPairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+        return pairlist

freqtrade/pairlist/VolumePairList.py

@@ -68,6 +68,31 @@ class VolumePairList(IPairList):
         """
         return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
 
+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        # Generate dynamic whitelist
+        # Must always run if this pairlist is not the first in the list.
+        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
+            self._last_refresh = int(datetime.now().timestamp())
+
+            # Use fresh pairlist
+            # Check if pair quote currency equals to the stake currency.
+            filtered_tickers = [
+                    v for k, v in tickers.items()
+                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
+                        and v[self._sort_key] is not None)]
+            pairlist = [s['symbol'] for s in filtered_tickers]
+        else:
+            # Use the cached pairlist if it's not time yet to refresh
+            pairlist = cached_pairlist
+
+        return pairlist
+
     def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
         """
         Filters and sorts pairlist and returns the whitelist again.
@@ -76,37 +101,8 @@ class VolumePairList(IPairList):
         :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: new whitelist
         """
-        # Generate dynamic whitelist
+        # Use the incoming pairlist.
-        # Must always run if this pairlist is not the first in the list.
+        filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
-        if (self._pairlist_pos != 0 or
-                (self._last_refresh + self.refresh_period < datetime.now().timestamp())):
-
-            self._last_refresh = int(datetime.now().timestamp())
-            pairs = self._gen_pair_whitelist(pairlist, tickers)
-        else:
-            pairs = pairlist
-
-        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
-
-        return pairs
-
-    def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]:
-        """
-        Updates the whitelist with with a dynamically generated list
-        :param pairlist: pairlist to filter or sort
-        :param tickers: Tickers (from exchange.get_tickers()).
-        :return: List of pairs
-        """
-        if self._pairlist_pos == 0:
-            # If VolumePairList is the first in the list, use fresh pairlist
-            # Check if pair quote currency equals to the stake currency.
-            filtered_tickers = [
-                    v for k, v in tickers.items()
-                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
-                        and v[self._sort_key] is not None)]
-        else:
-            # If other pairlist is in front, use the incoming pairlist.
-            filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
 
         if self._min_value > 0:
             filtered_tickers = [
@@ -120,4 +116,6 @@ class VolumePairList(IPairList):
         # Limit pairlist to the requested number of pairs
         pairs = pairs[:self._number_pairs]
 
+        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
+
         return pairs

freqtrade/pairlist/pairlistmanager.py

@@ -87,6 +87,9 @@ class PairListManager():
         # Adjust whitelist if filters are using tickers
         pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
 
+        # Generate the pairlist with first Pairlist Handler in the chain
+        pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
+
         # Process all Pairlist Handlers in the chain
         for pairlist_handler in self._pairlist_handlers:
             pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)

tests/pairlist/test_pairlist.py

@@ -19,7 +19,8 @@ def whitelist_conf(default_conf):
         'TKN/BTC',
         'TRST/BTC',
         'SWT/BTC',
-        'BCC/BTC'
+        'BCC/BTC',
+        'HOT/BTC',
     ]
     default_conf['exchange']['pair_blacklist'] = [
         'BLK/BTC'
@@ -201,21 +202,21 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
     assert set(whitelist) == set(pairslist)
 
 
-@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [
+@pytest.mark.parametrize("pairlists,base_currency,whitelist_result,operational_exception", [
     # VolumePairList only
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'], False),
     # Different sorting depending on quote or bid volume
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-     "BTC",  ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
+     "BTC",  ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False),
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']),
+     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT'], False),
     # No pair for ETH, VolumePairList
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "ETH", []),
+     "ETH", [], False),
     # No pair for ETH, StaticPairList
     ([{"method": "StaticPairList"}],
-     "ETH", []),
+     "ETH", [], False),
     # No pair for ETH, all handlers
     ([{"method": "StaticPairList"},
       {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
@@ -223,57 +224,87 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
       {"method": "PriceFilter", "low_price_ratio": 0.03},
       {"method": "SpreadFilter", "max_spread_ratio": 0.005},
       {"method": "ShuffleFilter"}],
-     "ETH", []),
+     "ETH", [], False),
     # Precisionfilter and quote volume
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+      {"method": "PrecisionFilter"}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False),
     # Precisionfilter bid
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
-      {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
+      {"method": "PrecisionFilter"}],
+     "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False),
     # PriceFilter and VolumePairList
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PriceFilter", "low_price_ratio": 0.03}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False),
     # PriceFilter and VolumePairList
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "PriceFilter", "low_price_ratio": 0.03}],
-     "USDT", ['ETH/USDT', 'NANO/USDT']),
+     "USDT", ['ETH/USDT', 'NANO/USDT'], False),
     # Hot is removed by precision_filter, Fuel by low_price_filter.
     ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"},
       {"method": "PrecisionFilter"},
       {"method": "PriceFilter", "low_price_ratio": 0.02}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False),
     # HOT and XRP are removed because below 1250 quoteVolume
     ([{"method": "VolumePairList", "number_assets": 5,
        "sort_key": "quoteVolume", "min_value": 1250}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], False),
     # StaticPairlist only
     ([{"method": "StaticPairList"}],
-     "BTC", ['ETH/BTC', 'TKN/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC'], False),
     # Static Pairlist before VolumePairList - sorting changes
     ([{"method": "StaticPairList"},
       {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-     "BTC", ['TKN/BTC', 'ETH/BTC']),
+     "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC'], False),
     # SpreadFilter
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
-     "USDT", ['ETH/USDT']),
+     "USDT", ['ETH/USDT'], False),
     # ShuffleFilter
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter", "seed": 77}],
-     "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']),
+     "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT'], False),
     # ShuffleFilter, other seed
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter", "seed": 42}],
-     "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']),
+     "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT'], False),
     # ShuffleFilter, no seed
     ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
       {"method": "ShuffleFilter"}],
-     "USDT", 3),
+     "USDT", 3, False),
+    # PrecisionFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "PrecisionFilter"}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], False),
+    # PrecisionFilter only
+    ([{"method": "PrecisionFilter"}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], True),
+    # PriceFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "PriceFilter", "low_price_ratio": 0.02}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], False),
+    # PriceFilter only
+    ([{"method": "PriceFilter", "low_price_ratio": 0.02}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], True),
+    # ShuffleFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "ShuffleFilter", "seed": 42}],
+     "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], False),
+    # ShuffleFilter only
+    ([{"method": "ShuffleFilter", "seed": 42}],
+     "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], True),
+    # SpreadFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], False),
+    # SpreadFilter only
+    ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], True),
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                       pairlists, base_currency, whitelist_result,
-                                      caplog) -> None:
+                                      operational_exception, caplog) -> None:
     whitelist_conf['pairlists'] = pairlists
     whitelist_conf['stake_currency'] = base_currency
 
@@ -285,32 +316,38 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                           markets=PropertyMock(return_value=shitcoinmarkets),
                           )
 
-    freqtrade.pairlists.refresh_pairlist()
+    if operational_exception:
-    whitelist = freqtrade.pairlists.whitelist
+        with pytest.raises(OperationalException,
-
+                           match=r"This Pairlist Handler should not be used at the first position "
-    assert isinstance(whitelist, list)
+                                 r"in the list of Pairlist Handlers."):
-
+            freqtrade.pairlists.refresh_pairlist()
-    # Verify length of pairlist matches (used for ShuffleFilter without seed)
-    if type(whitelist_result) is list:
-        assert whitelist == whitelist_result
     else:
-        len(whitelist) == whitelist_result
+        freqtrade.pairlists.refresh_pairlist()
+        whitelist = freqtrade.pairlists.whitelist
 
-    for pairlist in pairlists:
+        assert isinstance(whitelist, list)
-        if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
+
-            assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
+        # Verify length of pairlist matches (used for ShuffleFilter without seed)
-                              r'would be <= stop limit.*', caplog)
+        if type(whitelist_result) is list:
-        if pairlist['method'] == 'PriceFilter' and whitelist_result:
+            assert whitelist == whitelist_result
-            assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or
+        else:
-                    log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] is empty.*",
+            len(whitelist) == whitelist_result
-                               caplog))
+
-        if pairlist['method'] == 'VolumePairList':
+        for pairlist in pairlists:
-            logmsg = ("DEPRECATED: using any key other than quoteVolume for "
+            if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
-                      "VolumePairList is deprecated.")
+                assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
-            if pairlist['sort_key'] != 'quoteVolume':
+                                  r'would be <= stop limit.*', caplog)
-                assert log_has(logmsg, caplog)
+            if pairlist['method'] == 'PriceFilter' and whitelist_result:
-            else:
+                assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or
-                assert not log_has(logmsg, caplog)
+                        log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] "
+                                   r"is empty.*", caplog))
+            if pairlist['method'] == 'VolumePairList':
+                logmsg = ("DEPRECATED: using any key other than quoteVolume for "
+                          "VolumePairList is deprecated.")
+                if pairlist['sort_key'] != 'quoteVolume':
+                    assert log_has(logmsg, caplog)
+                else:
+                    assert not log_has(logmsg, caplog)
 
 
 def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: