Split the generation logic and filtering

2020-05-22 15:03:49 +03:00 · 2020-05-22 15:03:49 +03:00 · 8e89802b2d
commit 8e89802b2d
parent 74056e768a
5 changed files with 135 additions and 77 deletions
--- a/freqtrade/pairlist/IPairList.py
+++ b/freqtrade/pairlist/IPairList.py
@ -8,6 +8,7 @@ from typing import Any, Dict, List

 from cachetools import TTLCache, cached

+from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import market_is_active


@ -90,6 +91,16 @@ class IPairList(ABC):
        """
        raise NotImplementedError()

+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        raise OperationalException("This Pairlist Handler should not be used "
+                                   "at the first position in the list of Pairlist Handlers.")
+
    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
--- a/freqtrade/pairlist/StaticPairList.py
+++ b/freqtrade/pairlist/StaticPairList.py
@ -30,6 +30,15 @@ class StaticPairList(IPairList):
        """
        return f"{self.name}"

+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+
    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
@ -38,4 +47,4 @@ class StaticPairList(IPairList):
        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
        :return: new whitelist
        """
-        return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
+        return pairlist
--- a/freqtrade/pairlist/VolumePairList.py
+++ b/freqtrade/pairlist/VolumePairList.py
@ -68,6 +68,31 @@ class VolumePairList(IPairList):
        """
        return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."

+    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Generate the pairlist
+        :param cached_pairlist: Previously generated pairlist (cached)
+        :param tickers: Tickers (from exchange.get_tickers()).
+        :return: List of pairs
+        """
+        # Generate dynamic whitelist
+        # Must always run if this pairlist is not the first in the list.
+        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
+            self._last_refresh = int(datetime.now().timestamp())
+
+            # Use fresh pairlist
+            # Check if pair quote currency equals to the stake currency.
+            filtered_tickers = [
+                    v for k, v in tickers.items()
+                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
+                        and v[self._sort_key] is not None)]
+            pairlist = [s['symbol'] for s in filtered_tickers]
+        else:
+            # Use the cached pairlist if it's not time yet to refresh
+            pairlist = cached_pairlist
+
+        return pairlist
+
    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
@ -76,37 +101,8 @@ class VolumePairList(IPairList):
        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
        :return: new whitelist
        """
-        # Generate dynamic whitelist
-        # Must always run if this pairlist is not the first in the list.
-        if (self._pairlist_pos != 0 or
-                (self._last_refresh + self.refresh_period < datetime.now().timestamp())):
-
-            self._last_refresh = int(datetime.now().timestamp())
-            pairs = self._gen_pair_whitelist(pairlist, tickers)
-        else:
-            pairs = pairlist
-
-        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
-
-        return pairs
-
-    def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]:
-        """
-        Updates the whitelist with with a dynamically generated list
-        :param pairlist: pairlist to filter or sort
-        :param tickers: Tickers (from exchange.get_tickers()).
-        :return: List of pairs
-        """
-        if self._pairlist_pos == 0:
-            # If VolumePairList is the first in the list, use fresh pairlist
-            # Check if pair quote currency equals to the stake currency.
-            filtered_tickers = [
-                    v for k, v in tickers.items()
-                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
-                        and v[self._sort_key] is not None)]
-        else:
-            # If other pairlist is in front, use the incoming pairlist.
-            filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
+        # Use the incoming pairlist.
+        filtered_tickers = [v for k, v in tickers.items() if k in pairlist]

        if self._min_value > 0:
            filtered_tickers = [
@ -120,4 +116,6 @@ class VolumePairList(IPairList):
        # Limit pairlist to the requested number of pairs
        pairs = pairs[:self._number_pairs]

+        self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
+
        return pairs
--- a/freqtrade/pairlist/pairlistmanager.py
+++ b/freqtrade/pairlist/pairlistmanager.py
@ -87,6 +87,9 @@ class PairListManager():
        # Adjust whitelist if filters are using tickers
        pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)

+        # Generate the pairlist with first Pairlist Handler in the chain
+        pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
+
        # Process all Pairlist Handlers in the chain
        for pairlist_handler in self._pairlist_handlers:
            pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)
--- a/tests/pairlist/test_pairlist.py
+++ b/tests/pairlist/test_pairlist.py
@ -19,7 +19,8 @@ def whitelist_conf(default_conf):
        'TKN/BTC',
        'TRST/BTC',
        'SWT/BTC',
-        'BCC/BTC'
+        'BCC/BTC',
+        'HOT/BTC',
    ]
    default_conf['exchange']['pair_blacklist'] = [
        'BLK/BTC'
@ -201,21 +202,21 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
    assert set(whitelist) == set(pairslist)


-@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [
+@pytest.mark.parametrize("pairlists,base_currency,whitelist_result,operational_exception", [
    # VolumePairList only
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'], False),
    # Different sorting depending on quote or bid volume
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-     "BTC",  ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
+     "BTC",  ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False),
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']),
+     "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT'], False),
    # No pair for ETH, VolumePairList
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}],
-     "ETH", []),
+     "ETH", [], False),
    # No pair for ETH, StaticPairList
    ([{"method": "StaticPairList"}],
-     "ETH", []),
+     "ETH", [], False),
    # No pair for ETH, all handlers
    ([{"method": "StaticPairList"},
      {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
@ -223,57 +224,87 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
      {"method": "PriceFilter", "low_price_ratio": 0.03},
      {"method": "SpreadFilter", "max_spread_ratio": 0.005},
      {"method": "ShuffleFilter"}],
-     "ETH", []),
+     "ETH", [], False),
    # Precisionfilter and quote volume
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
-      {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+      {"method": "PrecisionFilter"}],
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False),
    # Precisionfilter bid
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"},
-      {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']),
+      {"method": "PrecisionFilter"}],
+     "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False),
    # PriceFilter and VolumePairList
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "PriceFilter", "low_price_ratio": 0.03}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False),
    # PriceFilter and VolumePairList
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "PriceFilter", "low_price_ratio": 0.03}],
-     "USDT", ['ETH/USDT', 'NANO/USDT']),
+     "USDT", ['ETH/USDT', 'NANO/USDT'], False),
    # Hot is removed by precision_filter, Fuel by low_price_filter.
    ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"},
      {"method": "PrecisionFilter"},
      {"method": "PriceFilter", "low_price_ratio": 0.02}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False),
    # HOT and XRP are removed because below 1250 quoteVolume
    ([{"method": "VolumePairList", "number_assets": 5,
       "sort_key": "quoteVolume", "min_value": 1250}],
-     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], False),
    # StaticPairlist only
    ([{"method": "StaticPairList"}],
-     "BTC", ['ETH/BTC', 'TKN/BTC']),
+     "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC'], False),
    # Static Pairlist before VolumePairList - sorting changes
    ([{"method": "StaticPairList"},
      {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}],
-     "BTC", ['TKN/BTC', 'ETH/BTC']),
+     "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC'], False),
    # SpreadFilter
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
-     "USDT", ['ETH/USDT']),
+     "USDT", ['ETH/USDT'], False),
    # ShuffleFilter
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "ShuffleFilter", "seed": 77}],
-     "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']),
+     "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT'], False),
    # ShuffleFilter, other seed
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "ShuffleFilter", "seed": 42}],
-     "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']),
+     "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT'], False),
    # ShuffleFilter, no seed
    ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"},
      {"method": "ShuffleFilter"}],
-     "USDT", 3),
+     "USDT", 3, False),
+    # PrecisionFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "PrecisionFilter"}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], False),
+    # PrecisionFilter only
+    ([{"method": "PrecisionFilter"}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], True),
+    # PriceFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "PriceFilter", "low_price_ratio": 0.02}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], False),
+    # PriceFilter only
+    ([{"method": "PriceFilter", "low_price_ratio": 0.02}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], True),
+    # ShuffleFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "ShuffleFilter", "seed": 42}],
+     "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], False),
+    # ShuffleFilter only
+    ([{"method": "ShuffleFilter", "seed": 42}],
+     "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], True),
+    # SpreadFilter after StaticPairList
+    ([{"method": "StaticPairList"},
+      {"method": "SpreadFilter", "max_spread_ratio": 0.005}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], False),
+    # SpreadFilter only
+    ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}],
+     "BTC", ['ETH/BTC', 'TKN/BTC'], True),
 ])
 def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers,
                                      pairlists, base_currency, whitelist_result,
-                                      caplog) -> None:
+                                      operational_exception, caplog) -> None:
    whitelist_conf['pairlists'] = pairlists
    whitelist_conf['stake_currency'] = base_currency

@ -285,32 +316,38 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
                          markets=PropertyMock(return_value=shitcoinmarkets),
                          )

-    freqtrade.pairlists.refresh_pairlist()
-    whitelist = freqtrade.pairlists.whitelist
-
-    assert isinstance(whitelist, list)
-
-    # Verify length of pairlist matches (used for ShuffleFilter without seed)
-    if type(whitelist_result) is list:
-        assert whitelist == whitelist_result
+    if operational_exception:
+        with pytest.raises(OperationalException,
+                           match=r"This Pairlist Handler should not be used at the first position "
+                                 r"in the list of Pairlist Handlers."):
+            freqtrade.pairlists.refresh_pairlist()
    else:
-        len(whitelist) == whitelist_result
+        freqtrade.pairlists.refresh_pairlist()
+        whitelist = freqtrade.pairlists.whitelist

-    for pairlist in pairlists:
-        if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
-            assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
-                              r'would be <= stop limit.*', caplog)
-        if pairlist['method'] == 'PriceFilter' and whitelist_result:
-            assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or
-                    log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] is empty.*",
-                               caplog))
-        if pairlist['method'] == 'VolumePairList':
-            logmsg = ("DEPRECATED: using any key other than quoteVolume for "
-                      "VolumePairList is deprecated.")
-            if pairlist['sort_key'] != 'quoteVolume':
-                assert log_has(logmsg, caplog)
-            else:
-                assert not log_has(logmsg, caplog)
+        assert isinstance(whitelist, list)
+
+        # Verify length of pairlist matches (used for ShuffleFilter without seed)
+        if type(whitelist_result) is list:
+            assert whitelist == whitelist_result
+        else:
+            len(whitelist) == whitelist_result
+
+        for pairlist in pairlists:
+            if pairlist['method'] == 'PrecisionFilter' and whitelist_result:
+                assert log_has_re(r'^Removed .* from whitelist, because stop price .* '
+                                  r'would be <= stop limit.*', caplog)
+            if pairlist['method'] == 'PriceFilter' and whitelist_result:
+                assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or
+                        log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] "
+                                   r"is empty.*", caplog))
+            if pairlist['method'] == 'VolumePairList':
+                logmsg = ("DEPRECATED: using any key other than quoteVolume for "
+                          "VolumePairList is deprecated.")
+                if pairlist['sort_key'] != 'quoteVolume':
+                    assert log_has(logmsg, caplog)
+                else:
+                    assert not log_has(logmsg, caplog)


 def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: