Merge pull request #4133 from freqtrade/dynamic_pairlist

Wildcard based blacklist

docs/includes/pairlists.md

@@ -10,6 +10,14 @@ If multiple Pairlist Handlers are used, they are chained and a combination of al
 
 Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist.
 
+### Pair blacklist
+
+The pair blacklist (configured via `exchange.pair_blacklist` in the configuration) disallows certain pairs from trading.
+This can be as simple as excluding `DOGE/BTC` - which will remove exactly this pair.
+
+The pair-blacklist does also support wildcards (in regex-style) - so `BNB/.*` will exclude ALL pairs that start with BNB.
+You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged tokens (check Pair naming conventions for your exchange!)
+
 ### Available Pairlist Handlers
 
 * [`StaticPairList`](#static-pair-list) (default, if not configured differently)

freqtrade/exchange/exchange.py

@@ -208,7 +208,7 @@ class Exchange:
         return self._api.precisionMode
 
     def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None,
-                    pairs_only: bool = False, active_only: bool = False) -> Dict:
+                    pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]:
         """
         Return exchange ccxt markets, filtered out by base currency and quote currency
         if this was requested in parameters.

freqtrade/plugins/pairlist/pairlist_helpers.py

@@ -0,0 +1,23 @@
+import re
+from typing import List
+
+
+def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]:
+    """
+    Expand pairlist potentially containing wildcards based on available markets.
+    This will implicitly filter all pairs in the wildcard-list which are not in available_pairs.
+    :param wildcardpl: List of Pairlists, which may contain regex
+    :param available_pairs: List of all available pairs (`exchange.get_markets().keys()`)
+    :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs.
+    :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`)
+    """
+    result = []
+    for pair_wc in wildcardpl:
+        try:
+            comp = re.compile(pair_wc)
+            result += [
+                pair for pair in available_pairs if re.match(comp, pair)
+            ]
+        except re.error as err:
+            raise ValueError(f"Wildcard error in {pair_wc}, {err}")
+    return result

freqtrade/plugins/pairlistmanager.py

@@ -10,6 +10,7 @@ from cachetools import TTLCache, cached
 from freqtrade.constants import ListPairsWithTimeframes
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.IPairList import IPairList
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.resolvers import PairListResolver
 
 
@@ -42,30 +43,29 @@ class PairListManager():
 
     @property
     def whitelist(self) -> List[str]:
-        """
+        """The current whitelist"""
-        Has the current whitelist
-        """
         return self._whitelist
 
     @property
     def blacklist(self) -> List[str]:
         """
-        Has the current blacklist
+        The current blacklist
         -> no need to overwrite in subclasses
         """
         return self._blacklist
 
+    @property
+    def expanded_blacklist(self) -> List[str]:
+        """The expanded blacklist (including wildcard expansion)"""
+        return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
+
     @property
     def name_list(self) -> List[str]:
-        """
+        """Get list of loaded Pairlist Handler names"""
-        Get list of loaded Pairlist Handler names
-        """
         return [p.name for p in self._pairlist_handlers]
 
     def short_desc(self) -> List[Dict]:
-        """
+        """List of short_desc for each Pairlist Handler"""
-        List of short_desc for each Pairlist Handler
-        """
         return [{p.name: p.short_desc()} for p in self._pairlist_handlers]
 
     @cached(TTLCache(maxsize=1, ttl=1800))
@@ -73,9 +73,7 @@ class PairListManager():
         return self._exchange.get_tickers()
 
     def refresh_pairlist(self) -> None:
-        """
+        """Run pairlist through all configured Pairlist Handlers."""
-        Run pairlist through all configured Pairlist Handlers.
-        """
         # Tickers should be cached to avoid calling the exchange on each call.
         tickers: Dict = {}
         if self._tickers_needed:
@@ -120,8 +118,13 @@ class PairListManager():
         :param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
         :return: pairlist - blacklisted pairs
         """
+        try:
+            blacklist = self.expanded_blacklist
+        except ValueError as err:
+            logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
+            return []
         for pair in deepcopy(pairlist):
-            if pair in self._blacklist:
+            if pair in blacklist:
                 logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...")
                 pairlist.remove(pair)
         return pairlist

freqtrade/rpc/rpc.py

@@ -20,6 +20,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
 from freqtrade.misc import shorten_date
 from freqtrade.persistence import PairLocks, Trade
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
 from freqtrade.state import State
 from freqtrade.strategy.interface import SellType
@@ -673,23 +674,23 @@ class RPC:
         """ Returns the currently active blacklist"""
         errors = {}
         if add:
-            stake_currency = self._freqtrade.config.get('stake_currency')
             for pair in add:
-                if self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency:
+                if pair not in self._freqtrade.pairlists.blacklist:
-                    if pair not in self._freqtrade.pairlists.blacklist:
+                    try:
+                        expand_pairlist([pair], self._freqtrade.exchange.get_markets().keys())
                         self._freqtrade.pairlists.blacklist.append(pair)
-                    else:
-                        errors[pair] = {
-                            'error_msg': f'Pair {pair} already in pairlist.'}
 
+                    except ValueError:
+                        errors[pair] = {
+                            'error_msg': f'Pair {pair} is not a valid wildcard.'}
                 else:
                     errors[pair] = {
-                        'error_msg': f"Pair {pair} does not match stake currency."
+                        'error_msg': f'Pair {pair} already in pairlist.'}
-                    }
 
         res = {'method': self._freqtrade.pairlists.name_list,
                'length': len(self._freqtrade.pairlists.blacklist),
                'blacklist': self._freqtrade.pairlists.blacklist,
+               'blacklist_expanded': self._freqtrade.pairlists.expanded_blacklist,
                'errors': errors,
                }
         return res

tests/plugins/test_pairlist.py

@@ -6,6 +6,7 @@ import pytest
 
 from freqtrade.constants import AVAILABLE_PAIRLISTS
 from freqtrade.exceptions import OperationalException
+from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.resolvers import PairListResolver
 from tests.conftest import get_patched_freqtradebot, log_has, log_has_re
@@ -155,6 +156,23 @@ def test_refresh_static_pairlist(mocker, markets, static_pl_conf):
     assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
 
 
+def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog):
+    static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC']
+    freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        exchange_has=MagicMock(return_value=True),
+        markets=PropertyMock(return_value=markets),
+    )
+    freqtrade.pairlists.refresh_pairlist()
+    # List ordered by BaseVolume
+    whitelist = []
+    # Ensure all except those in whitelist are removed
+    assert set(whitelist) == set(freqtrade.pairlists.whitelist)
+    assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
+    log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog)
+
+
 def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf):
 
     mocker.patch.multiple(
@@ -804,3 +822,34 @@ def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, o
     freqtrade.pairlists.refresh_pairlist()
     allowlist = freqtrade.pairlists.whitelist
     assert allowlist == allowlist_result
+
+
+@pytest.mark.parametrize('wildcardlist,pairs,expected', [
+    (['BTC/USDT'],
+     ['BTC/USDT'],
+     ['BTC/USDT']),
+    (['BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETH/USDT']),
+    (['BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT'], ['BTC/USDT']),  # Test one too many
+    (['.*/USDT'],
+     ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']),  # Wildcard simple
+    (['.*C/USDT'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']),  # Wildcard exclude one
+    (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
+     ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']),  # Wildcard exclude one
+    (['BTC/.*', 'ETH/.*'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'],
+     ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']),  # Wildcard exclude one
+    (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
+     ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
+     None),
+])
+def test_expand_pairlist(wildcardlist, pairs, expected):
+    if expected is None:
+        with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'):
+            expand_pairlist(wildcardlist, pairs)
+    else:
+        assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected)

tests/rpc/test_rpc.py

@@ -957,14 +957,24 @@ def test_rpc_blacklist(mocker, default_conf) -> None:
     assert isinstance(ret['errors'], dict)
     assert ret['errors']['ETH/BTC']['error_msg'] == 'Pair ETH/BTC already in pairlist.'
 
-    ret = rpc._rpc_blacklist(["ETH/ETH"])
+    ret = rpc._rpc_blacklist(["*/BTC"])
     assert 'StaticPairList' in ret['method']
     assert len(ret['blacklist']) == 3
     assert ret['blacklist'] == default_conf['exchange']['pair_blacklist']
     assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC']
+    assert ret['blacklist_expanded'] == ['ETH/BTC']
+    assert 'errors' in ret
+    assert isinstance(ret['errors'], dict)
+    assert ret['errors'] == {'*/BTC': {'error_msg': 'Pair */BTC is not a valid wildcard.'}}
+
+    ret = rpc._rpc_blacklist(["XRP/.*"])
+    assert 'StaticPairList' in ret['method']
+    assert len(ret['blacklist']) == 4
+    assert ret['blacklist'] == default_conf['exchange']['pair_blacklist']
+    assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC', 'XRP/.*']
+    assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC']
     assert 'errors' in ret
     assert isinstance(ret['errors'], dict)
-    assert ret['errors']['ETH/ETH']['error_msg'] == 'Pair ETH/ETH does not match stake currency.'
 
 
 def test_rpc_edge_disabled(mocker, default_conf) -> None:

tests/rpc/test_rpc_apiserver.py

@@ -730,7 +730,9 @@ def test_api_blacklist(botclient, mocker):
 
     rc = client_get(client, f"{BASE_URI}/blacklist")
     assert_response(rc)
+    # DOGE and HOT are not in the markets mock!
     assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"],
+                       "blacklist_expanded": [],
                        "length": 2,
                        "method": ["StaticPairList"],
                        "errors": {},
@@ -741,11 +743,22 @@ def test_api_blacklist(botclient, mocker):
                      data='{"blacklist": ["ETH/BTC"]}')
     assert_response(rc)
     assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"],
+                       "blacklist_expanded": ["ETH/BTC"],
                        "length": 3,
                        "method": ["StaticPairList"],
                        "errors": {},
                        }
 
+    rc = client_post(client, f"{BASE_URI}/blacklist",
+                     data='{"blacklist": ["XRP/.*"]}')
+    assert_response(rc)
+    assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"],
+                       "blacklist_expanded": ["ETH/BTC", "XRP/BTC"],
+                       "length": 4,
+                       "method": ["StaticPairList"],
+                       "errors": {},
+                       }
+
 
 def test_api_whitelist(botclient):
     ftbot, client = botclient

tests/rpc/test_rpc_telegram.py

@@ -1011,15 +1011,13 @@ def test_blacklist_static(default_conf, update, mocker) -> None:
 
     msg_mock.reset_mock()
     context = MagicMock()
-    context.args = ["ETH/ETH"]
+    context.args = ["XRP/.*"]
     telegram._blacklist(update=update, context=context)
-    assert msg_mock.call_count == 2
+    assert msg_mock.call_count == 1
-    assert ("Error adding `ETH/ETH` to blacklist: `Pair ETH/ETH does not match stake currency.`"
-            in msg_mock.call_args_list[0][0][0])
 
-    assert ("Blacklist contains 3 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC`"
+    assert ("Blacklist contains 4 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC, XRP/.*`"
-            in msg_mock.call_args_list[1][0][0])
+            in msg_mock.call_args_list[0][0][0])
-    assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC"]
+    assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"]
 
 
 def test_telegram_logs(default_conf, update, mocker) -> None: