diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 732dfa5bb..8919c4e3d 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -10,6 +10,14 @@ If multiple Pairlist Handlers are used, they are chained and a combination of al Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. +### Pair blacklist + +The pair blacklist (configured via `exchange.pair_blacklist` in the configuration) disallows certain pairs from trading. +This can be as simple as excluding `DOGE/BTC` - which will remove exactly this pair. + +The pair-blacklist does also support wildcards (in regex-style) - so `BNB/.*` will exclude ALL pairs that start with BNB. +You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged tokens (check Pair naming conventions for your exchange!) + ### Available Pairlist Handlers * [`StaticPairList`](#static-pair-list) (default, if not configured differently) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6f495e605..11a0ef8e6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -208,7 +208,7 @@ class Exchange: return self._api.precisionMode def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None, - pairs_only: bool = False, active_only: bool = False) -> Dict: + pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]: """ Return exchange ccxt markets, filtered out by base currency and quote currency if this was requested in parameters. diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py new file mode 100644 index 000000000..3352777f0 --- /dev/null +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -0,0 +1,23 @@ +import re +from typing import List + + +def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]: + """ + Expand pairlist potentially containing wildcards based on available markets. + This will implicitly filter all pairs in the wildcard-list which are not in available_pairs. + :param wildcardpl: List of Pairlists, which may contain regex + :param available_pairs: List of all available pairs (`exchange.get_markets().keys()`) + :return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs. + :raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`) + """ + result = [] + for pair_wc in wildcardpl: + try: + comp = re.compile(pair_wc) + result += [ + pair for pair in available_pairs if re.match(comp, pair) + ] + except re.error as err: + raise ValueError(f"Wildcard error in {pair_wc}, {err}") + return result diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index b71f02898..ad7b46cb8 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -10,6 +10,7 @@ from cachetools import TTLCache, cached from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import PairListResolver @@ -42,30 +43,29 @@ class PairListManager(): @property def whitelist(self) -> List[str]: - """ - Has the current whitelist - """ + """The current whitelist""" return self._whitelist @property def blacklist(self) -> List[str]: """ - Has the current blacklist + The current blacklist -> no need to overwrite in subclasses """ return self._blacklist + @property + def expanded_blacklist(self) -> List[str]: + """The expanded blacklist (including wildcard expansion)""" + return expand_pairlist(self._blacklist, self._exchange.get_markets().keys()) + @property def name_list(self) -> List[str]: - """ - Get list of loaded Pairlist Handler names - """ + """Get list of loaded Pairlist Handler names""" return [p.name for p in self._pairlist_handlers] def short_desc(self) -> List[Dict]: - """ - List of short_desc for each Pairlist Handler - """ + """List of short_desc for each Pairlist Handler""" return [{p.name: p.short_desc()} for p in self._pairlist_handlers] @cached(TTLCache(maxsize=1, ttl=1800)) @@ -73,9 +73,7 @@ class PairListManager(): return self._exchange.get_tickers() def refresh_pairlist(self) -> None: - """ - Run pairlist through all configured Pairlist Handlers. - """ + """Run pairlist through all configured Pairlist Handlers.""" # Tickers should be cached to avoid calling the exchange on each call. tickers: Dict = {} if self._tickers_needed: @@ -120,8 +118,13 @@ class PairListManager(): :param logmethod: Function that'll be called, `logger.info` or `logger.warning`. :return: pairlist - blacklisted pairs """ + try: + blacklist = self.expanded_blacklist + except ValueError as err: + logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") + return [] for pair in deepcopy(pairlist): - if pair in self._blacklist: + if pair in blacklist: logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 42ab76622..70a99a186 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -20,6 +20,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler from freqtrade.misc import shorten_date from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State from freqtrade.strategy.interface import SellType @@ -673,23 +674,23 @@ class RPC: """ Returns the currently active blacklist""" errors = {} if add: - stake_currency = self._freqtrade.config.get('stake_currency') for pair in add: - if self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: - if pair not in self._freqtrade.pairlists.blacklist: + if pair not in self._freqtrade.pairlists.blacklist: + try: + expand_pairlist([pair], self._freqtrade.exchange.get_markets().keys()) self._freqtrade.pairlists.blacklist.append(pair) - else: - errors[pair] = { - 'error_msg': f'Pair {pair} already in pairlist.'} + except ValueError: + errors[pair] = { + 'error_msg': f'Pair {pair} is not a valid wildcard.'} else: errors[pair] = { - 'error_msg': f"Pair {pair} does not match stake currency." - } + 'error_msg': f'Pair {pair} already in pairlist.'} res = {'method': self._freqtrade.pairlists.name_list, 'length': len(self._freqtrade.pairlists.blacklist), 'blacklist': self._freqtrade.pairlists.blacklist, + 'blacklist_expanded': self._freqtrade.pairlists.expanded_blacklist, 'errors': errors, } return res diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1795fc27f..d822f8319 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -6,6 +6,7 @@ import pytest from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from tests.conftest import get_patched_freqtradebot, log_has, log_has_re @@ -155,6 +156,23 @@ def test_refresh_static_pairlist(mocker, markets, static_pl_conf): assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist +def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): + static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC'] + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) + freqtrade.pairlists.refresh_pairlist() + # List ordered by BaseVolume + whitelist = [] + # Ensure all except those in whitelist are removed + assert set(whitelist) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist + log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog) + + def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): mocker.patch.multiple( @@ -804,3 +822,34 @@ def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, o freqtrade.pairlists.refresh_pairlist() allowlist = freqtrade.pairlists.whitelist assert allowlist == allowlist_result + + +@pytest.mark.parametrize('wildcardlist,pairs,expected', [ + (['BTC/USDT'], + ['BTC/USDT'], + ['BTC/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETH/USDT']), + (['BTC/USDT', 'ETH/USDT'], + ['BTC/USDT'], ['BTC/USDT']), # Test one too many + (['.*/USDT'], + ['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']), # Wildcard simple + (['.*C/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']), # Wildcard exclude one + (['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + ['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']), # Wildcard exclude one + (['BTC/.*', 'ETH/.*'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'], + ['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']), # Wildcard exclude one + (['*UP/USDT', 'BTC/USDT', 'ETH/USDT'], + ['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'], + None), +]) +def test_expand_pairlist(wildcardlist, pairs, expected): + if expected is None: + with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'): + expand_pairlist(wildcardlist, pairs) + else: + assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 19788c067..8ec356d54 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -957,14 +957,24 @@ def test_rpc_blacklist(mocker, default_conf) -> None: assert isinstance(ret['errors'], dict) assert ret['errors']['ETH/BTC']['error_msg'] == 'Pair ETH/BTC already in pairlist.' - ret = rpc._rpc_blacklist(["ETH/ETH"]) + ret = rpc._rpc_blacklist(["*/BTC"]) assert 'StaticPairList' in ret['method'] assert len(ret['blacklist']) == 3 assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC'] + assert ret['blacklist_expanded'] == ['ETH/BTC'] + assert 'errors' in ret + assert isinstance(ret['errors'], dict) + assert ret['errors'] == {'*/BTC': {'error_msg': 'Pair */BTC is not a valid wildcard.'}} + + ret = rpc._rpc_blacklist(["XRP/.*"]) + assert 'StaticPairList' in ret['method'] + assert len(ret['blacklist']) == 4 + assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] + assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC', 'XRP/.*'] + assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC'] assert 'errors' in ret assert isinstance(ret['errors'], dict) - assert ret['errors']['ETH/ETH']['error_msg'] == 'Pair ETH/ETH does not match stake currency.' def test_rpc_edge_disabled(mocker, default_conf) -> None: diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 5e608fb25..8da4ebfe7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -730,7 +730,9 @@ def test_api_blacklist(botclient, mocker): rc = client_get(client, f"{BASE_URI}/blacklist") assert_response(rc) + # DOGE and HOT are not in the markets mock! assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"], + "blacklist_expanded": [], "length": 2, "method": ["StaticPairList"], "errors": {}, @@ -741,11 +743,22 @@ def test_api_blacklist(botclient, mocker): data='{"blacklist": ["ETH/BTC"]}') assert_response(rc) assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], + "blacklist_expanded": ["ETH/BTC"], "length": 3, "method": ["StaticPairList"], "errors": {}, } + rc = client_post(client, f"{BASE_URI}/blacklist", + data='{"blacklist": ["XRP/.*"]}') + assert_response(rc) + assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"], + "blacklist_expanded": ["ETH/BTC", "XRP/BTC"], + "length": 4, + "method": ["StaticPairList"], + "errors": {}, + } + def test_api_whitelist(botclient): ftbot, client = botclient diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 97b9e5e7c..a21a19e3a 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1011,15 +1011,13 @@ def test_blacklist_static(default_conf, update, mocker) -> None: msg_mock.reset_mock() context = MagicMock() - context.args = ["ETH/ETH"] + context.args = ["XRP/.*"] telegram._blacklist(update=update, context=context) - assert msg_mock.call_count == 2 - assert ("Error adding `ETH/ETH` to blacklist: `Pair ETH/ETH does not match stake currency.`" - in msg_mock.call_args_list[0][0][0]) + assert msg_mock.call_count == 1 - assert ("Blacklist contains 3 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC`" - in msg_mock.call_args_list[1][0][0]) - assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC"] + assert ("Blacklist contains 4 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC, XRP/.*`" + in msg_mock.call_args_list[0][0][0]) + assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"] def test_telegram_logs(default_conf, update, mocker) -> None: