adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4133 from freqtrade/dynamic_pairlist

Browse Source 
Wildcard based blacklist
This commit is contained in:
Matthias 2020-12-31 10:02:30 +01:00 committed by GitHub
commit 12de29dd3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 137 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/includes/pairlists.md
View File

@ -10,6 +10,14 @@ If multiple Pairlist Handlers are used, they are chained and a combination of al
Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist.
### Pair blacklist
The pair blacklist (configured via `exchange.pair_blacklist` in the configuration) disallows certain pairs from trading.
This can be as simple as excluding `DOGE/BTC` - which will remove exactly this pair.
The pair-blacklist does also support wildcards (in regex-style) - so `BNB/.*` will exclude ALL pairs that start with BNB.
You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged tokens (check Pair naming conventions for your exchange!)
### Available Pairlist Handlers
* [`StaticPairList`](#static-pair-list) (default, if not configured differently)

2
freqtrade/exchange/exchange.py
View File

@ -208,7 +208,7 @@ class Exchange:
return self._api.precisionMode
def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None,
pairs_only: bool = False, active_only: bool = False) -> Dict:
pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]:
"""
Return exchange ccxt markets, filtered out by base currency and quote currency
if this was requested in parameters.

23
freqtrade/plugins/pairlist/pairlist_helpers.py Normal file
View File

@ -0,0 +1,23 @@
import re
from typing import List
def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]:
"""
Expand pairlist potentially containing wildcards based on available markets.
This will implicitly filter all pairs in the wildcard-list which are not in available_pairs.
:param wildcardpl: List of Pairlists, which may contain regex
:param available_pairs: List of all available pairs (`exchange.get_markets().keys()`)
:return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs.
:raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`)
"""
result = []
for pair_wc in wildcardpl:
try:
comp = re.compile(pair_wc)
result += [
pair for pair in available_pairs if re.match(comp, pair)
]
except re.error as err:
raise ValueError(f"Wildcard error in {pair_wc}, {err}")
return result

31
freqtrade/plugins/pairlistmanager.py
View File

@ -10,6 +10,7 @@ from cachetools import TTLCache, cached
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import PairListResolver
@ -42,30 +43,29 @@ class PairListManager():
@property
def whitelist(self) -> List[str]:
"""
Has the current whitelist
"""
"""The current whitelist"""
return self._whitelist
@property
def blacklist(self) -> List[str]:
"""
Has the current blacklist
The current blacklist
-> no need to overwrite in subclasses
"""
return self._blacklist
@property
def expanded_blacklist(self) -> List[str]:
"""The expanded blacklist (including wildcard expansion)"""
return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
@property
def name_list(self) -> List[str]:
"""
Get list of loaded Pairlist Handler names
"""
"""Get list of loaded Pairlist Handler names"""
return [p.name for p in self._pairlist_handlers]
def short_desc(self) -> List[Dict]:
"""
List of short_desc for each Pairlist Handler
"""
"""List of short_desc for each Pairlist Handler"""
return [{p.name: p.short_desc()} for p in self._pairlist_handlers]
@cached(TTLCache(maxsize=1, ttl=1800))
@ -73,9 +73,7 @@ class PairListManager():
return self._exchange.get_tickers()
def refresh_pairlist(self) -> None:
"""
Run pairlist through all configured Pairlist Handlers.
"""
"""Run pairlist through all configured Pairlist Handlers."""
# Tickers should be cached to avoid calling the exchange on each call.
tickers: Dict = {}
if self._tickers_needed:
@ -120,8 +118,13 @@ class PairListManager():
:param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
:return: pairlist - blacklisted pairs
"""
try:
blacklist = self.expanded_blacklist
except ValueError as err:
logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
return []
for pair in deepcopy(pairlist):
if pair in self._blacklist:
if pair in blacklist:
logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...")
pairlist.remove(pair)
return pairlist

15
freqtrade/rpc/rpc.py
View File

@ -20,6 +20,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
from freqtrade.misc import shorten_date
from freqtrade.persistence import PairLocks, Trade
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
from freqtrade.state import State
from freqtrade.strategy.interface import SellType
@ -673,23 +674,23 @@ class RPC:
""" Returns the currently active blacklist"""
errors = {}
if add:
stake_currency = self._freqtrade.config.get('stake_currency')
for pair in add:
if self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency:
if pair not in self._freqtrade.pairlists.blacklist:
try:
expand_pairlist([pair], self._freqtrade.exchange.get_markets().keys())
self._freqtrade.pairlists.blacklist.append(pair)
except ValueError:
errors[pair] = {
'error_msg': f'Pair {pair} is not a valid wildcard.'}
else:
errors[pair] = {
'error_msg': f'Pair {pair} already in pairlist.'}
else:
errors[pair] = {
'error_msg': f"Pair {pair} does not match stake currency."
}
res = {'method': self._freqtrade.pairlists.name_list,
'length': len(self._freqtrade.pairlists.blacklist),
'blacklist': self._freqtrade.pairlists.blacklist,
'blacklist_expanded': self._freqtrade.pairlists.expanded_blacklist,
'errors': errors,
}
return res

49
tests/plugins/test_pairlist.py
View File

@ -6,6 +6,7 @@ import pytest
from freqtrade.constants import AVAILABLE_PAIRLISTS
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.plugins.pairlistmanager import PairListManager
from freqtrade.resolvers import PairListResolver
from tests.conftest import get_patched_freqtradebot, log_has, log_has_re
@ -155,6 +156,23 @@ def test_refresh_static_pairlist(mocker, markets, static_pl_conf):
assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog):
static_pl_conf['exchange']['pair_blacklist'] = ['*/BTC']
freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
exchange_has=MagicMock(return_value=True),
markets=PropertyMock(return_value=markets),
)
freqtrade.pairlists.refresh_pairlist()
# List ordered by BaseVolume
whitelist = []
# Ensure all except those in whitelist are removed
assert set(whitelist) == set(freqtrade.pairlists.whitelist)
assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog)
def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf):
mocker.patch.multiple(
@ -804,3 +822,34 @@ def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, o
freqtrade.pairlists.refresh_pairlist()
allowlist = freqtrade.pairlists.whitelist
assert allowlist == allowlist_result
@pytest.mark.parametrize('wildcardlist,pairs,expected', [
(['BTC/USDT'],
['BTC/USDT'],
['BTC/USDT']),
(['BTC/USDT', 'ETH/USDT'],
['BTC/USDT', 'ETH/USDT'],
['BTC/USDT', 'ETH/USDT']),
(['BTC/USDT', 'ETH/USDT'],
['BTC/USDT'], ['BTC/USDT']), # Test one too many
(['.*/USDT'],
['BTC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETH/USDT']), # Wildcard simple
(['.*C/USDT'],
['BTC/USDT', 'ETC/USDT', 'ETH/USDT'], ['BTC/USDT', 'ETC/USDT']), # Wildcard exclude one
(['.*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
['BTC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT']), # Wildcard exclude one
(['BTC/.*', 'ETH/.*'],
['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP'],
['BTC/USDT', 'ETH/USDT', 'BTC/USD', 'ETH/EUR', 'BTC/GBP']), # Wildcard exclude one
(['*UP/USDT', 'BTC/USDT', 'ETH/USDT'],
['BTC/USDT', 'ETC/USDT', 'ETH/USDT', 'BTCUP/USDT', 'XRPUP/USDT', 'XRPDOWN/USDT'],
None),
])
def test_expand_pairlist(wildcardlist, pairs, expected):
if expected is None:
with pytest.raises(ValueError, match=r'Wildcard error in \*UP/USDT,'):
expand_pairlist(wildcardlist, pairs)
else:
assert sorted(expand_pairlist(wildcardlist, pairs)) == sorted(expected)

14
tests/rpc/test_rpc.py
View File

@ -957,14 +957,24 @@ def test_rpc_blacklist(mocker, default_conf) -> None:
assert isinstance(ret['errors'], dict)
assert ret['errors']['ETH/BTC']['error_msg'] == 'Pair ETH/BTC already in pairlist.'
ret = rpc._rpc_blacklist(["ETH/ETH"])
ret = rpc._rpc_blacklist(["*/BTC"])
assert 'StaticPairList' in ret['method']
assert len(ret['blacklist']) == 3
assert ret['blacklist'] == default_conf['exchange']['pair_blacklist']
assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC']
assert ret['blacklist_expanded'] == ['ETH/BTC']
assert 'errors' in ret
assert isinstance(ret['errors'], dict)
assert ret['errors'] == {'*/BTC': {'error_msg': 'Pair */BTC is not a valid wildcard.'}}
ret = rpc._rpc_blacklist(["XRP/.*"])
assert 'StaticPairList' in ret['method']
assert len(ret['blacklist']) == 4
assert ret['blacklist'] == default_conf['exchange']['pair_blacklist']
assert ret['blacklist'] == ['DOGE/BTC', 'HOT/BTC', 'ETH/BTC', 'XRP/.*']
assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC']
assert 'errors' in ret
assert isinstance(ret['errors'], dict)
assert ret['errors']['ETH/ETH']['error_msg'] == 'Pair ETH/ETH does not match stake currency.'
def test_rpc_edge_disabled(mocker, default_conf) -> None:

13
tests/rpc/test_rpc_apiserver.py
View File

@ -730,7 +730,9 @@ def test_api_blacklist(botclient, mocker):
rc = client_get(client, f"{BASE_URI}/blacklist")
assert_response(rc)
# DOGE and HOT are not in the markets mock!
assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC"],
"blacklist_expanded": [],
"length": 2,
"method": ["StaticPairList"],
"errors": {},
@ -741,11 +743,22 @@ def test_api_blacklist(botclient, mocker):
data='{"blacklist": ["ETH/BTC"]}')
assert_response(rc)
assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"],
"blacklist_expanded": ["ETH/BTC"],
"length": 3,
"method": ["StaticPairList"],
"errors": {},
}
rc = client_post(client, f"{BASE_URI}/blacklist",
data='{"blacklist": ["XRP/.*"]}')
assert_response(rc)
assert rc.json == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"],
"blacklist_expanded": ["ETH/BTC", "XRP/BTC"],
"length": 4,
"method": ["StaticPairList"],
"errors": {},
}
def test_api_whitelist(botclient):
ftbot, client = botclient

12
tests/rpc/test_rpc_telegram.py
View File

@ -1011,15 +1011,13 @@ def test_blacklist_static(default_conf, update, mocker) -> None:
msg_mock.reset_mock()
context = MagicMock()
context.args = ["ETH/ETH"]
context.args = ["XRP/.*"]
telegram._blacklist(update=update, context=context)
assert msg_mock.call_count == 2
assert ("Error adding `ETH/ETH` to blacklist: `Pair ETH/ETH does not match stake currency.`"
in msg_mock.call_args_list[0][0][0])
assert msg_mock.call_count == 1
assert ("Blacklist contains 3 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC`"
in msg_mock.call_args_list[1][0][0])
assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC"]
assert ("Blacklist contains 4 pairs\n`DOGE/BTC, HOT/BTC, ETH/BTC, XRP/.*`"
in msg_mock.call_args_list[0][0][0])
assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"]
def test_telegram_logs(default_conf, update, mocker) -> None: