adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3313 from hroff-1902/refactor-pairlists 

Cleanup in pairlists
This commit is contained in:
Matthias 2020-05-17 20:15:39 +02:00 committed by GitHub
parent 15ef6df950 16622bbfad
commit 9d63fada24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 133 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
freqtrade/pairlist/IPairList.py
View File

@ -1,9 +1,6 @@
"""
Static List provider
Provides lists as configured in config.json
"""
PairList base class
"""
import logging
from abc import ABC, abstractmethod, abstractproperty
from copy import deepcopy
@ -13,6 +10,7 @@ from cachetools import TTLCache, cached
from freqtrade.exchange import market_is_active
logger = logging.getLogger(__name__)

28
freqtrade/pairlist/PrecisionFilter.py
View File

@ -1,14 +1,26 @@
"""
Precision pair list filter
"""
import logging
from copy import deepcopy
from typing import Dict, List
from typing import Any, Dict, List
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class PrecisionFilter(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
# Precalculate sanitized stoploss value to avoid recalculation for every pair
self._stoploss = 1 - abs(self._config['stoploss'])
@property
def needstickers(self) -> bool:
"""
@ -31,34 +43,32 @@ class PrecisionFilter(IPairList):
:param ticker: ticker dict as returned from ccxt.load_markets()
:param stoploss: stoploss value as set in the configuration
(already cleaned to be 1 - stoploss)
:return: True if the pair can stay, false if it should be removed
:return: True if the pair can stay, False if it should be removed
"""
stop_price = ticker['ask'] * stoploss
# Adjust stop-prices to precision
sp = self._exchange.price_to_precision(ticker["symbol"], stop_price)
stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99)
logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}")
if sp <= stop_gap_price:
self.log_on_refresh(logger.info,
f"Removed {ticker['symbol']} from whitelist, "
f"because stop price {sp} would be <= stop limit {stop_gap_price}")
return False
return True
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlists and assigns and returns them again.
"""
stoploss = self._config.get('stoploss')
if stoploss is not None:
# Precalculate sanitized stoploss value to avoid recalculation for every pair
stoploss = 1 - abs(stoploss)
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
ticker = tickers.get(p)
# Filter out assets which would not allow setting a stoploss
if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)):
if not self._validate_precision_filter(tickers[p], self._stoploss):
pairlist.remove(p)
continue
return pairlist

25
freqtrade/pairlist/PriceFilter.py
View File

@ -1,9 +1,13 @@
"""
Price pair list filter
"""
import logging
from copy import deepcopy
from typing import Any, Dict, List
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
@ -38,14 +42,12 @@ class PriceFilter(IPairList):
:return: True if the pair can stay, false if it should be removed
"""
if ticker['last'] is None:
self.log_on_refresh(logger.info,
f"Removed {ticker['symbol']} from whitelist, because "
"ticker['last'] is empty (Usually no trade in the last 24h).")
return False
compare = ticker['last'] + self._exchange.price_get_one_pip(ticker['symbol'],
ticker['last'])
changeperc = (compare - ticker['last']) / ticker['last']
compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last'])
changeperc = compare / ticker['last']
if changeperc > self._low_price_ratio:
self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because 1 unit is {changeperc * 100:.3f}%")
@ -60,14 +62,11 @@ class PriceFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
ticker = tickers.get(p)
if not ticker:
pairlist.remove(p)
# Filter out assets which would not allow setting a stoploss
if self._low_price_ratio and not self._validate_ticker_lowprice(ticker):
pairlist.remove(p)
if self._low_price_ratio:
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
# Filter out assets which would not allow setting a stoploss
if not self._validate_ticker_lowprice(tickers[p]):
pairlist.remove(p)
return pairlist

37
freqtrade/pairlist/SpreadFilter.py
View File

@ -1,9 +1,13 @@
"""
Spread pair list filter
"""
import logging
from copy import deepcopy
from typing import Dict, List
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
@ -31,8 +35,24 @@ class SpreadFilter(IPairList):
return (f"{self.name} - Filtering pairs with ask/bid diff above "
f"{self._max_spread_ratio * 100}%.")
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
def _validate_spread(self, ticker: dict) -> bool:
"""
Validate spread for the ticker
:param ticker: ticker dict as returned from ccxt.load_markets()
:return: True if the pair can stay, False if it should be removed
"""
if 'bid' in ticker and 'ask' in ticker:
spread = 1 - ticker['bid'] / ticker['ask']
if spread > self._max_spread_ratio:
self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because spread {spread * 100:.3f}% >"
f"{self._max_spread_ratio * 100}%")
return False
else:
return True
return False
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
@ -41,19 +61,10 @@ class SpreadFilter(IPairList):
:return: new whitelist
"""
# Copy list since we're modifying this list
spread = None
for p in deepcopy(pairlist):
ticker = tickers.get(p)
assert ticker is not None
if 'bid' in ticker and 'ask' in ticker:
spread = 1 - ticker['bid'] / ticker['ask']
if not ticker or spread > self._max_spread_ratio:
self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, "
f"because spread {spread * 100:.3f}% >"
f"{self._max_spread_ratio * 100}%")
pairlist.remove(p)
else:
ticker = tickers[p]
# Filter out assets
if not self._validate_spread(ticker):
pairlist.remove(p)
return pairlist

8
freqtrade/pairlist/StaticPairList.py
View File

@ -1,14 +1,14 @@
"""
Static List provider
Static Pair List provider
Provides lists as configured in config.json
"""
Provides pair white list as it configured in config
"""
import logging
from typing import Dict, List
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)

41
freqtrade/pairlist/VolumePairList.py
View File

@ -1,9 +1,8 @@
"""
Volume PairList provider
Provides lists as configured in config.json
"""
Provides dynamic pair list based on trade volumes
"""
import logging
from datetime import datetime
from typing import Any, Dict, List
@ -11,8 +10,10 @@ from typing import Any, Dict, List
from freqtrade.exceptions import OperationalException
from freqtrade.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume']
@ -26,6 +27,8 @@ class VolumePairList(IPairList):
raise OperationalException(
f'`number_assets` not specified. Please check your configuration '
'for "pairlist.config.number_assets"')
self._stake_currency = config['stake_currency']
self._number_pairs = self._pairlistconfig['number_assets']
self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
self._min_value = self._pairlistconfig.get('min_value', 0)
@ -36,9 +39,11 @@ class VolumePairList(IPairList):
'Exchange does not support dynamic whitelist.'
'Please edit your config and restart the bot'
)
if not self._validate_keys(self._sort_key):
raise OperationalException(
f'key {self._sort_key} not in {SORT_VALUES}')
if self._sort_key != 'quoteVolume':
logger.warning(
"DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated."
@ -76,42 +81,42 @@ class VolumePairList(IPairList):
(self._last_refresh + self.refresh_period < datetime.now().timestamp())):
self._last_refresh = int(datetime.now().timestamp())
pairs = self._gen_pair_whitelist(pairlist, tickers,
self._config['stake_currency'],
self._sort_key, self._min_value)
pairs = self._gen_pair_whitelist(pairlist, tickers)
else:
pairs = pairlist
self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}")
return pairs
def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict,
base_currency: str, key: str, min_val: int) -> List[str]:
def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Updates the whitelist with with a dynamically generated list
:param base_currency: base currency as str
:param key: sort key (defaults to 'quoteVolume')
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers()).
:return: List of pairs
"""
if self._pairlist_pos == 0:
# If VolumePairList is the first in the list, use fresh pairlist
# Check if pair quote currency equals to the stake currency.
filtered_tickers = [v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == base_currency
and v[key] is not None)]
filtered_tickers = [
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and v[self._sort_key] is not None)]
else:
# If other pairlist is in front, use the incomming pairlist.
# If other pairlist is in front, use the incoming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
if min_val > 0:
filtered_tickers = list(filter(lambda t: t[key] > min_val, filtered_tickers))
if self._min_value > 0:
filtered_tickers = [
v for v in filtered_tickers if v[self._sort_key] > self._min_value]
sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key])
sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
pairs = self._verify_blacklist(pairs, aswarning=False)
# Limit to X number of pairs
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]
return pairs

33
freqtrade/pairlist/pairlistmanager.py
View File

@ -1,10 +1,8 @@
"""
Static List provider
Provides lists as configured in config.json
"""
PairList manager class
"""
import logging
from copy import deepcopy
from typing import Dict, List
from cachetools import TTLCache, cached
@ -13,6 +11,7 @@ from freqtrade.exceptions import OperationalException
from freqtrade.pairlist.IPairList import IPairList
from freqtrade.resolvers import PairListResolver
logger = logging.getLogger(__name__)
@ -78,19 +77,33 @@ class PairListManager():
"""
Run pairlist through all configured pairlists.
"""
pairlist = self._whitelist.copy()
# tickers should be cached to avoid calling the exchange on each call.
# Tickers should be cached to avoid calling the exchange on each call.
tickers: Dict = {}
if self._tickers_needed:
tickers = self._get_cached_tickers()
# Adjust whitelist if filters are using tickers
pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
# Process all pairlists in chain
for pl in self._pairlists:
pairlist = pl.filter_pairlist(pairlist, tickers)
# Validation against blacklist happens after the pairlists to ensure blacklist is respected.
# Validation against blacklist happens after the pairlists to ensure
# blacklist is respected.
pairlist = IPairList.verify_blacklist(pairlist, self.blacklist, True)
self._whitelist = pairlist
def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]:
"""
Prepare sanitized pairlist for Pairlist Filters that use tickers data - remove
pairs that do not have ticker available
"""
if self._tickers_needed:
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
if p not in tickers:
pairlist.remove(p)
return pairlist

50
tests/pairlist/test_pairlist.py
View File

@ -69,44 +69,44 @@ def test_log_on_refresh(mocker, static_pl_conf, markets, tickers):
def test_load_pairlist_noexist(mocker, markets, default_conf):
bot = get_patched_freqtradebot(mocker, default_conf)
freqtrade = get_patched_freqtradebot(mocker, default_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
plm = PairListManager(bot.exchange, default_conf)
plm = PairListManager(freqtrade.exchange, default_conf)
with pytest.raises(OperationalException,
match=r"Impossible to load Pairlist 'NonexistingPairList'. "
r"This class does not exist or contains Python code errors."):
PairListResolver.load_pairlist('NonexistingPairList', bot.exchange, plm,
PairListResolver.load_pairlist('NonexistingPairList', freqtrade.exchange, plm,
default_conf, {}, 1)
def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf):
freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf)
freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
freqtradebot.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist()
# List ordered by BaseVolume
whitelist = ['ETH/BTC', 'TKN/BTC']
# Ensure all except those in whitelist are removed
assert set(whitelist) == set(freqtradebot.pairlists.whitelist)
assert set(whitelist) == set(freqtrade.pairlists.whitelist)
# Ensure config dict hasn't been changed
assert (static_pl_conf['exchange']['pair_whitelist'] ==
freqtradebot.config['exchange']['pair_whitelist'])
freqtrade.config['exchange']['pair_whitelist'])
def test_refresh_static_pairlist(mocker, markets, static_pl_conf):
freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf)
freqtrade = get_patched_freqtradebot(mocker, static_pl_conf)
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
exchange_has=MagicMock(return_value=True),
markets=PropertyMock(return_value=markets),
)
freqtradebot.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist()
# List ordered by BaseVolume
whitelist = ['ETH/BTC', 'TKN/BTC']
# Ensure all except those in whitelist are removed
assert set(whitelist) == set(freqtradebot.pairlists.whitelist)
assert static_pl_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist
assert set(whitelist) == set(freqtrade.pairlists.whitelist)
assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist
def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf):
@ -116,7 +116,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co
get_tickers=tickers,
exchange_has=MagicMock(return_value=True),
)
bot = get_patched_freqtradebot(mocker, whitelist_conf)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
# Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture
mocker.patch.multiple(
'freqtrade.exchange.Exchange',
@ -124,9 +124,9 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co
)
# argument: use the whitelist dynamically by exchange-volume
whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']
bot.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist()
assert whitelist == bot.pairlists.whitelist
assert whitelist == freqtrade.pairlists.whitelist
whitelist_conf['pairlists'] = [{'method': 'VolumePairList',
'config': {}
@ -136,7 +136,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co
with pytest.raises(OperationalException,
match=r'`number_assets` not specified. Please check your configuration '
r'for "pairlist.config.number_assets"'):
PairListManager(bot.exchange, whitelist_conf)
PairListManager(freqtrade.exchange, whitelist_conf)
def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
@ -144,13 +144,13 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf):
'freqtrade.exchange.Exchange',
exchange_has=MagicMock(return_value=True),
)
freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf)
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_empty))
# argument: use the whitelist dynamically by exchange-volume
whitelist = []
whitelist_conf['exchange']['pair_whitelist'] = []
freqtradebot.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist()
pairslist = whitelist_conf['exchange']['pair_whitelist']
assert set(whitelist) == set(pairslist)
@ -206,6 +206,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
pairlists, base_currency, whitelist_result,
caplog) -> None:
whitelist_conf['pairlists'] = pairlists
whitelist_conf['stake_currency'] = base_currency
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
@ -215,7 +216,6 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t
markets=PropertyMock(return_value=shitcoinmarkets),
)
freqtrade.config['stake_currency'] = base_currency
freqtrade.pairlists.refresh_pairlist()
whitelist = freqtrade.pairlists.whitelist
@ -312,18 +312,18 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers):
exchange_has=MagicMock(return_value=True),
get_tickers=tickers
)
bot = get_patched_freqtradebot(mocker, whitelist_conf)
assert bot.pairlists._pairlists[0]._last_refresh == 0
freqtrade = get_patched_freqtradebot(mocker, whitelist_conf)
assert freqtrade.pairlists._pairlists[0]._last_refresh == 0
assert tickers.call_count == 0
bot.pairlists.refresh_pairlist()
freqtrade.pairlists.refresh_pairlist()
assert tickers.call_count == 1
assert bot.pairlists._pairlists[0]._last_refresh != 0
lrf = bot.pairlists._pairlists[0]._last_refresh
bot.pairlists.refresh_pairlist()
assert freqtrade.pairlists._pairlists[0]._last_refresh != 0
lrf = freqtrade.pairlists._pairlists[0]._last_refresh
freqtrade.pairlists.refresh_pairlist()
assert tickers.call_count == 1
# Time should not be updated.
assert bot.pairlists._pairlists[0]._last_refresh == lrf
assert freqtrade.pairlists._pairlists[0]._last_refresh == lrf
def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog):