diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index e089e546c..e2eb364bc 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -1,9 +1,6 @@ """ -Static List provider - -Provides lists as configured in config.json - - """ +PairList base class +""" import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy @@ -13,6 +10,7 @@ from cachetools import TTLCache, cached from freqtrade.exchange import market_is_active + logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index 2a2ba46b7..6bd9c594e 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -1,14 +1,26 @@ +""" +Precision pair list filter +""" import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) class PrecisionFilter(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + # Precalculate sanitized stoploss value to avoid recalculation for every pair + self._stoploss = 1 - abs(self._config['stoploss']) + @property def needstickers(self) -> bool: """ @@ -31,34 +43,32 @@ class PrecisionFilter(IPairList): :param ticker: ticker dict as returned from ccxt.load_markets() :param stoploss: stoploss value as set in the configuration (already cleaned to be 1 - stoploss) - :return: True if the pair can stay, false if it should be removed + :return: True if the pair can stay, False if it should be removed """ stop_price = ticker['ask'] * stoploss + # Adjust stop-prices to precision sp = self._exchange.price_to_precision(ticker["symbol"], stop_price) + stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99) logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " f"because stop price {sp} would be <= stop limit {stop_gap_price}") return False + return True def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlists and assigns and returns them again. """ - stoploss = self._config.get('stoploss') - if stoploss is not None: - # Precalculate sanitized stoploss value to avoid recalculation for every pair - stoploss = 1 - abs(stoploss) # Copy list since we're modifying this list for p in deepcopy(pairlist): - ticker = tickers.get(p) # Filter out assets which would not allow setting a stoploss - if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)): + if not self._validate_precision_filter(tickers[p], self._stoploss): pairlist.remove(p) - continue return pairlist diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py index 2f7e98e24..167717656 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -1,9 +1,13 @@ +""" +Price pair list filter +""" import logging from copy import deepcopy from typing import Any, Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) @@ -38,14 +42,12 @@ class PriceFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ if ticker['last'] is None: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).") return False - compare = ticker['last'] + self._exchange.price_get_one_pip(ticker['symbol'], - ticker['last']) - changeperc = (compare - ticker['last']) / ticker['last'] + compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last']) + changeperc = compare / ticker['last'] if changeperc > self._low_price_ratio: self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " f"because 1 unit is {changeperc * 100:.3f}%") @@ -60,14 +62,11 @@ class PriceFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - # Copy list since we're modifying this list - for p in deepcopy(pairlist): - ticker = tickers.get(p) - if not ticker: - pairlist.remove(p) - - # Filter out assets which would not allow setting a stoploss - if self._low_price_ratio and not self._validate_ticker_lowprice(ticker): - pairlist.remove(p) + if self._low_price_ratio: + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + # Filter out assets which would not allow setting a stoploss + if not self._validate_ticker_lowprice(tickers[p]): + pairlist.remove(p) return pairlist diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py index 49731ef11..88e143a50 100644 --- a/freqtrade/pairlist/SpreadFilter.py +++ b/freqtrade/pairlist/SpreadFilter.py @@ -1,9 +1,13 @@ +""" +Spread pair list filter +""" import logging from copy import deepcopy from typing import Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) @@ -31,8 +35,24 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio * 100}%.") - def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + def _validate_spread(self, ticker: dict) -> bool: + """ + Validate spread for the ticker + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, False if it should be removed + """ + if 'bid' in ticker and 'ask' in ticker: + spread = 1 - ticker['bid'] / ticker['ask'] + if spread > self._max_spread_ratio: + self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because spread {spread * 100:.3f}% >" + f"{self._max_spread_ratio * 100}%") + return False + else: + return True + return False + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary @@ -41,19 +61,10 @@ class SpreadFilter(IPairList): :return: new whitelist """ # Copy list since we're modifying this list - - spread = None for p in deepcopy(pairlist): - ticker = tickers.get(p) - assert ticker is not None - if 'bid' in ticker and 'ask' in ticker: - spread = 1 - ticker['bid'] / ticker['ask'] - if not ticker or spread > self._max_spread_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because spread {spread * 100:.3f}% >" - f"{self._max_spread_ratio * 100}%") - pairlist.remove(p) - else: + ticker = tickers[p] + # Filter out assets + if not self._validate_spread(ticker): pairlist.remove(p) return pairlist diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 0050fbd5c..07e559168 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -1,14 +1,14 @@ """ -Static List provider +Static Pair List provider -Provides lists as configured in config.json - - """ +Provides pair white list as it configured in config +""" import logging from typing import Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index eb44fe725..e20fb3577 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -1,9 +1,8 @@ """ Volume PairList provider -Provides lists as configured in config.json - - """ +Provides dynamic pair list based on trade volumes +""" import logging from datetime import datetime from typing import Any, Dict, List @@ -11,8 +10,10 @@ from typing import Any, Dict, List from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) + SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] @@ -26,6 +27,8 @@ class VolumePairList(IPairList): raise OperationalException( f'`number_assets` not specified. Please check your configuration ' 'for "pairlist.config.number_assets"') + + self._stake_currency = config['stake_currency'] self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') self._min_value = self._pairlistconfig.get('min_value', 0) @@ -36,9 +39,11 @@ class VolumePairList(IPairList): 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) + if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + if self._sort_key != 'quoteVolume': logger.warning( "DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated." @@ -76,42 +81,42 @@ class VolumePairList(IPairList): (self._last_refresh + self.refresh_period < datetime.now().timestamp())): self._last_refresh = int(datetime.now().timestamp()) - pairs = self._gen_pair_whitelist(pairlist, tickers, - self._config['stake_currency'], - self._sort_key, self._min_value) + pairs = self._gen_pair_whitelist(pairlist, tickers) else: pairs = pairlist + self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + return pairs - def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict, - base_currency: str, key: str, min_val: int) -> List[str]: + def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'quoteVolume') + :param pairlist: pairlist to filter or sort :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ if self._pairlist_pos == 0: # If VolumePairList is the first in the list, use fresh pairlist # Check if pair quote currency equals to the stake currency. - filtered_tickers = [v for k, v in tickers.items() - if (self._exchange.get_pair_quote_currency(k) == base_currency - and v[key] is not None)] + filtered_tickers = [ + v for k, v in tickers.items() + if (self._exchange.get_pair_quote_currency(k) == self._stake_currency + and v[self._sort_key] is not None)] else: - # If other pairlist is in front, use the incomming pairlist. + # If other pairlist is in front, use the incoming pairlist. filtered_tickers = [v for k, v in tickers.items() if k in pairlist] - if min_val > 0: - filtered_tickers = list(filter(lambda t: t[key] > min_val, filtered_tickers)) + if self._min_value > 0: + filtered_tickers = [ + v for v in filtered_tickers if v[self._sort_key] > self._min_value] - sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key]) + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) # Validate whitelist to only have active market pairs pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) pairs = self._verify_blacklist(pairs, aswarning=False) - # Limit to X number of pairs + # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] return pairs diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 5b4c5b602..b01f1342b 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -1,10 +1,8 @@ """ -Static List provider - -Provides lists as configured in config.json - - """ +PairList manager class +""" import logging +from copy import deepcopy from typing import Dict, List from cachetools import TTLCache, cached @@ -13,6 +11,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver + logger = logging.getLogger(__name__) @@ -78,19 +77,33 @@ class PairListManager(): """ Run pairlist through all configured pairlists. """ - - pairlist = self._whitelist.copy() - - # tickers should be cached to avoid calling the exchange on each call. + # Tickers should be cached to avoid calling the exchange on each call. tickers: Dict = {} if self._tickers_needed: tickers = self._get_cached_tickers() + # Adjust whitelist if filters are using tickers + pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers) + # Process all pairlists in chain for pl in self._pairlists: pairlist = pl.filter_pairlist(pairlist, tickers) - # Validation against blacklist happens after the pairlists to ensure blacklist is respected. + # Validation against blacklist happens after the pairlists to ensure + # blacklist is respected. pairlist = IPairList.verify_blacklist(pairlist, self.blacklist, True) self._whitelist = pairlist + + def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]: + """ + Prepare sanitized pairlist for Pairlist Filters that use tickers data - remove + pairs that do not have ticker available + """ + if self._tickers_needed: + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + if p not in tickers: + pairlist.remove(p) + + return pairlist diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index f9e2893c3..61f6b4bd5 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -69,44 +69,44 @@ def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): def test_load_pairlist_noexist(mocker, markets, default_conf): - bot = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - plm = PairListManager(bot.exchange, default_conf) + plm = PairListManager(freqtrade.exchange, default_conf) with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver.load_pairlist('NonexistingPairList', bot.exchange, plm, + PairListResolver.load_pairlist('NonexistingPairList', freqtrade.exchange, plm, default_conf, {}, 1) def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): - freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - freqtradebot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert set(whitelist) == set(freqtradebot.pairlists.whitelist) + assert set(whitelist) == set(freqtrade.pairlists.whitelist) # Ensure config dict hasn't been changed assert (static_pl_conf['exchange']['pair_whitelist'] == - freqtradebot.config['exchange']['pair_whitelist']) + freqtrade.config['exchange']['pair_whitelist']) def test_refresh_static_pairlist(mocker, markets, static_pl_conf): - freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', exchange_has=MagicMock(return_value=True), markets=PropertyMock(return_value=markets), ) - freqtradebot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert set(whitelist) == set(freqtradebot.pairlists.whitelist) - assert static_pl_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist + assert set(whitelist) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): @@ -116,7 +116,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co get_tickers=tickers, exchange_has=MagicMock(return_value=True), ) - bot = get_patched_freqtradebot(mocker, whitelist_conf) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -124,9 +124,9 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co ) # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'] - bot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() - assert whitelist == bot.pairlists.whitelist + assert whitelist == freqtrade.pairlists.whitelist whitelist_conf['pairlists'] = [{'method': 'VolumePairList', 'config': {} @@ -136,7 +136,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co with pytest.raises(OperationalException, match=r'`number_assets` not specified. Please check your configuration ' r'for "pairlist.config.number_assets"'): - PairListManager(bot.exchange, whitelist_conf) + PairListManager(freqtrade.exchange, whitelist_conf) def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @@ -144,13 +144,13 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): 'freqtrade.exchange.Exchange', exchange_has=MagicMock(return_value=True), ) - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_empty)) # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) @@ -206,6 +206,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t pairlists, base_currency, whitelist_result, caplog) -> None: whitelist_conf['pairlists'] = pairlists + whitelist_conf['stake_currency'] = base_currency mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) @@ -215,7 +216,6 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t markets=PropertyMock(return_value=shitcoinmarkets), ) - freqtrade.config['stake_currency'] = base_currency freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist @@ -312,18 +312,18 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): exchange_has=MagicMock(return_value=True), get_tickers=tickers ) - bot = get_patched_freqtradebot(mocker, whitelist_conf) - assert bot.pairlists._pairlists[0]._last_refresh == 0 + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + assert freqtrade.pairlists._pairlists[0]._last_refresh == 0 assert tickers.call_count == 0 - bot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() assert tickers.call_count == 1 - assert bot.pairlists._pairlists[0]._last_refresh != 0 - lrf = bot.pairlists._pairlists[0]._last_refresh - bot.pairlists.refresh_pairlist() + assert freqtrade.pairlists._pairlists[0]._last_refresh != 0 + lrf = freqtrade.pairlists._pairlists[0]._last_refresh + freqtrade.pairlists.refresh_pairlist() assert tickers.call_count == 1 # Time should not be updated. - assert bot.pairlists._pairlists[0]._last_refresh == lrf + assert freqtrade.pairlists._pairlists[0]._last_refresh == lrf def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog):