From 8e89802b2d099566ffb1de5effdeec10a52f43d2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 22 May 2020 15:03:49 +0300 Subject: [PATCH] Split the generation logic and filtering --- freqtrade/pairlist/IPairList.py | 11 +++ freqtrade/pairlist/StaticPairList.py | 11 ++- freqtrade/pairlist/VolumePairList.py | 60 ++++++------ freqtrade/pairlist/pairlistmanager.py | 3 + tests/pairlist/test_pairlist.py | 127 +++++++++++++++++--------- 5 files changed, 135 insertions(+), 77 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index e49ad1561..0a291fdf7 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List from cachetools import TTLCache, cached +from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active @@ -90,6 +91,16 @@ class IPairList(ABC): """ raise NotImplementedError() + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + raise OperationalException("This Pairlist Handler should not be used " + "at the first position in the list of Pairlist Handlers.") + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 07e559168..0218833e3 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -30,6 +30,15 @@ class StaticPairList(IPairList): """ return f"{self.name}" + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -38,4 +47,4 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + return pairlist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 6f39ae6d6..d32be3dc9 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -68,6 +68,31 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + # Generate dynamic whitelist + # Must always run if this pairlist is not the first in the list. + if self._last_refresh + self.refresh_period < datetime.now().timestamp(): + self._last_refresh = int(datetime.now().timestamp()) + + # Use fresh pairlist + # Check if pair quote currency equals to the stake currency. + filtered_tickers = [ + v for k, v in tickers.items() + if (self._exchange.get_pair_quote_currency(k) == self._stake_currency + and v[self._sort_key] is not None)] + pairlist = [s['symbol'] for s in filtered_tickers] + else: + # Use the cached pairlist if it's not time yet to refresh + pairlist = cached_pairlist + + return pairlist + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -76,37 +101,8 @@ class VolumePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - # Generate dynamic whitelist - # Must always run if this pairlist is not the first in the list. - if (self._pairlist_pos != 0 or - (self._last_refresh + self.refresh_period < datetime.now().timestamp())): - - self._last_refresh = int(datetime.now().timestamp()) - pairs = self._gen_pair_whitelist(pairlist, tickers) - else: - pairs = pairlist - - self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") - - return pairs - - def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param pairlist: pairlist to filter or sort - :param tickers: Tickers (from exchange.get_tickers()). - :return: List of pairs - """ - if self._pairlist_pos == 0: - # If VolumePairList is the first in the list, use fresh pairlist - # Check if pair quote currency equals to the stake currency. - filtered_tickers = [ - v for k, v in tickers.items() - if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - and v[self._sort_key] is not None)] - else: - # If other pairlist is in front, use the incoming pairlist. - filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + # Use the incoming pairlist. + filtered_tickers = [v for k, v in tickers.items() if k in pairlist] if self._min_value > 0: filtered_tickers = [ @@ -120,4 +116,6 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] + self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + return pairs diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 6ad6c610b..177f79083 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -87,6 +87,9 @@ class PairListManager(): # Adjust whitelist if filters are using tickers pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers) + # Generate the pairlist with first Pairlist Handler in the chain + pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers) + # Process all Pairlist Handlers in the chain for pairlist_handler in self._pairlist_handlers: pairlist = pairlist_handler.filter_pairlist(pairlist, tickers) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index e9e688b78..febb8b163 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -19,7 +19,8 @@ def whitelist_conf(default_conf): 'TKN/BTC', 'TRST/BTC', 'SWT/BTC', - 'BCC/BTC' + 'BCC/BTC', + 'HOT/BTC', ] default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' @@ -201,21 +202,21 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ +@pytest.mark.parametrize("pairlists,base_currency,whitelist_result,operational_exception", [ # VolumePairList only ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'], False), # Different sorting depending on quote or bid volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT'], False), # No pair for ETH, VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "ETH", []), + "ETH", [], False), # No pair for ETH, StaticPairList ([{"method": "StaticPairList"}], - "ETH", []), + "ETH", [], False), # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, @@ -223,57 +224,87 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, {"method": "ShuffleFilter"}], - "ETH", []), + "ETH", [], False), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + {"method": "PrecisionFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), # Precisionfilter bid ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, - {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + {"method": "PrecisionFilter"}], + "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "USDT", ['ETH/USDT', 'NANO/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT'], False), # Hot is removed by precision_filter, Fuel by low_price_filter. ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), # HOT and XRP are removed because below 1250 quoteVolume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "min_value": 1250}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], False), # StaticPairlist only ([{"method": "StaticPairList"}], - "BTC", ['ETH/BTC', 'TKN/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC'], False), # Static Pairlist before VolumePairList - sorting changes ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['TKN/BTC', 'ETH/BTC']), + "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC'], False), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], - "USDT", ['ETH/USDT']), + "USDT", ['ETH/USDT'], False), # ShuffleFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 77}], - "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']), + "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT'], False), # ShuffleFilter, other seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 42}], - "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']), + "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT'], False), # ShuffleFilter, no seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter"}], - "USDT", 3), + "USDT", 3, False), + # PrecisionFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PrecisionFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC'], False), + # PrecisionFilter only + ([{"method": "PrecisionFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC'], True), + # PriceFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PriceFilter", "low_price_ratio": 0.02}], + "BTC", ['ETH/BTC', 'TKN/BTC'], False), + # PriceFilter only + ([{"method": "PriceFilter", "low_price_ratio": 0.02}], + "BTC", ['ETH/BTC', 'TKN/BTC'], True), + # ShuffleFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "ShuffleFilter", "seed": 42}], + "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], False), + # ShuffleFilter only + ([{"method": "ShuffleFilter", "seed": 42}], + "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], True), + # SpreadFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005}], + "BTC", ['ETH/BTC', 'TKN/BTC'], False), + # SpreadFilter only + ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}], + "BTC", ['ETH/BTC', 'TKN/BTC'], True), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, - caplog) -> None: + operational_exception, caplog) -> None: whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency @@ -285,32 +316,38 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t markets=PropertyMock(return_value=shitcoinmarkets), ) - freqtrade.pairlists.refresh_pairlist() - whitelist = freqtrade.pairlists.whitelist - - assert isinstance(whitelist, list) - - # Verify length of pairlist matches (used for ShuffleFilter without seed) - if type(whitelist_result) is list: - assert whitelist == whitelist_result + if operational_exception: + with pytest.raises(OperationalException, + match=r"This Pairlist Handler should not be used at the first position " + r"in the list of Pairlist Handlers."): + freqtrade.pairlists.refresh_pairlist() else: - len(whitelist) == whitelist_result + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist - for pairlist in pairlists: - if pairlist['method'] == 'PrecisionFilter' and whitelist_result: - assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' - r'would be <= stop limit.*', caplog) - if pairlist['method'] == 'PriceFilter' and whitelist_result: - assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or - log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] is empty.*", - caplog)) - if pairlist['method'] == 'VolumePairList': - logmsg = ("DEPRECATED: using any key other than quoteVolume for " - "VolumePairList is deprecated.") - if pairlist['sort_key'] != 'quoteVolume': - assert log_has(logmsg, caplog) - else: - assert not log_has(logmsg, caplog) + assert isinstance(whitelist, list) + + # Verify length of pairlist matches (used for ShuffleFilter without seed) + if type(whitelist_result) is list: + assert whitelist == whitelist_result + else: + len(whitelist) == whitelist_result + + for pairlist in pairlists: + if pairlist['method'] == 'PrecisionFilter' and whitelist_result: + assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' + r'would be <= stop limit.*', caplog) + if pairlist['method'] == 'PriceFilter' and whitelist_result: + assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or + log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] " + r"is empty.*", caplog)) + if pairlist['method'] == 'VolumePairList': + logmsg = ("DEPRECATED: using any key other than quoteVolume for " + "VolumePairList is deprecated.") + if pairlist['sort_key'] != 'quoteVolume': + assert log_has(logmsg, caplog) + else: + assert not log_has(logmsg, caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: