From 8e89802b2d099566ffb1de5effdeec10a52f43d2 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 22 May 2020 15:03:49 +0300 Subject: [PATCH 1/7] Split the generation logic and filtering --- freqtrade/pairlist/IPairList.py | 11 +++ freqtrade/pairlist/StaticPairList.py | 11 ++- freqtrade/pairlist/VolumePairList.py | 60 ++++++------ freqtrade/pairlist/pairlistmanager.py | 3 + tests/pairlist/test_pairlist.py | 127 +++++++++++++++++--------- 5 files changed, 135 insertions(+), 77 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index e49ad1561..0a291fdf7 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List from cachetools import TTLCache, cached +from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active @@ -90,6 +91,16 @@ class IPairList(ABC): """ raise NotImplementedError() + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + raise OperationalException("This Pairlist Handler should not be used " + "at the first position in the list of Pairlist Handlers.") + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 07e559168..0218833e3 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -30,6 +30,15 @@ class StaticPairList(IPairList): """ return f"{self.name}" + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -38,4 +47,4 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + return pairlist diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 6f39ae6d6..d32be3dc9 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -68,6 +68,31 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." + def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + """ + Generate the pairlist + :param cached_pairlist: Previously generated pairlist (cached) + :param tickers: Tickers (from exchange.get_tickers()). + :return: List of pairs + """ + # Generate dynamic whitelist + # Must always run if this pairlist is not the first in the list. + if self._last_refresh + self.refresh_period < datetime.now().timestamp(): + self._last_refresh = int(datetime.now().timestamp()) + + # Use fresh pairlist + # Check if pair quote currency equals to the stake currency. + filtered_tickers = [ + v for k, v in tickers.items() + if (self._exchange.get_pair_quote_currency(k) == self._stake_currency + and v[self._sort_key] is not None)] + pairlist = [s['symbol'] for s in filtered_tickers] + else: + # Use the cached pairlist if it's not time yet to refresh + pairlist = cached_pairlist + + return pairlist + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -76,37 +101,8 @@ class VolumePairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - # Generate dynamic whitelist - # Must always run if this pairlist is not the first in the list. - if (self._pairlist_pos != 0 or - (self._last_refresh + self.refresh_period < datetime.now().timestamp())): - - self._last_refresh = int(datetime.now().timestamp()) - pairs = self._gen_pair_whitelist(pairlist, tickers) - else: - pairs = pairlist - - self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") - - return pairs - - def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]: - """ - Updates the whitelist with with a dynamically generated list - :param pairlist: pairlist to filter or sort - :param tickers: Tickers (from exchange.get_tickers()). - :return: List of pairs - """ - if self._pairlist_pos == 0: - # If VolumePairList is the first in the list, use fresh pairlist - # Check if pair quote currency equals to the stake currency. - filtered_tickers = [ - v for k, v in tickers.items() - if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - and v[self._sort_key] is not None)] - else: - # If other pairlist is in front, use the incoming pairlist. - filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + # Use the incoming pairlist. + filtered_tickers = [v for k, v in tickers.items() if k in pairlist] if self._min_value > 0: filtered_tickers = [ @@ -120,4 +116,6 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] + self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + return pairs diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 6ad6c610b..177f79083 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -87,6 +87,9 @@ class PairListManager(): # Adjust whitelist if filters are using tickers pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers) + # Generate the pairlist with first Pairlist Handler in the chain + pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers) + # Process all Pairlist Handlers in the chain for pairlist_handler in self._pairlist_handlers: pairlist = pairlist_handler.filter_pairlist(pairlist, tickers) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index e9e688b78..febb8b163 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -19,7 +19,8 @@ def whitelist_conf(default_conf): 'TKN/BTC', 'TRST/BTC', 'SWT/BTC', - 'BCC/BTC' + 'BCC/BTC', + 'HOT/BTC', ] default_conf['exchange']['pair_blacklist'] = [ 'BLK/BTC' @@ -201,21 +202,21 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ +@pytest.mark.parametrize("pairlists,base_currency,whitelist_result,operational_exception", [ # VolumePairList only ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'], False), # Different sorting depending on quote or bid volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT'], False), # No pair for ETH, VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "ETH", []), + "ETH", [], False), # No pair for ETH, StaticPairList ([{"method": "StaticPairList"}], - "ETH", []), + "ETH", [], False), # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, @@ -223,57 +224,87 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, {"method": "ShuffleFilter"}], - "ETH", []), + "ETH", [], False), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + {"method": "PrecisionFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), # Precisionfilter bid ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, - {"method": "PrecisionFilter"}], "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), + {"method": "PrecisionFilter"}], + "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "USDT", ['ETH/USDT', 'NANO/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT'], False), # Hot is removed by precision_filter, Fuel by low_price_filter. ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), # HOT and XRP are removed because below 1250 quoteVolume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "min_value": 1250}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], False), # StaticPairlist only ([{"method": "StaticPairList"}], - "BTC", ['ETH/BTC', 'TKN/BTC']), + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC'], False), # Static Pairlist before VolumePairList - sorting changes ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['TKN/BTC', 'ETH/BTC']), + "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC'], False), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], - "USDT", ['ETH/USDT']), + "USDT", ['ETH/USDT'], False), # ShuffleFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 77}], - "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']), + "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT'], False), # ShuffleFilter, other seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 42}], - "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']), + "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT'], False), # ShuffleFilter, no seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter"}], - "USDT", 3), + "USDT", 3, False), + # PrecisionFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PrecisionFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC'], False), + # PrecisionFilter only + ([{"method": "PrecisionFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC'], True), + # PriceFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PriceFilter", "low_price_ratio": 0.02}], + "BTC", ['ETH/BTC', 'TKN/BTC'], False), + # PriceFilter only + ([{"method": "PriceFilter", "low_price_ratio": 0.02}], + "BTC", ['ETH/BTC', 'TKN/BTC'], True), + # ShuffleFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "ShuffleFilter", "seed": 42}], + "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], False), + # ShuffleFilter only + ([{"method": "ShuffleFilter", "seed": 42}], + "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], True), + # SpreadFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005}], + "BTC", ['ETH/BTC', 'TKN/BTC'], False), + # SpreadFilter only + ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}], + "BTC", ['ETH/BTC', 'TKN/BTC'], True), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, - caplog) -> None: + operational_exception, caplog) -> None: whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency @@ -285,32 +316,38 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t markets=PropertyMock(return_value=shitcoinmarkets), ) - freqtrade.pairlists.refresh_pairlist() - whitelist = freqtrade.pairlists.whitelist - - assert isinstance(whitelist, list) - - # Verify length of pairlist matches (used for ShuffleFilter without seed) - if type(whitelist_result) is list: - assert whitelist == whitelist_result + if operational_exception: + with pytest.raises(OperationalException, + match=r"This Pairlist Handler should not be used at the first position " + r"in the list of Pairlist Handlers."): + freqtrade.pairlists.refresh_pairlist() else: - len(whitelist) == whitelist_result + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist - for pairlist in pairlists: - if pairlist['method'] == 'PrecisionFilter' and whitelist_result: - assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' - r'would be <= stop limit.*', caplog) - if pairlist['method'] == 'PriceFilter' and whitelist_result: - assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or - log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] is empty.*", - caplog)) - if pairlist['method'] == 'VolumePairList': - logmsg = ("DEPRECATED: using any key other than quoteVolume for " - "VolumePairList is deprecated.") - if pairlist['sort_key'] != 'quoteVolume': - assert log_has(logmsg, caplog) - else: - assert not log_has(logmsg, caplog) + assert isinstance(whitelist, list) + + # Verify length of pairlist matches (used for ShuffleFilter without seed) + if type(whitelist_result) is list: + assert whitelist == whitelist_result + else: + len(whitelist) == whitelist_result + + for pairlist in pairlists: + if pairlist['method'] == 'PrecisionFilter' and whitelist_result: + assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' + r'would be <= stop limit.*', caplog) + if pairlist['method'] == 'PriceFilter' and whitelist_result: + assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or + log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] " + r"is empty.*", caplog)) + if pairlist['method'] == 'VolumePairList': + logmsg = ("DEPRECATED: using any key other than quoteVolume for " + "VolumePairList is deprecated.") + if pairlist['sort_key'] != 'quoteVolume': + assert log_has(logmsg, caplog) + else: + assert not log_has(logmsg, caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: From 0e416dc4f58983af5ee5440a493d52ccadaf97cc Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 22 May 2020 16:42:02 +0300 Subject: [PATCH 2/7] Simplify tests --- tests/pairlist/test_pairlist.py | 59 +++++++++++++++++---------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index febb8b163..3e3241071 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -202,21 +202,21 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -@pytest.mark.parametrize("pairlists,base_currency,whitelist_result,operational_exception", [ +@pytest.mark.parametrize("pairlists,base_currency,whitelist_result", [ # VolumePairList only ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), # Different sorting depending on quote or bid volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False), + "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT'], False), + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), # No pair for ETH, VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "ETH", [], False), + "ETH", []), # No pair for ETH, StaticPairList ([{"method": "StaticPairList"}], - "ETH", [], False), + "ETH", []), # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, @@ -224,87 +224,87 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, {"method": "ShuffleFilter"}], - "ETH", [], False), + "ETH", []), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # Precisionfilter bid ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, {"method": "PrecisionFilter"}], - "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC'], False), + "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], - "USDT", ['ETH/USDT', 'NANO/USDT'], False), + "USDT", ['ETH/USDT', 'NANO/USDT']), # Hot is removed by precision_filter, Fuel by low_price_filter. ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # HOT and XRP are removed because below 1250 quoteVolume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "min_value": 1250}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), # StaticPairlist only ([{"method": "StaticPairList"}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), # Static Pairlist before VolumePairList - sorting changes ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC'], False), + "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC']), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], - "USDT", ['ETH/USDT'], False), + "USDT", ['ETH/USDT']), # ShuffleFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 77}], - "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT'], False), + "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']), # ShuffleFilter, other seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 42}], - "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT'], False), + "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']), # ShuffleFilter, no seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter"}], - "USDT", 3, False), + "USDT", 3), # whitelist_result is integer -- check only lenght of randomized pairlist # PrecisionFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "PrecisionFilter"}], - "BTC", ['ETH/BTC', 'TKN/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC']), # PrecisionFilter only ([{"method": "PrecisionFilter"}], - "BTC", ['ETH/BTC', 'TKN/BTC'], True), + "BTC", None), # OperationalException expected # PriceFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", ['ETH/BTC', 'TKN/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC']), # PriceFilter only ([{"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", ['ETH/BTC', 'TKN/BTC'], True), + "BTC", None), # OperationalException expected # ShuffleFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "ShuffleFilter", "seed": 42}], - "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], False), + "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC']), # ShuffleFilter only ([{"method": "ShuffleFilter", "seed": 42}], - "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC'], True), + "BTC", None), # OperationalException expected # SpreadFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], - "BTC", ['ETH/BTC', 'TKN/BTC'], False), + "BTC", ['ETH/BTC', 'TKN/BTC']), # SpreadFilter only ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}], - "BTC", ['ETH/BTC', 'TKN/BTC'], True), + "BTC", None), # OperationalException expected ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, - operational_exception, caplog) -> None: + caplog) -> None: whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency @@ -316,7 +316,8 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t markets=PropertyMock(return_value=shitcoinmarkets), ) - if operational_exception: + # Set whitelist_result to None if pairlist is invalid and should produce exception + if whitelist_result is None: with pytest.raises(OperationalException, match=r"This Pairlist Handler should not be used at the first position " r"in the list of Pairlist Handlers."): From c3206d72cbed5c1b24ba91cd8c9bd31dfbf4a7be Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 25 May 2020 22:49:57 +0300 Subject: [PATCH 3/7] Adjust docstring for IPairList.gen_pairlist() --- freqtrade/pairlist/IPairList.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 0a291fdf7..f48a7dcfd 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -93,7 +93,15 @@ class IPairList(ABC): def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: """ - Generate the pairlist + Generate the pairlist. + + This method is called once by the pairlistmanager in the refresh_pairlist() + method to supply the starting pairlist for the chain of the Pairlist Handlers. + Pairlist Filters (those Pairlist Handlers that cannot be used at the first + position in the chain) shall not override this base implementation -- + it will raise the exception if a Pairlist Handler is used at the first + position in the chain. + :param cached_pairlist: Previously generated pairlist (cached) :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs From a48412427218109f937b17bf2309d6cd70cd79bc Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 25 May 2020 23:14:51 +0300 Subject: [PATCH 4/7] Raise exception if StaticPairList on a non-first position --- freqtrade/pairlist/StaticPairList.py | 7 ++++++- tests/pairlist/test_pairlist.py | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 0218833e3..4c260d2fe 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -6,6 +6,7 @@ Provides pair white list as it configured in config import logging from typing import Dict, List +from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList @@ -47,4 +48,8 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - return pairlist + if self._pairlist_pos != 0: + raise OperationalException(f"{self.name} can only be used in the first position " + "in the list of Pairlist Handlers.") + else: + return pairlist diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 3e3241071..9c35eae1b 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -279,28 +279,32 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "BTC", ['ETH/BTC', 'TKN/BTC']), # PrecisionFilter only ([{"method": "PrecisionFilter"}], - "BTC", None), # OperationalException expected + "BTC", 'filter_at_the_beginning'), # OperationalException expected # PriceFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], "BTC", ['ETH/BTC', 'TKN/BTC']), # PriceFilter only ([{"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", None), # OperationalException expected + "BTC", 'filter_at_the_beginning'), # OperationalException expected # ShuffleFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "ShuffleFilter", "seed": 42}], "BTC", ['TKN/BTC', 'ETH/BTC', 'HOT/BTC']), # ShuffleFilter only ([{"method": "ShuffleFilter", "seed": 42}], - "BTC", None), # OperationalException expected + "BTC", 'filter_at_the_beginning'), # OperationalException expected # SpreadFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], "BTC", ['ETH/BTC', 'TKN/BTC']), # SpreadFilter only ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}], - "BTC", None), # OperationalException expected + "BTC", 'filter_at_the_beginning'), # OperationalException expected + # Static Pairlist after VolumePairList, on a non-first position + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, + {"method": "StaticPairList"}], + "BTC", 'static_in_the_middle'), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, @@ -317,11 +321,16 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ) # Set whitelist_result to None if pairlist is invalid and should produce exception - if whitelist_result is None: + if whitelist_result == 'filter_at_the_beginning': with pytest.raises(OperationalException, match=r"This Pairlist Handler should not be used at the first position " r"in the list of Pairlist Handlers."): freqtrade.pairlists.refresh_pairlist() + elif whitelist_result == 'static_in_the_middle': + with pytest.raises(OperationalException, + match=r"StaticPairList can only be used in the first position " + r"in the list of Pairlist Handlers."): + freqtrade.pairlists.refresh_pairlist() else: freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist From 909b81255005839eb7f41a02f5138f844e0a67bf Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Fri, 29 May 2020 00:28:24 +0300 Subject: [PATCH 5/7] Update developer.md --- docs/developer.md | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 34b2f1ba5..a680bc8dc 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -92,13 +92,13 @@ docker-compose exec freqtrade_develop /bin/bash You have a great idea for a new pair selection algorithm you would like to try out? Great. Hopefully you also want to contribute this back upstream. -Whatever your motivations are - This should get you off the ground in trying to develop a new Pairlist provider. +Whatever your motivations are - This should get you off the ground in trying to develop a new Pairlist Handler. -First of all, have a look at the [VolumePairList](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/pairlist/VolumePairList.py) provider, and best copy this file with a name of your new Pairlist Provider. +First of all, have a look at the [VolumePairList](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/pairlist/VolumePairList.py) Handler, and best copy this file with a name of your new Pairlist Handler. -This is a simple provider, which however serves as a good example on how to start developing. +This is a simple Handler, which however serves as a good example on how to start developing. -Next, modify the classname of the provider (ideally align this with the Filename). +Next, modify the classname of the Handler (ideally align this with the module filename). The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists. @@ -114,28 +114,44 @@ Now, let's step through the methods which require actions: #### Pairlist configuration -Configuration for PairListProvider is done in the bot configuration file in the element `"pairlist"`. -This Pairlist-object may contain configurations with additional configurations for the configured pairlist. -By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the whitelist. Please follow this to ensure a consistent user experience. +Configuration for the chain of Pairlist Handlers is done in the bot configuration file in the element `"pairlists"`, an array of configuration parameters for each Pairlist Handlers in the chain. -Additional elements can be configured as needed. `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. +By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the pairlist. Please follow this to ensure a consistent user experience. + +Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. #### short_desc Returns a description used for Telegram messages. -This should contain the name of the Provider, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. + +This should contain the name of the Pairlist Handler, as well as a short description containing the number of assets. Please follow the format `"PairlistName - top/bottom X pairs"`. + +#### gen_pairlist + +Override this method if the Pairlist Handler can be used as the leading Pairlist Handler in the chain, defining the initial pairlist which is then handled by all Pairlist Handlers in the chain. Examples are `StaticPairList` and `VolumePairList`. + +This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. + +It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers). + +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. #### filter_pairlist -Override this method and run all calculations needed in this method. +This method is called for each Pairlist Handler in the chain by the pairlist manager. + This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. -It must return the resulting pairlist (which may then be passed into the next pairlist filter). +The default implementation in the base class simply calls the `_validate_pair()` method for each pair in the pairlist, but you may override it. So you should either implement the `_validate_pair()` in your Pairlist Handler or override `filter_pairlist()` to do something else. + +If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain). Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. +In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned. + ##### sample ``` python @@ -145,11 +161,6 @@ Validations are optional, the parent class exposes a `_verify_blacklist(pairlist return pairs ``` -#### _gen_pair_whitelist - -This is a simple method used by `VolumePairList` - however serves as a good example. -In VolumePairList, this implements different methods of sorting, does early validation so only the expected number of pairs is returned. - ## Implement a new Exchange (WIP) !!! Note From 43225cfdcf71df41314ababe06860ea29c2e0638 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Fri, 29 May 2020 11:28:09 +0300 Subject: [PATCH 6/7] Update docs/developer.md Co-authored-by: Matthias --- docs/developer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer.md b/docs/developer.md index a680bc8dc..036109d5b 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -130,7 +130,7 @@ This should contain the name of the Pairlist Handler, as well as a short descrip Override this method if the Pairlist Handler can be used as the leading Pairlist Handler in the chain, defining the initial pairlist which is then handled by all Pairlist Handlers in the chain. Examples are `StaticPairList` and `VolumePairList`. -This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. +This is called with each iteration of the bot (only if the Pairlist Handler is at the first location) - so consider implementing caching for compute/network heavy calculations. It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers). From a4cf9ba85b2a3c823fef5a9622c6f809745b8a76 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 29 May 2020 12:40:05 +0300 Subject: [PATCH 7/7] Move check for position for StaticPairList to init --- freqtrade/pairlist/StaticPairList.py | 17 +++++++++++------ tests/pairlist/test_pairlist.py | 14 ++++++++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 4c260d2fe..b5c1bc767 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -4,7 +4,7 @@ Static Pair List provider Provides pair white list as it configured in config """ import logging -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList @@ -15,6 +15,15 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + if self._pairlist_pos != 0: + raise OperationalException(f"{self.name} can only be used in the first position " + "in the list of Pairlist Handlers.") + @property def needstickers(self) -> bool: """ @@ -48,8 +57,4 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - if self._pairlist_pos != 0: - raise OperationalException(f"{self.name} can only be used in the first position " - "in the list of Pairlist Handlers.") - else: - return pairlist + return pairlist diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 9c35eae1b..421f06911 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -313,8 +313,15 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t whitelist_conf['stake_currency'] = base_currency mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + if whitelist_result == 'static_in_the_middle': + with pytest.raises(OperationalException, + match=r"StaticPairList can only be used in the first position " + r"in the list of Pairlist Handlers."): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + return + + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, markets=PropertyMock(return_value=shitcoinmarkets), @@ -326,11 +333,6 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t match=r"This Pairlist Handler should not be used at the first position " r"in the list of Pairlist Handlers."): freqtrade.pairlists.refresh_pairlist() - elif whitelist_result == 'static_in_the_middle': - with pytest.raises(OperationalException, - match=r"StaticPairList can only be used in the first position " - r"in the list of Pairlist Handlers."): - freqtrade.pairlists.refresh_pairlist() else: freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist