diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index ef9ea1d04..4a83bc939 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -46,7 +46,7 @@ class IPairList(ABC): """ @abstractmethod - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary @@ -97,5 +97,6 @@ class IPairList(ABC): if pair not in sanitized_whitelist: sanitized_whitelist.append(pair) + sanitized_whitelist = self._verify_blacklist(sanitized_whitelist) # We need to remove pairs that are unknown return sanitized_whitelist diff --git a/freqtrade/pairlist/LowPriceFilter.py b/freqtrade/pairlist/LowPriceFilter.py index 9552b56b8..4e1ba52c8 100644 --- a/freqtrade/pairlist/LowPriceFilter.py +++ b/freqtrade/pairlist/LowPriceFilter.py @@ -46,7 +46,7 @@ class LowPriceFilter(IPairList): return False return True - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. @@ -57,7 +57,9 @@ class LowPriceFilter(IPairList): """ # Copy list since we're modifying this list for p in deepcopy(pairlist): - ticker = [t for t in tickers if t['symbol'] == p][0] + ticker = tickers.get(p) + if not ticker: + pairlist.remove(p) # Filter out assets which would not allow setting a stoploss if self._low_price_percent and not self._validate_ticker_lowprice(ticker): diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index b18237b00..d7b2c96ae 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -44,7 +44,7 @@ class PrecisionFilter(IPairList): return False return True - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlists and assigns and returns them again. """ @@ -53,9 +53,9 @@ class PrecisionFilter(IPairList): stoploss = 1 - abs(self._config.get('stoploss')) # Copy list since we're modifying this list for p in deepcopy(pairlist): - ticker = [t for t in tickers if t['symbol'] == p][0] + ticker = tickers.get(p) # Filter out assets which would not allow setting a stoploss - if (stoploss and not self._validate_precision_filter(ticker, stoploss)): + if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)): pairlist.remove(p) continue diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 83450c4bc..a7b71875c 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -33,7 +33,7 @@ class StaticPairList(IPairList): """ return f"{self.name}" - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 610986a72..e6ff69daf 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -5,10 +5,9 @@ Provides lists as configured in config.json """ import logging +from datetime import datetime from typing import Dict, List -from cachetools import TTLCache, cached - from freqtrade import OperationalException from freqtrade.pairlist.IPairList import IPairList @@ -28,6 +27,7 @@ class VolumePairList(IPairList): 'for "pairlist.config.number_assets"') self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') + self._ttl = self._pairlistconfig.get('ttl', 1800) if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -37,6 +37,7 @@ class VolumePairList(IPairList): if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + self._last_refresh = 0 @property def needstickers(self) -> bool: @@ -56,7 +57,7 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." - def filter_pairlist(self, pairlist: List[str], tickers: List[Dict]) -> List[str]: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary @@ -65,10 +66,17 @@ class VolumePairList(IPairList): :return: new whitelist """ # Generate dynamic whitelist - return self._gen_pair_whitelist(self._config['stake_currency'], self._sort_key) + if self._last_refresh + self._ttl < datetime.now().timestamp(): + self._last_refresh = datetime.now().timestamp() + return self._gen_pair_whitelist(pairlist, + tickers, + self._config['stake_currency'], + self._sort_key, + ) + else: + return pairlist - @cached(TTLCache(maxsize=1, ttl=1800)) - def _gen_pair_whitelist(self, base_currency: str, key: str) -> List[str]: + def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]: """ Updates the whitelist with with a dynamically generated list :param base_currency: base currency as str @@ -77,7 +85,6 @@ class VolumePairList(IPairList): :return: List of pairs """ - tickers = self._exchange.get_tickers() # check length so that we make sure that '/' is actually in the string tickers = [v for k, v in tickers.items() if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index e6b2b6103..03451e725 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -64,7 +64,7 @@ class PairListManager(): pairlist = self._whitelist.copy() # tickers should be cached to avoid calling the exchange on each call. - tickers: List[Dict] = [] + tickers: Dict = {} if self._tickers_needed: tickers = self._exchange.get_tickers() diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index ffebaf60f..5ad7fdf5a 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -16,7 +16,6 @@ from tests.conftest import get_patched_freqtradebot, log_has_re @pytest.fixture(scope="function") def whitelist_conf(default_conf): default_conf['stake_currency'] = 'BTC' - default_conf['exchange']['name'] = 'binance' default_conf['exchange']['pair_whitelist'] = [ 'ETH/BTC', 'TKN/BTC', @@ -137,31 +136,37 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @pytest.mark.parametrize("filters,base_currency,key,whitelist_result", [ - ({}, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), - ({}, "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), - ({}, "USDT", "quoteVolume", ['ETH/USDT']), - ({}, "ETH", "quoteVolume", []), - ({"PrecisionFilter": {}}, "BTC", "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), - ({"PrecisionFilter": {}}, "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), - ({"LowPriceFilter": {"low_price_percent": 0.03}}, "BTC", - "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), + ([], "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), + ([], "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'HOT/BTC', 'FUEL/BTC']), + ([], "USDT", "quoteVolume", ['ETH/USDT']), + ([], "ETH", "quoteVolume", []), + ([{"method": "PrecisionFilter"}], "BTC", + "quoteVolume", ["LTC/BTC", "ETH/BTC", "TKN/BTC", 'FUEL/BTC']), + ([{"method": "PrecisionFilter"}], + "BTC", "bidVolume", ["LTC/BTC", "TKN/BTC", "ETH/BTC", 'FUEL/BTC']), + ([{"method": "LowPriceFilter", "config": {"low_price_percent": 0.03}}], + "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC', 'FUEL/BTC']), # Hot is removed by precision_filter, Fuel by low_price_filter. - ({"PrecisionFilter": {}, "LowPriceFilter": {"low_price_percent": 0.02}}, - "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC']), -]) + ([{"method": "PrecisionFilter"}, + {"method": "LowPriceFilter", "config": {"low_price_percent": 0.02}} + ], "BTC", "bidVolume", ['LTC/BTC', 'TKN/BTC', 'ETH/BTC'])]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, filters, base_currency, key, whitelist_result, caplog) -> None: - whitelist_conf['pairlist']['method'] = 'VolumePairList' - whitelist_conf['pairlist']['filters'] = filters + whitelist_conf['pairlists'].extend(filters) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=shitcoinmarkets)) - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=shitcoinmarkets), + ) freqtrade.config['stake_currency'] = base_currency - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist + assert sorted(whitelist) == sorted(whitelist_result) if 'PrecisionFilter' in filters: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' @@ -172,11 +177,14 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: - default_conf['pairlist'] = {'method': 'VolumePairList', - 'config': {'number_assets': 10} - } - mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=False)) + default_conf['pairlists'] = [{'method': 'VolumePairList', + 'config': {'number_assets': 10} + }] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + exchange_has=MagicMock(return_value=False), + ) with pytest.raises(OperationalException): get_patched_freqtradebot(mocker, default_conf) @@ -202,18 +210,22 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): (['ETH/BTC', 'TKN/BTC'], ""), (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available - (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist + (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "), # BLK/BTC in blacklist (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") # BTT/BTC is inactive ]) def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, - log_message): - whitelist_conf['pairlist']['method'] = pairlist - mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + log_message, tickers): + whitelist_conf['pairlists'][0]['method'] = pairlist + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) caplog.clear() - new_whitelist = freqtrade.pairlists._whitelist_for_active_markets(whitelist) + # Assign starting whitelist + new_whitelist = freqtrade.pairlists._pairlists[0]._whitelist_for_active_markets(whitelist) assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC']) assert log_message in caplog.text