From 39232cbcbbe440368f2389c7d03271542b6cca4f Mon Sep 17 00:00:00 2001 From: iuvbio Date: Tue, 5 Mar 2019 21:23:55 +0100 Subject: [PATCH 01/12] loop over whitelist only instead of all markets --- freqtrade/pairlist/IPairList.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 5559c582f..f9a66fe89 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -65,29 +65,26 @@ class IPairList(ABC): :return: the list of pairs the user wants to trade without the one unavailable or black_listed """ - sanitized_whitelist = whitelist markets = self._freqtrade.exchange.markets # Filter to markets in stake currency - markets = [markets[pair] for pair in markets if - markets[pair]['quote'] == self._config['stake_currency']] - known_pairs = set() + stake_pairs = [pair for pair in markets if + pair.endswith(self._config['stake_currency'])] - # TODO: we should loop over whitelist instead of all markets - for market in markets: - pair = market['symbol'] + sanitized_whitelist = [] + for pair in whitelist: # pair is not in the generated dynamic market, or in the blacklist ... ignore it - if pair not in whitelist or pair in self.blacklist: + if pair in self.blacklist or pair not in stake_pairs: continue - # else the pair is valid - known_pairs.add(pair) - # Market is not active + # Check if market is active + market = markets[pair] if not market['active']: - sanitized_whitelist.remove(pair) logger.info( 'Ignoring %s from whitelist. Market is not active.', pair ) + continue + sanitized_whitelist.append(pair) # We need to remove pairs that are unknown - return [x for x in sanitized_whitelist if x in known_pairs] + return sanitized_whitelist From a241e950f292a7da931ae9a04a87896acd6dfc94 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Wed, 6 Mar 2019 22:41:46 +0100 Subject: [PATCH 02/12] prune validate_pairs --- freqtrade/exchange/exchange.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 33f62f2f7..e4838b74c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -239,13 +239,9 @@ class Exchange(object): logger.warning('Unable to validate pairs (assuming they are correct).') # return - stake_cur = self._conf['stake_currency'] for pair in pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format - if not pair.endswith(stake_cur): - raise OperationalException( - f'Pair {pair} not compatible with stake_currency: {stake_cur}') if self.markets and pair not in self.markets: raise OperationalException( f'Pair {pair} is not available on {self.name}. ' From c907e80c10d5bc8c85e79b356e45333aebe79be4 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Wed, 6 Mar 2019 22:42:05 +0100 Subject: [PATCH 03/12] make sure no dups --- freqtrade/pairlist/IPairList.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index f9a66fe89..8e3b11ff4 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -68,10 +68,10 @@ class IPairList(ABC): markets = self._freqtrade.exchange.markets # Filter to markets in stake currency - stake_pairs = [pair for pair in markets if - pair.endswith(self._config['stake_currency'])] + stake_pairs = [pair for pair in markets if # pair.endswith(self._config['stake_currency']) + markets[pair]["quote"] == self._config['stake_currency']] - sanitized_whitelist = [] + sanitized_whitelist = set() for pair in whitelist: # pair is not in the generated dynamic market, or in the blacklist ... ignore it if pair in self.blacklist or pair not in stake_pairs: @@ -84,7 +84,7 @@ class IPairList(ABC): pair ) continue - sanitized_whitelist.append(pair) + sanitized_whitelist.add(pair) # We need to remove pairs that are unknown - return sanitized_whitelist + return list(sanitized_whitelist) From e38a3051a14a8faca1ef3aac0dcb368c0742b45c Mon Sep 17 00:00:00 2001 From: iuvbio Date: Mon, 11 Mar 2019 21:10:22 +0100 Subject: [PATCH 04/12] update docstring --- freqtrade/pairlist/IPairList.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 8e3b11ff4..841699251 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -60,9 +60,8 @@ class IPairList(ABC): def _validate_whitelist(self, whitelist: List[str]) -> List[str]: """ Check available markets and remove pair from whitelist if necessary - :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to - trade - :return: the list of pairs the user wants to trade without the one unavailable or + :param whitelist: the sorted list of pairs the user might want to trade + :return: the list of pairs the user wants to trade without those unavailable or black_listed """ markets = self._freqtrade.exchange.markets From d4543be8eb2f2d227291857156aba69ea318b5a4 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Mon, 11 Mar 2019 21:48:55 +0100 Subject: [PATCH 05/12] edit comment --- freqtrade/pairlist/IPairList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 841699251..7960e82fc 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -66,7 +66,7 @@ class IPairList(ABC): """ markets = self._freqtrade.exchange.markets - # Filter to markets in stake currency + # keep only pairs with stake currency as quote stake_pairs = [pair for pair in markets if # pair.endswith(self._config['stake_currency']) markets[pair]["quote"] == self._config['stake_currency']] From d4d37667e1d1b912804008a5d248ba8c20b6b10a Mon Sep 17 00:00:00 2001 From: iuvbio Date: Tue, 12 Mar 2019 18:26:47 +0100 Subject: [PATCH 06/12] use pairname for stake cur comparison --- freqtrade/pairlist/IPairList.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 7960e82fc..fcc129814 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -67,8 +67,8 @@ class IPairList(ABC): markets = self._freqtrade.exchange.markets # keep only pairs with stake currency as quote - stake_pairs = [pair for pair in markets if # pair.endswith(self._config['stake_currency']) - markets[pair]["quote"] == self._config['stake_currency']] + stake_pairs = [pair for pair in markets if pair.endswith(self._config['stake_currency'])] + # markets[pair]["quote"] == self._config['stake_currency'] sanitized_whitelist = set() for pair in whitelist: From 7f9c76a6fcfb31591c0568adadcb86da09a49987 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Thu, 14 Mar 2019 21:02:21 +0100 Subject: [PATCH 07/12] move stake check to the same condition as the other checks --- freqtrade/pairlist/IPairList.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index fcc129814..2564c484c 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -66,14 +66,11 @@ class IPairList(ABC): """ markets = self._freqtrade.exchange.markets - # keep only pairs with stake currency as quote - stake_pairs = [pair for pair in markets if pair.endswith(self._config['stake_currency'])] - # markets[pair]["quote"] == self._config['stake_currency'] - sanitized_whitelist = set() for pair in whitelist: # pair is not in the generated dynamic market, or in the blacklist ... ignore it - if pair in self.blacklist or pair not in stake_pairs: + if (pair in self.blacklist or pair not in markets + or not pair.endswith(self._config['stake_currency'])): continue # Check if market is active market = markets[pair] From 8386496456b7d51943b7c4a5bfc710aa1312018a Mon Sep 17 00:00:00 2001 From: iuvbio Date: Thu, 14 Mar 2019 21:53:42 +0100 Subject: [PATCH 08/12] remove tests that are no longer applicable --- freqtrade/tests/exchange/test_exchange.py | 29 ----------------------- 1 file changed, 29 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 7c757df09..736f2298a 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -305,19 +305,6 @@ def test_validate_pairs_not_available(default_conf, mocker): Exchange(default_conf) -def test_validate_pairs_not_compatible(default_conf, mocker): - api_mock = MagicMock() - type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': '', 'TKN/BTC': '', 'TRST/BTC': '', 'SWT/BTC': '', 'BCC/BTC': '' - }) - default_conf['stake_currency'] = 'ETH' - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) - with pytest.raises(OperationalException, match=r'not compatible'): - Exchange(default_conf) - - def test_validate_pairs_exception(default_conf, mocker, caplog): caplog.set_level(logging.INFO) api_mock = MagicMock() @@ -337,22 +324,6 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): caplog.record_tuples) -def test_validate_pairs_stake_exception(default_conf, mocker, caplog): - caplog.set_level(logging.INFO) - default_conf['stake_currency'] = 'ETH' - api_mock = MagicMock() - api_mock.name = MagicMock(return_value='binance') - mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) - - with pytest.raises( - OperationalException, - match=r'Pair ETH/BTC not compatible with stake_currency: ETH' - ): - Exchange(default_conf) - - def test_validate_timeframes(default_conf, mocker): default_conf["ticker_interval"] = "5m" api_mock = MagicMock() From 4de4a70be7f1b63a9eaf180c5b26e1066f83f7e6 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 17 Mar 2019 18:18:35 +0100 Subject: [PATCH 09/12] update log messages --- freqtrade/pairlist/IPairList.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index 2564c484c..d08bd2587 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -71,13 +71,15 @@ class IPairList(ABC): # pair is not in the generated dynamic market, or in the blacklist ... ignore it if (pair in self.blacklist or pair not in markets or not pair.endswith(self._config['stake_currency'])): + logger.warning(f"Pair {pair} is not compatible with exchange " + f"{self._freqtrade.exchange.name} or contained in " + f"your blacklist. Removing it from whitelist..") continue # Check if market is active market = markets[pair] if not market['active']: logger.info( - 'Ignoring %s from whitelist. Market is not active.', - pair + f"Ignoring {pair} from whitelist. Market is not active." ) continue sanitized_whitelist.add(pair) From c2076af43b00ea040cd3b7db9cdf0b013879139b Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 17 Mar 2019 18:18:44 +0100 Subject: [PATCH 10/12] update tests --- freqtrade/tests/pairlist/test_pairlist.py | 74 +++++++++++------------ 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 52f44c41b..5fe2dfc08 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver -from freqtrade.tests.conftest import get_patched_freqtradebot +from freqtrade.tests.conftest import get_patched_freqtradebot, log_has import pytest # whitelist, blacklist @@ -107,7 +107,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): assert set(whitelist) == set(pairslist) -def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) -> None: +@pytest.mark.parametrize("precision_filter,base_currency,key,whitelist_result", [ + (False, "BTC", "quoteVolume", ['ETH/BTC', 'TKN/BTC', 'BTT/BTC']), + (False, "BTC", "bidVolume", ['BTT/BTC', 'TKN/BTC', 'ETH/BTC']), + (False, "USDT", "quoteVolume", ['ETH/USDT', 'LTC/USDT']), + (False, "ETH", "quoteVolume", []), # this replaces tests that were removed from test_exchange + (True, "BTC", "quoteVolume", ["ETH/BTC", "TKN/BTC"]), + (True, "BTC", "bidVolume", ["TKN/BTC", "ETH/BTC"]) +]) +def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers, base_currency, key, + whitelist_result, precision_filter) -> None: whitelist_conf['pairlist']['method'] = 'VolumePairList' mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) @@ -115,32 +124,10 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, markets, tickers) mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, p, r: round(r, 8)) - # Test to retrieved BTC sorted on quoteVolume (default) - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') - assert whitelist == ['ETH/BTC', 'TKN/BTC', 'BTT/BTC'] - - # Test to retrieve BTC sorted on bidVolume - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume') - assert whitelist == ['BTT/BTC', 'TKN/BTC', 'ETH/BTC'] - - # Test with USDT sorted on quoteVolume (default) - freqtrade.config['stake_currency'] = 'USDT' # this has to be set, otherwise markets are removed - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='USDT', key='quoteVolume') - assert whitelist == ['ETH/USDT', 'LTC/USDT'] - - # Test with ETH (our fixture does not have ETH, so result should be empty) - freqtrade.config['stake_currency'] = 'ETH' - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='ETH', key='quoteVolume') - assert whitelist == [] - - freqtrade.pairlists._precision_filter = True - freqtrade.config['stake_currency'] = 'BTC' - # Retest First 2 test-cases to make sure BTT is not in it (too low priced) - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='quoteVolume') - assert whitelist == ['ETH/BTC', 'TKN/BTC'] - - whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency='BTC', key='bidVolume') - assert whitelist == ['TKN/BTC', 'ETH/BTC'] + freqtrade.pairlists._precision_filter = precision_filter + freqtrade.config['stake_currency'] = base_currency + whitelist = freqtrade.pairlists._gen_pair_whitelist(base_currency=base_currency, key=key) + assert whitelist == whitelist_result def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: @@ -166,17 +153,24 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): assert isinstance(freqtrade.pairlists.whitelist, list) assert isinstance(freqtrade.pairlists.blacklist, list) - whitelist = ['ETH/BTC', 'TKN/BTC'] + +@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +@pytest.mark.parametrize("whitelist,log_message", [ + (['ETH/BTC', 'TKN/BTC'], ""), + (['ETH/BTC', 'TKN/BTC', 'TRX/ETH'], "is not compatible with exchange"), # TRX/ETH wrong stake + (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BCH/BTC not available + (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "is not compatible with exchange"), # BLK/BTC in blacklist + (['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], "Market is not active") # LTC/BTC is inactive +]) +def test_validate_whitelist(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, + log_message): + whitelist_conf['pairlist']['method'] = pairlist + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + caplog.clear() + new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) - assert set(whitelist) == set(new_whitelist) - - whitelist = ['ETH/BTC', 'TKN/BTC', 'TRX/ETH'] - new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) - # TRX/ETH was removed - assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist) - - whitelist = ['ETH/BTC', 'TKN/BTC', 'BLK/BTC'] - new_whitelist = freqtrade.pairlists._validate_whitelist(whitelist) - # BLK/BTC is in blacklist ... - assert set(['ETH/BTC', 'TKN/BTC']) == set(new_whitelist) + assert set(new_whitelist) == set(['ETH/BTC', 'TKN/BTC']) + assert log_message in caplog.text From 937399606e98e0110d27e0e0e804128622adbde3 Mon Sep 17 00:00:00 2001 From: iuvbio Date: Sun, 17 Mar 2019 18:24:29 +0100 Subject: [PATCH 11/12] fix flake8 --- freqtrade/tests/pairlist/test_pairlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/pairlist/test_pairlist.py b/freqtrade/tests/pairlist/test_pairlist.py index 5fe2dfc08..38a8d78c7 100644 --- a/freqtrade/tests/pairlist/test_pairlist.py +++ b/freqtrade/tests/pairlist/test_pairlist.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.resolvers import PairListResolver -from freqtrade.tests.conftest import get_patched_freqtradebot, log_has +from freqtrade.tests.conftest import get_patched_freqtradebot import pytest # whitelist, blacklist From 7fdb099097c0afb373ee4233036403c095249d54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 21 Mar 2019 06:14:43 +0100 Subject: [PATCH 12/12] Reformat log statement --- freqtrade/pairlist/IPairList.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index d08bd2587..a112c63b4 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -78,9 +78,7 @@ class IPairList(ABC): # Check if market is active market = markets[pair] if not market['active']: - logger.info( - f"Ignoring {pair} from whitelist. Market is not active." - ) + logger.info(f"Ignoring {pair} from whitelist. Market is not active.") continue sanitized_whitelist.add(pair)