From 52e267e864cb04cd95715255f203d72ae4f269d2 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 2 Jan 2018 12:04:47 +0100 Subject: [PATCH 1/7] fix for issue #283 --- freqtrade/main.py | 15 ++++++++++----- freqtrade/tests/test_acl_pair.py | 12 +++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index ff5d4b5aa..9caa54502 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -27,23 +27,28 @@ _CONF = {} def refresh_whitelist(whitelist: List[str]) -> List[str]: """ Check wallet health and remove pair from whitelist if necessary - :param whitelist: the pair the user might want to trade + :param whitelist: the sorted list (based on BaseVolume) of pairs the user might want to trade :return: the list of pairs the user wants to trade without the one unavailable or black_listed """ - sanitized_whitelist = [] + sanitized_whitelist = whitelist health = exchange.get_wallet_health() + known_pairs = set() for status in health: pair = '{}_{}'.format(_CONF['stake_currency'], status['Currency']) + known_pairs.add(pair) if pair not in whitelist or pair in _CONF['exchange'].get('pair_blacklist', []): continue - if status['IsActive']: - sanitized_whitelist.append(pair) + if not status['IsActive']: + sanitized_whitelist.remove(pair) else: logger.info( 'Ignoring %s from whitelist (reason: %s).', pair, status.get('Notice') or 'wallet is not active' ) - return sanitized_whitelist + + # We need to remove pairs that are unknown + final_list = [x for x in sanitized_whitelist if x in known_pairs] + return final_list def _process(nb_assets: Optional[int] = 0) -> bool: diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 0067eb302..acb7acc6e 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -22,10 +22,12 @@ def whitelist_conf(): def get_health(): return [{'Currency': 'ETH', - 'IsActive': True + 'IsActive': True, + 'BaseVolume' : 42 }, {'Currency': 'TKN', - 'IsActive': True + 'IsActive': True, + 'BaseVolume' : 1664 }] @@ -44,7 +46,7 @@ def test_refresh_whitelist(mocker): refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']) whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed - assert set(whitelist) == set(refreshedwhitelist) + assert whitelist == refreshedwhitelist def test_refresh_whitelist_dynamic(mocker): @@ -53,9 +55,9 @@ def test_refresh_whitelist_dynamic(mocker): mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) # argument: use the whitelist dynamically by exchange-volume - whitelist = ['BTC_ETH', 'BTC_TKN'] + whitelist = ['BTC_TKN', 'BTC_ETH'] refreshedwhitelist = refresh_whitelist(whitelist) - assert set(whitelist) == set(refreshedwhitelist) + assert whitelist == refreshedwhitelist def test_refresh_whitelist_dynamic_empty(mocker): From a3e827c144a06a8141f008c31d088b30fda11eae Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 2 Jan 2018 12:18:26 +0100 Subject: [PATCH 2/7] with flake8 code review --- freqtrade/tests/test_acl_pair.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index acb7acc6e..1a7f2aaea 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -22,12 +22,12 @@ def whitelist_conf(): def get_health(): return [{'Currency': 'ETH', - 'IsActive': True, - 'BaseVolume' : 42 + 'IsActive': True, + 'BaseVolume': 42 }, {'Currency': 'TKN', 'IsActive': True, - 'BaseVolume' : 1664 + 'BaseVolume': 1664 }] From 5344b711eabfe01331cbb00cb20668cb45568909 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 2 Jan 2018 13:42:10 +0100 Subject: [PATCH 3/7] Add two more unit tests for covering pair that are in a blacklist, and unknown pairs in the conf --- freqtrade/tests/test_acl_pair.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 1a7f2aaea..7d28b9a21 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -16,6 +16,9 @@ def whitelist_conf(): "BTC_SWT", "BTC_BCC" ], + "pair_blacklist": [ + "BTC_BLK" + ], }, } @@ -28,15 +31,27 @@ def get_health(): {'Currency': 'TKN', 'IsActive': True, 'BaseVolume': 1664 - }] + }, + {'Currency': 'BLK', + 'IsActive': True, + 'BaseVolume': 4096 + } +] def get_health_empty(): return [] -# below three test could be merged into a single -# test that ran randomlly generated health lists - +def test_refresh_market_pair_not_in_whitelist(mocker): + conf = whitelist_conf() + mocker.patch.dict('freqtrade.main._CONF', conf) + mocker.patch.multiple('freqtrade.main.exchange', + get_wallet_health=get_health) + refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']+ ['BTC_XXX']) + # List ordered by BaseVolume + whitelist = ['BTC_ETH', 'BTC_TKN'] + # Ensure all except those in whitelist are removed + assert whitelist == refreshedwhitelist def test_refresh_whitelist(mocker): conf = whitelist_conf() @@ -44,6 +59,7 @@ def test_refresh_whitelist(mocker): mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']) + # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed assert whitelist == refreshedwhitelist From 3f65fc014eebca5ea425f75b6213e1fe70476b07 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 2 Jan 2018 13:46:16 +0100 Subject: [PATCH 4/7] flake8 on tests --- freqtrade/tests/test_acl_pair.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_acl_pair.py b/freqtrade/tests/test_acl_pair.py index 7d28b9a21..1368ac25c 100644 --- a/freqtrade/tests/test_acl_pair.py +++ b/freqtrade/tests/test_acl_pair.py @@ -16,9 +16,9 @@ def whitelist_conf(): "BTC_SWT", "BTC_BCC" ], - "pair_blacklist": [ + "pair_blacklist": [ "BTC_BLK" - ], + ], }, } @@ -36,23 +36,25 @@ def get_health(): 'IsActive': True, 'BaseVolume': 4096 } -] + ] def get_health_empty(): return [] + def test_refresh_market_pair_not_in_whitelist(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) mocker.patch.multiple('freqtrade.main.exchange', get_wallet_health=get_health) - refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist']+ ['BTC_XXX']) + refreshedwhitelist = refresh_whitelist(conf['exchange']['pair_whitelist'] + ['BTC_XXX']) # List ordered by BaseVolume whitelist = ['BTC_ETH', 'BTC_TKN'] # Ensure all except those in whitelist are removed assert whitelist == refreshedwhitelist + def test_refresh_whitelist(mocker): conf = whitelist_conf() mocker.patch.dict('freqtrade.main._CONF', conf) From 90236fb537b5cf0c42ec7536de22a4e52ec14583 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Tue, 2 Jan 2018 15:15:59 +0100 Subject: [PATCH 5/7] Fixing error log on inactive wallet --- freqtrade/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 9caa54502..7fc7654ed 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -40,7 +40,6 @@ def refresh_whitelist(whitelist: List[str]) -> List[str]: continue if not status['IsActive']: sanitized_whitelist.remove(pair) - else: logger.info( 'Ignoring %s from whitelist (reason: %s).', pair, status.get('Notice') or 'wallet is not active' From ae52880f819a0df69f9fa6c8fe2ce6e01bcaaadf Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 2 Jan 2018 16:40:33 +0200 Subject: [PATCH 6/7] improve backtesting result formatting --- freqtrade/optimize/backtesting.py | 18 ++++++++++-------- freqtrade/tests/optimize/test_backtesting.py | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 984ca3e72..0a09282b6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -41,27 +41,29 @@ def generate_text_table( Generates and returns a text table for the given backtest data and the results dataframe :return: pretty printed table with tabulate as str """ + floatfmt = ('s', 'd', '.2f', '.8f', '.1f') tabular_data = [] - headers = ['pair', 'buy count', 'avg profit', 'total profit', 'avg duration'] + headers = ['pair', 'buy count', 'avg profit %', + 'total profit ' + stake_currency, 'avg duration'] for pair in data: result = results[results.currency == pair] tabular_data.append([ pair, len(result.index), - '{:.2f}%'.format(result.profit_percent.mean() * 100.0), - '{:.08f} {}'.format(result.profit_BTC.sum(), stake_currency), - '{:.2f}'.format(result.duration.mean() * ticker_interval), + result.profit_percent.mean() * 100.0, + result.profit_BTC.sum(), + result.duration.mean() * ticker_interval, ]) # Append Total tabular_data.append([ 'TOTAL', len(results.index), - '{:.2f}%'.format(results.profit_percent.mean() * 100.0), - '{:.08f} {}'.format(results.profit_BTC.sum(), stake_currency), - '{:.2f}'.format(results.duration.mean() * ticker_interval), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.duration.mean() * ticker_interval, ]) - return tabulate(tabular_data, headers=headers) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) def backtest(stake_amount: float, processed: Dict[str, DataFrame], diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 94c062cac..e3e34de1d 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -18,11 +18,12 @@ def test_generate_text_table(): 'duration': [10, 30] } ) + print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5)) assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == ( - 'pair buy count avg profit total profit avg duration\n' - '------- ----------- ------------ -------------- --------------\n' - 'BTC_ETH 2 15.00% 0.60000000 BTC 100\n' - 'TOTAL 2 15.00% 0.60000000 BTC 100') + 'pair buy count avg profit % total profit BTC avg duration\n' + '------- ----------- -------------- ------------------ --------------\n' + 'BTC_ETH 2 15.00 0.60000000 100.0\n' + 'TOTAL 2 15.00 0.60000000 100.0') def test_get_timeframe(): From 82e9ed2ac291ad7ee72d39ab6ddc51916488ed5d Mon Sep 17 00:00:00 2001 From: Janne Sinivirta Date: Tue, 2 Jan 2018 17:53:47 +0200 Subject: [PATCH 7/7] shorten table title to match table length --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0a09282b6..47be60802 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -175,6 +175,6 @@ def start(args): config['stake_amount'], preprocess(data), max_open_trades, args.realistic_simulation ) logger.info( - '\n====================== BACKTESTING REPORT ======================================\n%s', + '\n====================== BACKTESTING REPORT ================================\n%s', generate_text_table(data, results, config['stake_currency'], args.ticker_interval) )