From c176ace88906c3c2ce371199cf49199c82511822 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 3 Jan 2018 10:56:18 +0100 Subject: [PATCH 1/6] Adding sell_profit_only and stoploss in hyperopt --- freqtrade/optimize/backtesting.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6f7f14b8c..e87af5aad 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -64,7 +64,7 @@ def generate_text_table( def backtest(stake_amount: float, processed: Dict[str, DataFrame], - max_open_trades: int = 0, realistic: bool = True) -> DataFrame: + max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False, stoploss: int = -1.00) -> DataFrame: """ Implements backtesting functionality :param stake_amount: btc amount to use for each trade @@ -110,8 +110,10 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], # Increase trade_count_lock for every iteration trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 - if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1: - current_profit_percent = trade.calc_profit_percent(rate=row2.close) + current_profit_percent = trade.calc_profit_percent(rate=row2.close) + if (sell_profit_only and current_profit_percent < 0) : + continue + if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1 or current_profit_percent < stoploss: current_profit_btc = trade.calc_profit(rate=row2.close) lock_pair_until = row2.Index @@ -172,7 +174,7 @@ def start(args): # Execute backtest and print results results = backtest( - config['stake_amount'], preprocessed, max_open_trades, args.realistic_simulation + config['stake_amount'], preprocessed, max_open_trades, args.realistic_simulation, config.get('experimental',{}).get('sell_profit_only', False), config.get('stoploss') ) logger.info( '\n====================== BACKTESTING REPORT ================================\n%s', From 45f2d018959edf180cc79f940edb41a0504621cb Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 3 Jan 2018 11:19:46 +0100 Subject: [PATCH 2/6] - add a profit/loss counter - the use of the sell_signal is conditional now (taken from the config) --- freqtrade/optimize/backtesting.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e87af5aad..491e5c573 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -41,7 +41,7 @@ def generate_text_table( floatfmt = ('s', 'd', '.2f', '.8f', '.1f') tabular_data = [] headers = ['pair', 'buy count', 'avg profit %', - 'total profit ' + stake_currency, 'avg duration'] + 'total profit ' + stake_currency, 'avg duration', 'profit','loss'] for pair in data: result = results[results.currency == pair] tabular_data.append([ @@ -50,6 +50,8 @@ def generate_text_table( result.profit_percent.mean() * 100.0, result.profit_BTC.sum(), result.duration.mean() * ticker_interval, + result.profit.sum(), + result.loss.sum() ]) # Append Total @@ -59,12 +61,15 @@ def generate_text_table( results.profit_percent.mean() * 100.0, results.profit_BTC.sum(), results.duration.mean() * ticker_interval, + results.profit.sum(), + results.loss.sum(), + ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) def backtest(stake_amount: float, processed: Dict[str, DataFrame], - max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False, stoploss: int = -1.00) -> DataFrame: + max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False, stoploss: int = -1.00, use_sell_signal: bool = False) -> DataFrame: """ Implements backtesting functionality :param stake_amount: btc amount to use for each trade @@ -113,7 +118,7 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], current_profit_percent = trade.calc_profit_percent(rate=row2.close) if (sell_profit_only and current_profit_percent < 0) : continue - if min_roi_reached(trade, row2.close, row2.date) or row2.sell == 1 or current_profit_percent < stoploss: + if min_roi_reached(trade, row2.close, row2.date) or (row2.sell == 1 and use_sell_signal) or current_profit_percent <= stoploss: current_profit_btc = trade.calc_profit(rate=row2.close) lock_pair_until = row2.Index @@ -122,11 +127,13 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], pair, current_profit_percent, current_profit_btc, - row2.Index - row.Index + row2.Index - row.Index, + current_profit_btc > 0, + current_profit_btc < 0 ) ) break - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration'] + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit','loss'] return DataFrame.from_records(trades, columns=labels) @@ -174,7 +181,7 @@ def start(args): # Execute backtest and print results results = backtest( - config['stake_amount'], preprocessed, max_open_trades, args.realistic_simulation, config.get('experimental',{}).get('sell_profit_only', False), config.get('stoploss') + config['stake_amount'], preprocessed, max_open_trades, args.realistic_simulation, config.get('experimental',{}).get('sell_profit_only', False), config.get('stoploss'), config.get('experimental',{}).get('use_sell_signal',False) ) logger.info( '\n====================== BACKTESTING REPORT ================================\n%s', From 2d273a8509c234c4f8e0c03c398752b102bee3ec Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 3 Jan 2018 11:30:24 +0100 Subject: [PATCH 3/6] Update unittests --- freqtrade/tests/optimize/test_backtesting.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 8b416e64e..3b5cd5fd6 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -16,15 +16,17 @@ def test_generate_text_table(): 'currency': ['BTC_ETH', 'BTC_ETH'], 'profit_percent': [0.1, 0.2], 'profit_BTC': [0.2, 0.4], - 'duration': [10, 30] + 'duration': [10, 30], + 'profit': [2,0], + 'loss': [0,0] } ) print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5)) assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == ( - 'pair buy count avg profit % total profit BTC avg duration\n' - '------- ----------- -------------- ------------------ --------------\n' - 'BTC_ETH 2 15.00 0.60000000 100.0\n' - 'TOTAL 2 15.00 0.60000000 100.0') + 'pair buy count avg profit % total profit BTC avg duration profit loss\n' + '------- ----------- -------------- ------------------ -------------- -------- ------\n' + 'BTC_ETH 2 15.00 0.60000000 100.0 2 0\n' + 'TOTAL 2 15.00 0.60000000 100.0 2 0') def test_get_timeframe(): From eb53a796e2ba8043bb6ecca34d9a4d43aaf05d9c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 3 Jan 2018 11:35:54 +0100 Subject: [PATCH 4/6] pep8 compliance --- freqtrade/optimize/backtesting.py | 22 ++++++++++++-------- freqtrade/tests/optimize/test_backtesting.py | 4 ++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 491e5c573..50cb570b9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -41,7 +41,7 @@ def generate_text_table( floatfmt = ('s', 'd', '.2f', '.8f', '.1f') tabular_data = [] headers = ['pair', 'buy count', 'avg profit %', - 'total profit ' + stake_currency, 'avg duration', 'profit','loss'] + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: result = results[results.currency == pair] tabular_data.append([ @@ -62,14 +62,14 @@ def generate_text_table( results.profit_BTC.sum(), results.duration.mean() * ticker_interval, results.profit.sum(), - results.loss.sum(), - + results.loss.sum() ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt) def backtest(stake_amount: float, processed: Dict[str, DataFrame], - max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False, stoploss: int = -1.00, use_sell_signal: bool = False) -> DataFrame: + max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False, + stoploss: int = -1.00, use_sell_signal: bool = False) -> DataFrame: """ Implements backtesting functionality :param stake_amount: btc amount to use for each trade @@ -116,9 +116,11 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1 current_profit_percent = trade.calc_profit_percent(rate=row2.close) - if (sell_profit_only and current_profit_percent < 0) : + if (sell_profit_only and current_profit_percent < 0): continue - if min_roi_reached(trade, row2.close, row2.date) or (row2.sell == 1 and use_sell_signal) or current_profit_percent <= stoploss: + if min_roi_reached(trade, row2.close, row2.date) + or (row2.sell == 1 and use_sell_signal) + or current_profit_percent <= stoploss: current_profit_btc = trade.calc_profit(rate=row2.close) lock_pair_until = row2.Index @@ -127,13 +129,13 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], pair, current_profit_percent, current_profit_btc, - row2.Index - row.Index, + row2.Index - row.Index, current_profit_btc > 0, current_profit_btc < 0 ) ) break - labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit','loss'] + labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit', 'loss'] return DataFrame.from_records(trades, columns=labels) @@ -181,7 +183,9 @@ def start(args): # Execute backtest and print results results = backtest( - config['stake_amount'], preprocessed, max_open_trades, args.realistic_simulation, config.get('experimental',{}).get('sell_profit_only', False), config.get('stoploss'), config.get('experimental',{}).get('use_sell_signal',False) + config['stake_amount'], preprocessed, max_open_trades, args.realistic_simulation, + config.get('experimental', {}).get('sell_profit_only', False), config.get('stoploss'), + config.get('experimental', {}).get('use_sell_signal', False) ) logger.info( '\n====================== BACKTESTING REPORT ================================\n%s', diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 3b5cd5fd6..9b7b32198 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -17,8 +17,8 @@ def test_generate_text_table(): 'profit_percent': [0.1, 0.2], 'profit_BTC': [0.2, 0.4], 'duration': [10, 30], - 'profit': [2,0], - 'loss': [0,0] + 'profit': [2, 0], + 'loss': [0, 0] } ) print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5)) From ea6a1c629d9bb0489d6540fd621cdcd9e12663ff Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 3 Jan 2018 11:50:30 +0100 Subject: [PATCH 5/6] fixing pep8 compliance --- freqtrade/optimize/backtesting.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 50cb570b9..54cf59ac7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -118,23 +118,23 @@ def backtest(stake_amount: float, processed: Dict[str, DataFrame], current_profit_percent = trade.calc_profit_percent(rate=row2.close) if (sell_profit_only and current_profit_percent < 0): continue - if min_roi_reached(trade, row2.close, row2.date) - or (row2.sell == 1 and use_sell_signal) - or current_profit_percent <= stoploss: - current_profit_btc = trade.calc_profit(rate=row2.close) - lock_pair_until = row2.Index + if min_roi_reached(trade, row2.close, row2.date) or \ + (row2.sell == 1 and use_sell_signal) or \ + current_profit_percent <= stoploss: + current_profit_btc = trade.calc_profit(rate=row2.close) + lock_pair_until = row2.Index - trades.append( - ( - pair, - current_profit_percent, - current_profit_btc, - row2.Index - row.Index, - current_profit_btc > 0, - current_profit_btc < 0 + trades.append( + ( + pair, + current_profit_percent, + current_profit_btc, + row2.Index - row.Index, + current_profit_btc > 0, + current_profit_btc < 0 + ) ) - ) - break + break labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit', 'loss'] return DataFrame.from_records(trades, columns=labels) From 0f2d3adbbc9c4e81cf0b1ee9c95b3447519a564d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste LE STANG Date: Wed, 3 Jan 2018 17:36:40 +0100 Subject: [PATCH 6/6] applying pep8 --- freqtrade/tests/optimize/test_backtesting.py | 41 +++++++++++--------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 9b7b32198..a24042c62 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -23,14 +23,15 @@ def test_generate_text_table(): ) print(generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5)) assert generate_text_table({'BTC_ETH': {}}, results, 'BTC', 5) == ( - 'pair buy count avg profit % total profit BTC avg duration profit loss\n' - '------- ----------- -------------- ------------------ -------------- -------- ------\n' - 'BTC_ETH 2 15.00 0.60000000 100.0 2 0\n' - 'TOTAL 2 15.00 0.60000000 100.0 2 0') + 'pair buy count avg profit % total profit BTC avg duration profit loss\n' # noqa + '------- ----------- -------------- ------------------ -------------- -------- ------\n' # noqa + 'BTC_ETH 2 15.00 0.60000000 100.0 2 0\n' # noqa + 'TOTAL 2 15.00 0.60000000 100.0 2 0') # noqa def test_get_timeframe(): - data = preprocess(optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST'])) + data = preprocess(optimize.load_data( + ticker_interval=1, pairs=['BTC_UNITEST'])) min_date, max_date = get_timeframe(data) assert min_date.isoformat() == '2017-11-04T23:02:00+00:00' assert max_date.isoformat() == '2017-11-14T22:59:00+00:00' @@ -41,7 +42,8 @@ def test_backtest(default_conf, mocker): exchange._API = Bittrex({'key': '', 'secret': ''}) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) - results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) + results = backtest(default_conf['stake_amount'], + optimize.preprocess(data), 10, True) assert not results.empty @@ -51,7 +53,8 @@ def test_backtest_1min_ticker_interval(default_conf, mocker): # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(ticker_interval=1, pairs=['BTC_UNITEST']) - results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 1, True) + results = backtest(default_conf['stake_amount'], + optimize.preprocess(data), 1, True) assert not results.empty @@ -76,13 +79,13 @@ def load_data_test(what): base = 0.001 if what == 'raise': return {'BTC_UNITEST': - [{'T': pair[x]['T'], # Keep old dates - 'V': pair[x]['V'], # Keep old volume + [{'T': pair[x]['T'], # Keep old dates + 'V': pair[x]['V'], # Keep old volume 'BV': pair[x]['BV'], # keep too - 'O': x * base, # But replace O,H,L,C - 'H': x * base + 0.0001, - 'L': x * base - 0.0001, - 'C': x * base} for x in range(0, datalen)]} + 'O': x * base, # But replace O,H,L,C + 'H': x * base + 0.0001, + 'L': x * base - 0.0001, + 'C': x * base} for x in range(0, datalen)]} if what == 'lower': return {'BTC_UNITEST': [{'T': pair[x]['T'], # Keep old dates @@ -98,10 +101,11 @@ def load_data_test(what): [{'T': pair[x]['T'], # Keep old dates 'V': pair[x]['V'], # Keep old volume 'BV': pair[x]['BV'], # keep too - 'O': math.sin(x*hz) / 1000 + base, # But replace O,H,L,C - 'H': math.sin(x*hz) / 1000 + base + 0.0001, - 'L': math.sin(x*hz) / 1000 + base - 0.0001, - 'C': math.sin(x*hz) / 1000 + base} for x in range(0, datalen)]} + # But replace O,H,L,C + 'O': math.sin(x * hz) / 1000 + base, + 'H': math.sin(x * hz) / 1000 + base + 0.0001, + 'L': math.sin(x * hz) / 1000 + base - 0.0001, + 'C': math.sin(x * hz) / 1000 + base} for x in range(0, datalen)]} return data @@ -121,7 +125,8 @@ def simple_backtest(config, contour, num_results): def test_backtest2(default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = optimize.load_data(ticker_interval=5, pairs=['BTC_ETH']) - results = backtest(default_conf['stake_amount'], optimize.preprocess(data), 10, True) + results = backtest(default_conf['stake_amount'], + optimize.preprocess(data), 10, True) assert not results.empty