From 309186911590630cb01bdd503da840c9eca8518c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Dec 2019 14:30:14 +0100 Subject: [PATCH 1/6] refactor get_close_rate out of get_sell_trade-entry --- freqtrade/optimize/backtesting.py | 44 +++++++++++++++++-------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d9fb1f2d1..c7d25cb7b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -261,6 +261,29 @@ class Backtesting: ticker[pair] = [x for x in ticker_data.itertuples()] return ticker + def _get_close_rate(self, sell_row, trade: Trade, sell, trade_dur) -> float: + # Special handling if high or low hit STOP_LOSS or ROI + if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + # Set close_rate to stoploss + closerate = trade.stop_loss + elif sell.sell_type == (SellType.ROI): + roi = self.strategy.min_roi_reached_entry(trade_dur) + if roi is not None: + # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) + closerate = - (trade.open_rate * roi + trade.open_rate * + (1 + trade.fee_open)) / (trade.fee_close - 1) + + # Use the maximum between closerate and low as we + # cannot sell outside of a candle. + # Applies when using {"xx": -1} as roi to force sells after xx minutes + closerate = max(closerate, sell_row.low) + else: + # This should not be reached... + closerate = sell_row.open + else: + closerate = sell_row.open + return closerate + def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, partial_ticker: List, trade_count_lock: Dict, @@ -287,26 +310,7 @@ class Backtesting: sell_row.sell, low=sell_row.low, high=sell_row.high) if sell.sell_flag: trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60) - # Special handling if high or low hit STOP_LOSS or ROI - if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): - # Set close_rate to stoploss - closerate = trade.stop_loss - elif sell.sell_type == (SellType.ROI): - roi = self.strategy.min_roi_reached_entry(trade_dur) - if roi is not None: - # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) - closerate = - (trade.open_rate * roi + trade.open_rate * - (1 + trade.fee_open)) / (trade.fee_close - 1) - - # Use the maximum between closerate and low as we - # cannot sell outside of a candle. - # Applies when using {"xx": -1} as roi to force sells after xx minutes - closerate = max(closerate, sell_row.low) - else: - # This should not be reached... - closerate = sell_row.open - else: - closerate = sell_row.open + closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=closerate), From 1e6f9f9fe20cab92711b06f929e5641991c8ea60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Dec 2019 15:13:49 +0100 Subject: [PATCH 2/6] Add testcase for negative ROI sell using open --- tests/optimize/test_backtest_detail.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 3f6cc8c9a..e269009b1 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -265,6 +265,22 @@ tc16 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] ) +# Test 17: Buy, hold for 120 mins, then forcesell using roi=-1 +# Causes negative profit even though sell-reason is ROI. +# stop-loss: 10%, ROI: 10% (should not apply), -100% after 100 minutes (limits trade duration) +# Uses open as sell-rate (special case) - since the roi-time is a multiple of the ticker interval. +tc17 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], + [2, 4987, 5300, 4950, 5050, 6172, 0, 0], + [3, 4980, 5000, 4940, 4962, 6172, 0, 0], # ForceSell on ROI (roi=-1) + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10, "120": -1}, profit_perc=-0.004, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + TESTS = [ tc0, tc1, @@ -283,6 +299,7 @@ TESTS = [ tc14, tc15, tc16, + tc17, ] From 3163cbdf8a56f2eafefa5736945087327752ed10 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Dec 2019 15:18:12 +0100 Subject: [PATCH 3/6] Apply special case for negative ROI --- freqtrade/optimize/backtesting.py | 10 ++++++++-- freqtrade/strategy/interface.py | 8 ++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c7d25cb7b..6e7d36368 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -267,7 +267,7 @@ class Backtesting: # Set close_rate to stoploss closerate = trade.stop_loss elif sell.sell_type == (SellType.ROI): - roi = self.strategy.min_roi_reached_entry(trade_dur) + roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) if roi is not None: # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) closerate = - (trade.open_rate * roi + trade.open_rate * @@ -275,8 +275,14 @@ class Backtesting: # Use the maximum between closerate and low as we # cannot sell outside of a candle. - # Applies when using {"xx": -1} as roi to force sells after xx minutes + # Applies when a new ROI setting comes in place and the whole candle is above that. closerate = max(closerate, sell_row.low) + if roi == -1 and roi_entry % self.timeframe_mins == 0: + # When forceselling with ROI=-1, the roi-entry will always be "on the entry". + # If that entry is a multiple of the timeframe (so on open) + # - we'll use open instead of close + closerate = max(closerate, sell_row.open) + else: # This should not be reached... closerate = sell_row.open diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e208138e7..2b3a6194f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -394,7 +394,7 @@ class IStrategy(ABC): return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) - def min_roi_reached_entry(self, trade_dur: int) -> Optional[float]: + def min_roi_reached_entry(self, trade_dur: int) -> Tuple[Optional[int], Optional[float]]: """ Based on trade duration defines the ROI entry that may have been reached. :param trade_dur: trade duration in minutes @@ -403,9 +403,9 @@ class IStrategy(ABC): # Get highest entry in ROI dict where key <= trade-duration roi_list = list(filter(lambda x: x <= trade_dur, self.minimal_roi.keys())) if not roi_list: - return None + return None, None roi_entry = max(roi_list) - return self.minimal_roi[roi_entry] + return roi_entry, self.minimal_roi[roi_entry] def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: """ @@ -415,7 +415,7 @@ class IStrategy(ABC): """ # Check if time matches and current rate is above threshold trade_dur = int((current_time.timestamp() - trade.open_date.timestamp()) // 60) - roi = self.min_roi_reached_entry(trade_dur) + _, roi = self.min_roi_reached_entry(trade_dur) if roi is None: return False else: From 189835b9635e54ce10c88ae02ba0ce4936c7749d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Dec 2019 15:26:10 +0100 Subject: [PATCH 4/6] Add documentation for ROI-1 case --- docs/backtesting.md | 5 ++++- docs/configuration.md | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 68782bb9c..017289905 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -194,7 +194,10 @@ Since backtesting lacks some detailed information about what happens within a ca - Buys happen at open-price - Sell signal sells happen at open-price of the following candle - Low happens before high for stoploss, protecting capital first. -- ROI sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) +- ROI + - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) + - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit + - Forcesells caused by `=-1` ROI entries use low as sell value, unless N falls on the candle open (e.g. `120: -1` for 1h candles) - Stoploss sells happen exactly at stoploss price, even if low was lower - Trailing stoploss - High happens first - adjusting stoploss diff --git a/docs/configuration.md b/docs/configuration.md index 5ad1a886e..5bebbcfcd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -169,6 +169,9 @@ This parameter can be set in either Strategy or Configuration file. If you use i `minimal_roi` value from the strategy file. If it is not set in either Strategy or Configuration, a default of 1000% `{"0": 10}` is used, and minimal roi is disabled unless your trade generates 1000% profit. +!!! Note "Special case to forcesell after a specific time" + A special case presents using `"": -1` as ROI. This forces the bot to sell a trade after N Minutes, no matter if it's positive or negative, so represents a time-limited force-sell. + ### Understand stoploss Go to the [stoploss documentation](stoploss.md) for more details. From 45d12dbc83fd6a24f1d7b28bcf6bf5022053d454 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 Dec 2019 15:28:56 +0100 Subject: [PATCH 5/6] Avoid a few calculations during backtesting --- freqtrade/optimize/backtesting.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6e7d36368..7b5bd34b4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -262,13 +262,22 @@ class Backtesting: return ticker def _get_close_rate(self, sell_row, trade: Trade, sell, trade_dur) -> float: + """ + Get close rate for backtesting result + """ # Special handling if high or low hit STOP_LOSS or ROI if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): # Set close_rate to stoploss - closerate = trade.stop_loss + return trade.stop_loss elif sell.sell_type == (SellType.ROI): roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) if roi is not None: + if roi == -1 and roi_entry % self.timeframe_mins == 0: + # When forceselling with ROI=-1, the roi-entry will always be "on the entry". + # If that entry is a multiple of the timeframe (so on open) + # - we'll use open instead of close + return sell_row.open + # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) closerate = - (trade.open_rate * roi + trade.open_rate * (1 + trade.fee_open)) / (trade.fee_close - 1) @@ -276,19 +285,13 @@ class Backtesting: # Use the maximum between closerate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - closerate = max(closerate, sell_row.low) - if roi == -1 and roi_entry % self.timeframe_mins == 0: - # When forceselling with ROI=-1, the roi-entry will always be "on the entry". - # If that entry is a multiple of the timeframe (so on open) - # - we'll use open instead of close - closerate = max(closerate, sell_row.open) + return max(closerate, sell_row.low) else: # This should not be reached... - closerate = sell_row.open + return sell_row.open else: - closerate = sell_row.open - return closerate + return sell_row.open def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, From de33ec425086e1aacc093b6cea0694ca549db5cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 9 Dec 2019 16:52:12 +0100 Subject: [PATCH 6/6] use sell_row.open also when the active ROI value just changed --- freqtrade/optimize/backtesting.py | 19 ++++++---- tests/optimize/test_backtest_detail.py | 50 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7b5bd34b4..4e86acdc6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -273,19 +273,26 @@ class Backtesting: roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) if roi is not None: if roi == -1 and roi_entry % self.timeframe_mins == 0: - # When forceselling with ROI=-1, the roi-entry will always be "on the entry". - # If that entry is a multiple of the timeframe (so on open) + # When forceselling with ROI=-1, the roi time will always be equal to trade_dur. + # If that entry is a multiple of the timeframe (so on candle open) # - we'll use open instead of close return sell_row.open # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) - closerate = - (trade.open_rate * roi + trade.open_rate * - (1 + trade.fee_open)) / (trade.fee_close - 1) + close_rate = - (trade.open_rate * roi + trade.open_rate * + (1 + trade.fee_open)) / (trade.fee_close - 1) - # Use the maximum between closerate and low as we + if (trade_dur > 0 and trade_dur == roi_entry + and roi_entry % self.timeframe_mins == 0 + and sell_row.open > close_rate): + # new ROI entry came into effect. + # use Open rate if open_rate > calculated sell rate + return sell_row.open + + # Use the maximum between close_rate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - return max(closerate, sell_row.low) + return max(close_rate, sell_row.low) else: # This should not be reached... diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index e269009b1..eff236a5e 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -281,6 +281,53 @@ tc17 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] ) + +# Test 18: Buy, hold for 120 mins, then drop ROI to 1%, causing a sell in candle 3. +# stop-loss: 10%, ROI: 10% (should not apply), -100% after 100 minutes (limits trade duration) +# uses open_rate as sell-price +tc18 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], + [2, 4987, 5300, 4950, 5200, 6172, 0, 0], + [3, 5200, 5220, 4940, 4962, 6172, 0, 0], # Sell on ROI (sells on open) + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10, "120": 0.01}, profit_perc=0.04, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + +# Test 19: Buy, hold for 119 mins, then drop ROI to 1%, causing a sell in candle 3. +# stop-loss: 10%, ROI: 10% (should not apply), -100% after 100 minutes (limits trade duration) +# uses calculated ROI (1%) as sell rate, otherwise identical to tc18 +tc19 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], + [2, 4987, 5300, 4950, 5200, 6172, 0, 0], + [3, 5000, 5300, 4940, 4962, 6172, 0, 0], # Sell on ROI + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4550, 4975, 4925, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10, "120": 0.01}, profit_perc=0.01, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + +# Test 20: Buy, hold for 119 mins, then drop ROI to 1%, causing a sell in candle 3. +# stop-loss: 10%, ROI: 10% (should not apply), -100% after 100 minutes (limits trade duration) +# uses calculated ROI (1%) as sell rate, otherwise identical to tc18 +tc20 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], + [2, 4987, 5300, 4950, 5200, 6172, 0, 0], + [3, 5200, 5300, 4940, 4962, 6172, 0, 0], # Sell on ROI + [4, 4962, 4987, 4972, 4950, 6172, 0, 0], + [5, 4550, 4975, 4925, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10, "119": 0.01}, profit_perc=0.01, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + + TESTS = [ tc0, tc1, @@ -300,6 +347,9 @@ TESTS = [ tc15, tc16, tc17, + tc18, + tc19, + tc20, ]