From 53ecdb931b040ef6afe5d632e0f84249cea2879b Mon Sep 17 00:00:00 2001 From: dingzhoufeng Date: Tue, 8 Mar 2022 12:26:43 +0800 Subject: [PATCH 01/18] add leverage --- freqtrade/optimize/backtesting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fa3deb86f..2b5b4ee14 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -355,6 +355,8 @@ class Backtesting: def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, trade_dur: int) -> float: + leverage = trade.leverage or 1.0 + is_short = trade.is_short or False """ Get close rate for backtesting result """ @@ -382,7 +384,7 @@ class Backtesting: abs(self.strategy.trailing_stop_positive))) else: # Worst case: price ticks tiny bit above open and dives down. - stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct)) + stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct / leverage)) assert stop_rate < sell_row[HIGH_IDX] # Limit lower-end to candle low to avoid sells below the low. # This still remains "worst case" - but "worst realistic case". @@ -400,7 +402,7 @@ class Backtesting: return sell_row[OPEN_IDX] # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) - close_rate = - (trade.open_rate * roi + trade.open_rate * + close_rate = - (trade.open_rate * roi / leverage + trade.open_rate * (1 + trade.fee_open)) / (trade.fee_close - 1) if (trade_dur > 0 and trade_dur == roi_entry From 82e0eca128d3c7013b65f1ce750afb400871b338 Mon Sep 17 00:00:00 2001 From: adriance Date: Wed, 9 Mar 2022 20:00:06 +0800 Subject: [PATCH 02/18] add short close rate calu --- freqtrade/optimize/backtesting.py | 125 +++++++++++++++++++++--------- 1 file changed, 88 insertions(+), 37 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9523ca92f..a76138bbf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -362,11 +362,18 @@ class Backtesting: """ # Special handling if high or low hit STOP_LOSS or ROI if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): - if trade.stop_loss > sell_row[HIGH_IDX]: - # our stoploss was already higher than candle high, - # possibly due to a cancelled trade exit. - # sell at open price. - return sell_row[OPEN_IDX] + if is_short: + if trade.stop_loss < sell_row[LOW_IDX]: + # our stoploss was already lower than candle high, + # possibly due to a cancelled trade exit. + # sell at open price. + return sell_row[OPEN_IDX] + else: + if trade.stop_loss > sell_row[HIGH_IDX]: + # our stoploss was already higher than candle high, + # possibly due to a cancelled trade exit. + # sell at open price. + return sell_row[OPEN_IDX] # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and @@ -379,16 +386,29 @@ class Backtesting: and self.strategy.trailing_stop_positive ): # Worst case: price reaches stop_positive_offset and dives down. - stop_rate = (sell_row[OPEN_IDX] * + if is_short: + stop_rate = (sell_row[OPEN_IDX] * + (1 - abs(self.strategy.trailing_stop_positive_offset) + + abs(self.strategy.trailing_stop_positive))) + else: + stop_rate = (sell_row[OPEN_IDX] * (1 + abs(self.strategy.trailing_stop_positive_offset) - abs(self.strategy.trailing_stop_positive))) else: # Worst case: price ticks tiny bit above open and dives down. - stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct / leverage)) - assert stop_rate < sell_row[HIGH_IDX] + if is_short: + stop_rate = sell_row[OPEN_IDX] * (1 + abs(trade.stop_loss_pct / leverage)) + assert stop_rate > sell_row[HIGH_IDX] + else: + stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct / leverage)) + assert stop_rate < sell_row[HIGH_IDX] + # Limit lower-end to candle low to avoid sells below the low. # This still remains "worst case" - but "worst realistic case". - return max(sell_row[LOW_IDX], stop_rate) + if is_short: + return min(sell_row[HIGH_IDX], stop_rate) + else: + return max(sell_row[LOW_IDX], stop_rate) # Set close_rate to stoploss return trade.stop_loss @@ -402,32 +422,60 @@ class Backtesting: return sell_row[OPEN_IDX] # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) - close_rate = - (trade.open_rate * roi / leverage + trade.open_rate * + if is_short: + close_rate = (trade.open_rate * + (1 - trade.fee_open) - trade.open_rate * roi / leverage) / (trade.fee_close + 1) + if (trade_dur > 0 and trade_dur == roi_entry + and roi_entry % self.timeframe_min == 0 + and sell_row[OPEN_IDX] < close_rate): + # new ROI entry came into effect. + # use Open rate if open_rate > calculated sell rate + return sell_row[OPEN_IDX] + else: + close_rate = - (trade.open_rate * roi / leverage + trade.open_rate * (1 + trade.fee_open)) / (trade.fee_close - 1) - if (trade_dur > 0 and trade_dur == roi_entry - and roi_entry % self.timeframe_min == 0 - and sell_row[OPEN_IDX] > close_rate): - # new ROI entry came into effect. - # use Open rate if open_rate > calculated sell rate - return sell_row[OPEN_IDX] + if (trade_dur > 0 and trade_dur == roi_entry + and roi_entry % self.timeframe_min == 0 + and sell_row[OPEN_IDX] > close_rate): + # new ROI entry came into effect. + # use Open rate if open_rate > calculated sell rate + return sell_row[OPEN_IDX] + + if is_short: + if ( + trade_dur == 0 + # Red candle (for longs), TODO: green candle (for shorts) + and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle + and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate + and close_rate < sell_row[CLOSE_IDX] + ): + # ROI on opening candles with custom pricing can only + # trigger if the entry was at Open or lower. + # details: https: // github.com/freqtrade/freqtrade/issues/6261 + # If open_rate is < open, only allow sells below the close on red candles. + raise ValueError("Opening candle ROI on red candles.") + else: + if ( + trade_dur == 0 + # Red candle (for longs), TODO: green candle (for shorts) + and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle + and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate + and close_rate > sell_row[CLOSE_IDX] + ): + # ROI on opening candles with custom pricing can only + # trigger if the entry was at Open or lower. + # details: https: // github.com/freqtrade/freqtrade/issues/6261 + # If open_rate is < open, only allow sells below the close on red candles. + raise ValueError("Opening candle ROI on red candles.") - if ( - trade_dur == 0 - # Red candle (for longs), TODO: green candle (for shorts) - and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle - and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate - and close_rate > sell_row[CLOSE_IDX] - ): - # ROI on opening candles with custom pricing can only - # trigger if the entry was at Open or lower. - # details: https: // github.com/freqtrade/freqtrade/issues/6261 - # If open_rate is < open, only allow sells below the close on red candles. - raise ValueError("Opening candle ROI on red candles.") # Use the maximum between close_rate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) + if is_short: + return max(min(close_rate, sell_row[HIGH_IDX]), sell_row[LOW_IDX]) + else: + return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) else: # This should not be reached... @@ -610,7 +658,10 @@ class Backtesting: proposed_rate=propose_rate, entry_tag=entry_tag) # default value is the open rate # We can't place orders higher than current high (otherwise it'd be a stop limit buy) # which freqtrade does not support in live. - propose_rate = min(propose_rate, row[HIGH_IDX]) + if direction == "short": + propose_rate = max(propose_rate, row[LOW_IDX]) + else: + propose_rate = min(propose_rate, row[HIGH_IDX]) min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, propose_rate, -0.05) or 0 max_stake_amount = self.exchange.get_max_pair_stake_amount(pair, propose_rate) @@ -700,13 +751,13 @@ class Backtesting: trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - trade.set_isolated_liq(self.exchange.get_liquidation_price( - pair=pair, - open_rate=propose_rate, - amount=amount, - leverage=leverage, - is_short=is_short, - )) + # trade.set_isolated_liq(self.exchange.get_liquidation_price( + # pair=pair, + # open_rate=propose_rate, + # amount=amount, + # leverage=leverage, + # is_short=is_short, + # )) order = Order( id=self.order_id_counter, From 1c86e69c34e3e35923cac1a4b400de32a75525da Mon Sep 17 00:00:00 2001 From: adriance Date: Wed, 9 Mar 2022 21:55:13 +0800 Subject: [PATCH 03/18] use filled time calculate duration --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/persistence/models.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index a76138bbf..5aa3974d0 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -534,7 +534,7 @@ class Backtesting: if sell.sell_flag: trade.close_date = sell_candle_time - trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) + trade_dur = int((trade.close_date_utc - trade.filled_date_utc).total_seconds() // 60) try: closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) except ValueError: diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index b80d75dc0..3836424c4 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -366,6 +366,10 @@ class LocalTrade(): else: return self.amount + @property + def filled_date_utc(self): + return self.select_order('buy', is_open=False).order_filled_date.replace(tzinfo=timezone.utc) + @property def open_date_utc(self): return self.open_date.replace(tzinfo=timezone.utc) From d579febfec9cd7c02f8bc95c59c0d5f91b69173c Mon Sep 17 00:00:00 2001 From: adriance Date: Wed, 9 Mar 2022 23:55:57 +0800 Subject: [PATCH 04/18] add filled time --- freqtrade/data/btanalysis.py | 3 ++- freqtrade/optimize/backtesting.py | 3 ++- freqtrade/persistence/models.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 4df8b2838..f0e0ccfd8 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -19,7 +19,7 @@ from freqtrade.persistence import LocalTrade, Trade, init_db logger = logging.getLogger(__name__) # Newest format -BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', +BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'buy_filled_date', 'close_date', 'open_rate', 'close_rate', 'fee_open', 'fee_close', 'trade_duration', 'profit_ratio', 'profit_abs', 'sell_reason', @@ -316,6 +316,7 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: if len(df) > 0: df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) + df.loc[:, 'buy_filled_date'] = pd.to_datetime(df['buy_filled_date'], utc=True) df.loc[:, 'close_rate'] = df['close_rate'].astype('float64') return df diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5aa3974d0..cb057d8eb 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -534,7 +534,7 @@ class Backtesting: if sell.sell_flag: trade.close_date = sell_candle_time - trade_dur = int((trade.close_date_utc - trade.filled_date_utc).total_seconds() // 60) + trade_dur = int((trade.close_date_utc - trade.buy_filled_date_utc).total_seconds() // 60) try: closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) except ValueError: @@ -960,6 +960,7 @@ class Backtesting: if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time) trade.open_order_id = None + trade.buy_filled_date = current_time LocalTrade.add_bt_trade(trade) self.wallets.update() diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 3836424c4..fbf150ec5 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -302,6 +302,7 @@ class LocalTrade(): amount: float = 0.0 amount_requested: Optional[float] = None open_date: datetime + buy_filled_date: datetime close_date: Optional[datetime] = None open_order_id: Optional[str] = None # absolute value of the stop loss @@ -367,8 +368,8 @@ class LocalTrade(): return self.amount @property - def filled_date_utc(self): - return self.select_order('buy', is_open=False).order_filled_date.replace(tzinfo=timezone.utc) + def buy_filled_date_utc(self): + return self.buy_filled_date.replace(tzinfo=timezone.utc) @property def open_date_utc(self): @@ -448,6 +449,9 @@ class LocalTrade(): 'open_rate_requested': self.open_rate_requested, 'open_trade_value': round(self.open_trade_value, 8), + 'buy_filled_date': self.buy_filled_date.strftime(DATETIME_PRINT_FORMAT), + 'buy_filled_timestamp': int(self.buy_filled_date.replace(tzinfo=timezone.utc).timestamp() * 1000), + 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) if self.close_date else None), 'close_timestamp': int(self.close_date.replace( From 499e9c3e982d437e429aa4c9ae42724ca2f429ea Mon Sep 17 00:00:00 2001 From: adriance Date: Thu, 10 Mar 2022 00:34:59 +0800 Subject: [PATCH 05/18] fix duration --- freqtrade/optimize/backtesting.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cb057d8eb..de42aa5f4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -357,6 +357,7 @@ class Backtesting: trade_dur: int) -> float: leverage = trade.leverage or 1.0 is_short = trade.is_short or False + filled_dur = int((trade.close_date_utc - trade.buy_filled_date_utc).total_seconds() // 60) """ Get close rate for backtesting result """ @@ -378,7 +379,7 @@ class Backtesting: # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0: + if sell.sell_type == SellType.TRAILING_STOP_LOSS and (trade_dur == 0 or filled_dur == 0): if ( not self.strategy.use_custom_stoploss and self.strategy.trailing_stop and self.strategy.trailing_only_offset_is_reached @@ -425,7 +426,7 @@ class Backtesting: if is_short: close_rate = (trade.open_rate * (1 - trade.fee_open) - trade.open_rate * roi / leverage) / (trade.fee_close + 1) - if (trade_dur > 0 and trade_dur == roi_entry + if (trade_dur > 0 and filled_dur > 0 and trade_dur == roi_entry and roi_entry % self.timeframe_min == 0 and sell_row[OPEN_IDX] < close_rate): # new ROI entry came into effect. @@ -435,7 +436,7 @@ class Backtesting: close_rate = - (trade.open_rate * roi / leverage + trade.open_rate * (1 + trade.fee_open)) / (trade.fee_close - 1) - if (trade_dur > 0 and trade_dur == roi_entry + if (trade_dur > 0 and filled_dur > 0 and trade_dur == roi_entry and roi_entry % self.timeframe_min == 0 and sell_row[OPEN_IDX] > close_rate): # new ROI entry came into effect. @@ -444,7 +445,7 @@ class Backtesting: if is_short: if ( - trade_dur == 0 + (trade_dur == 0 or filled_dur == 0) # Red candle (for longs), TODO: green candle (for shorts) and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate @@ -457,7 +458,7 @@ class Backtesting: raise ValueError("Opening candle ROI on red candles.") else: if ( - trade_dur == 0 + (trade_dur == 0 or filled_dur == 0) # Red candle (for longs), TODO: green candle (for shorts) and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate @@ -534,7 +535,7 @@ class Backtesting: if sell.sell_flag: trade.close_date = sell_candle_time - trade_dur = int((trade.close_date_utc - trade.buy_filled_date_utc).total_seconds() // 60) + trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) try: closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) except ValueError: From 3d9c55d5193dedb4861d8ed7df9159e8453e4041 Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 11:29:26 +0800 Subject: [PATCH 06/18] restore set_isolated_liq --- freqtrade/optimize/backtesting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e7d340130..ff6db1b08 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -751,13 +751,13 @@ class Backtesting: trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True) - # trade.set_isolated_liq(self.exchange.get_liquidation_price( - # pair=pair, - # open_rate=propose_rate, - # amount=amount, - # leverage=leverage, - # is_short=is_short, - # )) + trade.set_isolated_liq(self.exchange.get_liquidation_price( + pair=pair, + open_rate=propose_rate, + amount=amount, + leverage=leverage, + is_short=is_short, + )) order = Order( id=self.order_id_counter, From f9e93cf3f8bf35822ed2d9be58efe9cfa23147e6 Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 11:55:36 +0800 Subject: [PATCH 07/18] fix buy filled date none --- freqtrade/persistence/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index fbf150ec5..50ff5db3d 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -302,7 +302,7 @@ class LocalTrade(): amount: float = 0.0 amount_requested: Optional[float] = None open_date: datetime - buy_filled_date: datetime + buy_filled_date: Optional[datetime] = None close_date: Optional[datetime] = None open_order_id: Optional[str] = None # absolute value of the stop loss From a7503697964c7dc333167786d62ee44ef726f8cc Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 12:09:13 +0800 Subject: [PATCH 08/18] adjust none --- freqtrade/persistence/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 50ff5db3d..f5e63159b 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -449,8 +449,10 @@ class LocalTrade(): 'open_rate_requested': self.open_rate_requested, 'open_trade_value': round(self.open_trade_value, 8), - 'buy_filled_date': self.buy_filled_date.strftime(DATETIME_PRINT_FORMAT), - 'buy_filled_timestamp': int(self.buy_filled_date.replace(tzinfo=timezone.utc).timestamp() * 1000), + 'buy_filled_date': (self.buy_filled_date.strftime(DATETIME_PRINT_FORMAT) + if self.buy_filled_date else None), + 'buy_filled_timestamp': int(self.buy_filled_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.buy_filled_date else None, 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) if self.close_date else None), From bea38a2e7c463cf3ed3670ca1c4923f68185ecfd Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 13:42:52 +0800 Subject: [PATCH 09/18] remove filled date logic --- freqtrade/data/btanalysis.py | 3 +-- freqtrade/optimize/backtesting.py | 12 ++++++------ freqtrade/persistence/models.py | 10 ---------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index f0e0ccfd8..4df8b2838 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -19,7 +19,7 @@ from freqtrade.persistence import LocalTrade, Trade, init_db logger = logging.getLogger(__name__) # Newest format -BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'buy_filled_date', 'close_date', +BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'open_rate', 'close_rate', 'fee_open', 'fee_close', 'trade_duration', 'profit_ratio', 'profit_abs', 'sell_reason', @@ -316,7 +316,6 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: if len(df) > 0: df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) - df.loc[:, 'buy_filled_date'] = pd.to_datetime(df['buy_filled_date'], utc=True) df.loc[:, 'close_rate'] = df['close_rate'].astype('float64') return df diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ff6db1b08..00dfca7d8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -356,7 +356,7 @@ class Backtesting: trade_dur: int) -> float: leverage = trade.leverage or 1.0 is_short = trade.is_short or False - filled_dur = int((trade.close_date_utc - trade.buy_filled_date_utc).total_seconds() // 60) + """ Get close rate for backtesting result """ @@ -378,7 +378,7 @@ class Backtesting: # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and (trade_dur == 0 or filled_dur == 0): + if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0: if ( not self.strategy.use_custom_stoploss and self.strategy.trailing_stop and self.strategy.trailing_only_offset_is_reached @@ -425,7 +425,7 @@ class Backtesting: if is_short: close_rate = (trade.open_rate * (1 - trade.fee_open) - trade.open_rate * roi / leverage) / (trade.fee_close + 1) - if (trade_dur > 0 and filled_dur > 0 and trade_dur == roi_entry + if (trade_dur > 0 and trade_dur == roi_entry and roi_entry % self.timeframe_min == 0 and sell_row[OPEN_IDX] < close_rate): # new ROI entry came into effect. @@ -435,7 +435,7 @@ class Backtesting: close_rate = - (trade.open_rate * roi / leverage + trade.open_rate * (1 + trade.fee_open)) / (trade.fee_close - 1) - if (trade_dur > 0 and filled_dur > 0 and trade_dur == roi_entry + if (trade_dur > 0 and trade_dur == roi_entry and roi_entry % self.timeframe_min == 0 and sell_row[OPEN_IDX] > close_rate): # new ROI entry came into effect. @@ -444,7 +444,7 @@ class Backtesting: if is_short: if ( - (trade_dur == 0 or filled_dur == 0) + trade_dur == 0 # Red candle (for longs), TODO: green candle (for shorts) and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate @@ -457,7 +457,7 @@ class Backtesting: raise ValueError("Opening candle ROI on red candles.") else: if ( - (trade_dur == 0 or filled_dur == 0) + trade_dur == 0 # Red candle (for longs), TODO: green candle (for shorts) and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f5e63159b..b80d75dc0 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -302,7 +302,6 @@ class LocalTrade(): amount: float = 0.0 amount_requested: Optional[float] = None open_date: datetime - buy_filled_date: Optional[datetime] = None close_date: Optional[datetime] = None open_order_id: Optional[str] = None # absolute value of the stop loss @@ -367,10 +366,6 @@ class LocalTrade(): else: return self.amount - @property - def buy_filled_date_utc(self): - return self.buy_filled_date.replace(tzinfo=timezone.utc) - @property def open_date_utc(self): return self.open_date.replace(tzinfo=timezone.utc) @@ -449,11 +444,6 @@ class LocalTrade(): 'open_rate_requested': self.open_rate_requested, 'open_trade_value': round(self.open_trade_value, 8), - 'buy_filled_date': (self.buy_filled_date.strftime(DATETIME_PRINT_FORMAT) - if self.buy_filled_date else None), - 'buy_filled_timestamp': int(self.buy_filled_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.buy_filled_date else None, - 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) if self.close_date else None), 'close_timestamp': int(self.close_date.replace( From 26a74220fdcf91a947b4afe19092fc3fc0416563 Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 13:43:42 +0800 Subject: [PATCH 10/18] remove buy filled logic --- freqtrade/optimize/backtesting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 00dfca7d8..cb54a6f4c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -960,7 +960,6 @@ class Backtesting: if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time) trade.open_order_id = None - trade.buy_filled_date = current_time LocalTrade.add_bt_trade(trade) self.wallets.update() From 1d4eeacc6df58fde771c60ef00ac444d8be390de Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 17:55:42 +0800 Subject: [PATCH 11/18] fix test_backtest__enter_trade_futures row data error --- tests/optimize/test_backtesting.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 0d15c23e8..50c383346 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -581,14 +581,18 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None: backtesting._set_strategy(backtesting.strategylist[0]) pair = 'UNITTEST/USDT:USDT' row = [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), - 1, # Buy - 0.001, # Open - 0.0011, # Close - 0, # Sell - 0.00099, # Low - 0.0012, # High - '', # Buy Signal Name + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), + 0.001, # Open + 0.0012, # High + 0.00099, # Low + 0.0011, # Close + 1, # enter_long + 0, # exit_long + 1, # enter_short + 0, # exit_hsort + '',# Long Signal Name + '', # Short Signal Name + '', # Exit Signal Name ] backtesting.strategy.leverage = MagicMock(return_value=5.0) From 31182c4d80927bac6b7c1ca9d8ce4231e107dc6f Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 18:38:44 +0800 Subject: [PATCH 12/18] format --- freqtrade/optimize/backtesting.py | 195 ++++++++++++++++++------------ 1 file changed, 117 insertions(+), 78 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cb54a6f4c..8e30b2215 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -354,26 +354,25 @@ class Backtesting: def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, trade_dur: int) -> float: - leverage = trade.leverage or 1.0 - is_short = trade.is_short or False - """ Get close rate for backtesting result """ # Special handling if high or low hit STOP_LOSS or ROI + is_short = trade.is_short or False + if is_short: + return self._get_short_close_rate(sell_row, trade, sell, trade_dur) + else: + return self._get_long_close_rate(sell_row, trade, sell, trade_dur) + + def _get_short_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, + trade_dur: int) -> float: + leverage = trade.leverage or 1.0 if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): - if is_short: - if trade.stop_loss < sell_row[LOW_IDX]: - # our stoploss was already lower than candle high, - # possibly due to a cancelled trade exit. - # sell at open price. - return sell_row[OPEN_IDX] - else: - if trade.stop_loss > sell_row[HIGH_IDX]: - # our stoploss was already higher than candle high, - # possibly due to a cancelled trade exit. - # sell at open price. - return sell_row[OPEN_IDX] + if trade.stop_loss < sell_row[LOW_IDX]: + # our stoploss was already lower than candle high, + # possibly due to a cancelled trade exit. + # sell at open price. + return sell_row[OPEN_IDX] # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and @@ -386,29 +385,96 @@ class Backtesting: and self.strategy.trailing_stop_positive ): # Worst case: price reaches stop_positive_offset and dives down. - if is_short: - stop_rate = (sell_row[OPEN_IDX] * + stop_rate = (sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive_offset) + abs(self.strategy.trailing_stop_positive))) - else: - stop_rate = (sell_row[OPEN_IDX] * - (1 + abs(self.strategy.trailing_stop_positive_offset) - - abs(self.strategy.trailing_stop_positive))) else: # Worst case: price ticks tiny bit above open and dives down. - if is_short: - stop_rate = sell_row[OPEN_IDX] * (1 + abs(trade.stop_loss_pct / leverage)) - assert stop_rate > sell_row[HIGH_IDX] - else: - stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct / leverage)) - assert stop_rate < sell_row[HIGH_IDX] + stop_rate = sell_row[OPEN_IDX] * (1 + abs(trade.stop_loss_pct / leverage)) + assert stop_rate > sell_row[HIGH_IDX] # Limit lower-end to candle low to avoid sells below the low. # This still remains "worst case" - but "worst realistic case". - if is_short: - return min(sell_row[HIGH_IDX], stop_rate) + return min(sell_row[HIGH_IDX], stop_rate) + + # Set close_rate to stoploss + return trade.stop_loss + elif sell.sell_type == (SellType.ROI): + roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) + if roi is not None and roi_entry is not None: + if roi == -1 and roi_entry % self.timeframe_min == 0: + # When forceselling with ROI=-1, the roi time will always be equal to trade_dur. + # If that entry is a multiple of the timeframe (so on candle open) + # - we'll use open instead of close + return sell_row[OPEN_IDX] + + # - (Expected abs profit - open_rate - open_fee) / (fee_close -1) + open_fee_rate = trade.open_rate * (1 - trade.fee_open) + roi_rate = trade.open_rate * roi / leverage + close_rate = (roi_rate - open_fee_rate) / (trade.fee_close + 1) + if (trade_dur > 0 and trade_dur == roi_entry + and roi_entry % self.timeframe_min == 0 + and sell_row[OPEN_IDX] < close_rate): + # new ROI entry came into effect. + # use Open rate if open_rate > calculated sell rate + return sell_row[OPEN_IDX] + + if ( + trade_dur == 0 + # Red candle (for longs), TODO: green candle (for shorts) + and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle + and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate + and close_rate < sell_row[CLOSE_IDX] + ): + # ROI on opening candles with custom pricing can only + # trigger if the entry was at Open or lower. + # details: https: // github.com/freqtrade/freqtrade/issues/6261 + # If open_rate is < open, only allow sells below the close on red candles. + raise ValueError("Opening candle ROI on red candles.") + + # Use the maximum between close_rate and low as we + # cannot sell outside of a candle. + # Applies when a new ROI setting comes in place and the whole candle is above that. + return max(min(close_rate, sell_row[HIGH_IDX]), sell_row[LOW_IDX]) + + else: + # This should not be reached... + return sell_row[OPEN_IDX] + else: + return sell_row[OPEN_IDX] + + def _get_long_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, + trade_dur: int) -> float: + leverage = trade.leverage or 1.0 + if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + if trade.stop_loss > sell_row[HIGH_IDX]: + # our stoploss was already higher than candle high, + # possibly due to a cancelled trade exit. + # sell at open price. + return sell_row[OPEN_IDX] + + # Special case: trailing triggers within same candle as trade opened. Assume most + # pessimistic price movement, which is moving just enough to arm stoploss and + # immediately going down to stop price. + if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0: + if ( + not self.strategy.use_custom_stoploss and self.strategy.trailing_stop + and self.strategy.trailing_only_offset_is_reached + and self.strategy.trailing_stop_positive_offset is not None + and self.strategy.trailing_stop_positive + ): + # Worst case: price reaches stop_positive_offset and dives down. + stop_rate = (sell_row[OPEN_IDX] * + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive))) else: - return max(sell_row[LOW_IDX], stop_rate) + # Worst case: price ticks tiny bit above open and dives down. + stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct / leverage)) + assert stop_rate < sell_row[HIGH_IDX] + + # Limit lower-end to candle low to avoid sells below the low. + # This still remains "worst case" - but "worst realistic case". + return max(sell_row[LOW_IDX], stop_rate) # Set close_rate to stoploss return trade.stop_loss @@ -422,60 +488,33 @@ class Backtesting: return sell_row[OPEN_IDX] # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) - if is_short: - close_rate = (trade.open_rate * - (1 - trade.fee_open) - trade.open_rate * roi / leverage) / (trade.fee_close + 1) - if (trade_dur > 0 and trade_dur == roi_entry - and roi_entry % self.timeframe_min == 0 - and sell_row[OPEN_IDX] < close_rate): - # new ROI entry came into effect. - # use Open rate if open_rate > calculated sell rate - return sell_row[OPEN_IDX] - else: - close_rate = - (trade.open_rate * roi / leverage + trade.open_rate * - (1 + trade.fee_open)) / (trade.fee_close - 1) + close_rate = -(trade.open_rate * roi / leverage + trade.open_rate * + (1 + trade.fee_open)) / (trade.fee_close - 1) - if (trade_dur > 0 and trade_dur == roi_entry - and roi_entry % self.timeframe_min == 0 - and sell_row[OPEN_IDX] > close_rate): - # new ROI entry came into effect. - # use Open rate if open_rate > calculated sell rate - return sell_row[OPEN_IDX] + if (trade_dur > 0 and trade_dur == roi_entry + and roi_entry % self.timeframe_min == 0 + and sell_row[OPEN_IDX] > close_rate): + # new ROI entry came into effect. + # use Open rate if open_rate > calculated sell rate + return sell_row[OPEN_IDX] - if is_short: - if ( - trade_dur == 0 - # Red candle (for longs), TODO: green candle (for shorts) - and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle - and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate - and close_rate < sell_row[CLOSE_IDX] - ): - # ROI on opening candles with custom pricing can only - # trigger if the entry was at Open or lower. - # details: https: // github.com/freqtrade/freqtrade/issues/6261 - # If open_rate is < open, only allow sells below the close on red candles. - raise ValueError("Opening candle ROI on red candles.") - else: - if ( - trade_dur == 0 - # Red candle (for longs), TODO: green candle (for shorts) - and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle - and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate - and close_rate > sell_row[CLOSE_IDX] - ): - # ROI on opening candles with custom pricing can only - # trigger if the entry was at Open or lower. - # details: https: // github.com/freqtrade/freqtrade/issues/6261 - # If open_rate is < open, only allow sells below the close on red candles. - raise ValueError("Opening candle ROI on red candles.") + if ( + trade_dur == 0 + # Red candle (for longs), TODO: green candle (for shorts) + and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle + and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate + and close_rate > sell_row[CLOSE_IDX] + ): + # ROI on opening candles with custom pricing can only + # trigger if the entry was at Open or lower. + # details: https: // github.com/freqtrade/freqtrade/issues/6261 + # If open_rate is < open, only allow sells below the close on red candles. + raise ValueError("Opening candle ROI on red candles.") # Use the maximum between close_rate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - if is_short: - return max(min(close_rate, sell_row[HIGH_IDX]), sell_row[LOW_IDX]) - else: - return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) + return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) else: # This should not be reached... From 7dd57e8c04c90dfee2066ddbf311c3c548822f5b Mon Sep 17 00:00:00 2001 From: adriance Date: Mon, 14 Mar 2022 18:39:11 +0800 Subject: [PATCH 13/18] format --- tests/optimize/test_backtesting.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 50c383346..3c6e5df4b 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -581,18 +581,18 @@ def test_backtest__enter_trade_futures(default_conf_usdt, fee, mocker) -> None: backtesting._set_strategy(backtesting.strategylist[0]) pair = 'UNITTEST/USDT:USDT' row = [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), - 0.001, # Open - 0.0012, # High - 0.00099, # Low - 0.0011, # Close - 1, # enter_long - 0, # exit_long - 1, # enter_short - 0, # exit_hsort - '',# Long Signal Name - '', # Short Signal Name - '', # Exit Signal Name + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), + 0.001, # Open + 0.0012, # High + 0.00099, # Low + 0.0011, # Close + 1, # enter_long + 0, # exit_long + 1, # enter_short + 0, # exit_hsort + '', # Long Signal Name + '', # Short Signal Name + '', # Exit Signal Name ] backtesting.strategy.leverage = MagicMock(return_value=5.0) From 7059892304c05c492b0d36de12fd610d2d67c83b Mon Sep 17 00:00:00 2001 From: adriance Date: Tue, 15 Mar 2022 12:04:02 +0800 Subject: [PATCH 14/18] Optimize the code. Fix stop_rate judgment error --- freqtrade/optimize/backtesting.py | 234 ++++++++++++------------------ 1 file changed, 94 insertions(+), 140 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8e30b2215..ba6aab71e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -358,168 +358,122 @@ class Backtesting: Get close rate for backtesting result """ # Special handling if high or low hit STOP_LOSS or ROI - is_short = trade.is_short or False - if is_short: - return self._get_short_close_rate(sell_row, trade, sell, trade_dur) - else: - return self._get_long_close_rate(sell_row, trade, sell, trade_dur) - - def _get_short_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, - trade_dur: int) -> float: - leverage = trade.leverage or 1.0 if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + return self._get_close_rate_for_stoploss(sell_row, trade, sell, trade_dur) + elif sell.sell_type == (SellType.ROI): + return self._get_close_rate_for_roi(sell_row, trade, sell, trade_dur) + else: + return sell_row[OPEN_IDX] + + def _get_close_rate_for_stoploss(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, + trade_dur: int) -> float: + # our stoploss was already lower than candle high, + # possibly due to a cancelled trade exit. + # sell at open price. + is_short = trade.is_short or False + leverage = trade.leverage or 1.0 + side_1 = -1 if is_short else 1 + if is_short: if trade.stop_loss < sell_row[LOW_IDX]: - # our stoploss was already lower than candle high, - # possibly due to a cancelled trade exit. - # sell at open price. + return sell_row[OPEN_IDX] + else: + if trade.stop_loss > sell_row[HIGH_IDX]: return sell_row[OPEN_IDX] - # Special case: trailing triggers within same candle as trade opened. Assume most - # pessimistic price movement, which is moving just enough to arm stoploss and - # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0: - if ( - not self.strategy.use_custom_stoploss and self.strategy.trailing_stop - and self.strategy.trailing_only_offset_is_reached - and self.strategy.trailing_stop_positive_offset is not None - and self.strategy.trailing_stop_positive - ): - # Worst case: price reaches stop_positive_offset and dives down. - stop_rate = (sell_row[OPEN_IDX] * - (1 - abs(self.strategy.trailing_stop_positive_offset) + - abs(self.strategy.trailing_stop_positive))) + # Special case: trailing triggers within same candle as trade opened. Assume most + # pessimistic price movement, which is moving just enough to arm stoploss and + # immediately going down to stop price. + if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0: + if ( + not self.strategy.use_custom_stoploss and self.strategy.trailing_stop + and self.strategy.trailing_only_offset_is_reached + and self.strategy.trailing_stop_positive_offset is not None + and self.strategy.trailing_stop_positive + ): + # Worst case: price reaches stop_positive_offset and dives down. + stop_rate = (sell_row[OPEN_IDX] * + (1 + side_1 * abs(self.strategy.trailing_stop_positive_offset) + + abs(self.strategy.trailing_stop_positive / leverage))) + else: + # Worst case: price ticks tiny bit above open and dives down. + stop_rate = sell_row[OPEN_IDX] * (1 - + side_1 * abs(trade.stop_loss_pct / leverage)) + if is_short: + assert stop_rate > sell_row[LOW_IDX] else: - # Worst case: price ticks tiny bit above open and dives down. - stop_rate = sell_row[OPEN_IDX] * (1 + abs(trade.stop_loss_pct / leverage)) - assert stop_rate > sell_row[HIGH_IDX] + assert stop_rate < sell_row[HIGH_IDX] - # Limit lower-end to candle low to avoid sells below the low. - # This still remains "worst case" - but "worst realistic case". + # Limit lower-end to candle low to avoid sells below the low. + # This still remains "worst case" - but "worst realistic case". + if is_short: return min(sell_row[HIGH_IDX], stop_rate) + else: + return max(sell_row[LOW_IDX], stop_rate) - # Set close_rate to stoploss - return trade.stop_loss - elif sell.sell_type == (SellType.ROI): - roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) - if roi is not None and roi_entry is not None: - if roi == -1 and roi_entry % self.timeframe_min == 0: - # When forceselling with ROI=-1, the roi time will always be equal to trade_dur. - # If that entry is a multiple of the timeframe (so on candle open) - # - we'll use open instead of close - return sell_row[OPEN_IDX] + # Set close_rate to stoploss + return trade.stop_loss - # - (Expected abs profit - open_rate - open_fee) / (fee_close -1) - open_fee_rate = trade.open_rate * (1 - trade.fee_open) - roi_rate = trade.open_rate * roi / leverage - close_rate = (roi_rate - open_fee_rate) / (trade.fee_close + 1) - if (trade_dur > 0 and trade_dur == roi_entry - and roi_entry % self.timeframe_min == 0 - and sell_row[OPEN_IDX] < close_rate): - # new ROI entry came into effect. - # use Open rate if open_rate > calculated sell rate - return sell_row[OPEN_IDX] + def _get_close_rate_for_roi(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, + trade_dur: int) -> float: + is_short = trade.is_short or False + leverage = trade.leverage or 1.0 + side_1 = -1 if is_short else 1 + roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) + if roi is not None and roi_entry is not None: + if roi == -1 and roi_entry % self.timeframe_min == 0: + # When forceselling with ROI=-1, the roi time will always be equal to trade_dur. + # If that entry is a multiple of the timeframe (so on candle open) + # - we'll use open instead of close + return sell_row[OPEN_IDX] - if ( - trade_dur == 0 + # - (Expected abs profit - open_rate - open_fee) / (fee_close -1) + roi_rate = trade.open_rate * roi / leverage + open_fee_rate = side_1 * trade.open_rate * (1 + side_1 * trade.fee_open) + close_rate = -side_1 * (roi_rate + open_fee_rate) / (trade.fee_close - side_1 * 1) + if is_short: + is_new_roi = sell_row[OPEN_IDX] < close_rate + else: + is_new_roi = sell_row[OPEN_IDX] > close_rate + if (trade_dur > 0 and trade_dur == roi_entry + and roi_entry % self.timeframe_min == 0 + and is_new_roi): + # new ROI entry came into effect. + # use Open rate if open_rate > calculated sell rate + return sell_row[OPEN_IDX] + + if (trade_dur == 0 and ( + ( + is_short # Red candle (for longs), TODO: green candle (for shorts) and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate and close_rate < sell_row[CLOSE_IDX] - ): - # ROI on opening candles with custom pricing can only - # trigger if the entry was at Open or lower. - # details: https: // github.com/freqtrade/freqtrade/issues/6261 - # If open_rate is < open, only allow sells below the close on red candles. - raise ValueError("Opening candle ROI on red candles.") - - # Use the maximum between close_rate and low as we - # cannot sell outside of a candle. - # Applies when a new ROI setting comes in place and the whole candle is above that. - return max(min(close_rate, sell_row[HIGH_IDX]), sell_row[LOW_IDX]) - - else: - # This should not be reached... - return sell_row[OPEN_IDX] - else: - return sell_row[OPEN_IDX] - - def _get_long_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, - trade_dur: int) -> float: - leverage = trade.leverage or 1.0 - if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): - if trade.stop_loss > sell_row[HIGH_IDX]: - # our stoploss was already higher than candle high, - # possibly due to a cancelled trade exit. - # sell at open price. - return sell_row[OPEN_IDX] - - # Special case: trailing triggers within same candle as trade opened. Assume most - # pessimistic price movement, which is moving just enough to arm stoploss and - # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0: - if ( - not self.strategy.use_custom_stoploss and self.strategy.trailing_stop - and self.strategy.trailing_only_offset_is_reached - and self.strategy.trailing_stop_positive_offset is not None - and self.strategy.trailing_stop_positive - ): - # Worst case: price reaches stop_positive_offset and dives down. - stop_rate = (sell_row[OPEN_IDX] * - (1 + abs(self.strategy.trailing_stop_positive_offset) - - abs(self.strategy.trailing_stop_positive))) - else: - # Worst case: price ticks tiny bit above open and dives down. - stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct / leverage)) - assert stop_rate < sell_row[HIGH_IDX] - - # Limit lower-end to candle low to avoid sells below the low. - # This still remains "worst case" - but "worst realistic case". - return max(sell_row[LOW_IDX], stop_rate) - - # Set close_rate to stoploss - return trade.stop_loss - elif sell.sell_type == (SellType.ROI): - roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) - if roi is not None and roi_entry is not None: - if roi == -1 and roi_entry % self.timeframe_min == 0: - # When forceselling with ROI=-1, the roi time will always be equal to trade_dur. - # If that entry is a multiple of the timeframe (so on candle open) - # - we'll use open instead of close - return sell_row[OPEN_IDX] - - # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) - close_rate = -(trade.open_rate * roi / leverage + trade.open_rate * - (1 + trade.fee_open)) / (trade.fee_close - 1) - - if (trade_dur > 0 and trade_dur == roi_entry - and roi_entry % self.timeframe_min == 0 - and sell_row[OPEN_IDX] > close_rate): - # new ROI entry came into effect. - # use Open rate if open_rate > calculated sell rate - return sell_row[OPEN_IDX] - - if ( - trade_dur == 0 + ) + or + ( + not is_short # Red candle (for longs), TODO: green candle (for shorts) and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate and close_rate > sell_row[CLOSE_IDX] - ): - # ROI on opening candles with custom pricing can only - # trigger if the entry was at Open or lower. - # details: https: // github.com/freqtrade/freqtrade/issues/6261 - # If open_rate is < open, only allow sells below the close on red candles. - raise ValueError("Opening candle ROI on red candles.") + ) + )): + # ROI on opening candles with custom pricing can only + # trigger if the entry was at Open or lower. + # details: https: // github.com/freqtrade/freqtrade/issues/6261 + # If open_rate is < open, only allow sells below the close on red candles. + raise ValueError("Opening candle ROI on red candles.") - # Use the maximum between close_rate and low as we - # cannot sell outside of a candle. - # Applies when a new ROI setting comes in place and the whole candle is above that. + # Use the maximum between close_rate and low as we + # cannot sell outside of a candle. + # Applies when a new ROI setting comes in place and the whole candle is above that. + if is_short: + return max(min(close_rate, sell_row[HIGH_IDX]), sell_row[LOW_IDX]) + else: return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) - else: - # This should not be reached... - return sell_row[OPEN_IDX] else: + # This should not be reached... return sell_row[OPEN_IDX] def _get_adjust_trade_entry_for_candle(self, trade: LocalTrade, row: Tuple From fd211166f067ab990ff0298e5616dfc673eb2e3c Mon Sep 17 00:00:00 2001 From: adriance Date: Tue, 15 Mar 2022 12:23:59 +0800 Subject: [PATCH 15/18] fixed side error --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ba6aab71e..c597c6748 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -392,8 +392,8 @@ class Backtesting: ): # Worst case: price reaches stop_positive_offset and dives down. stop_rate = (sell_row[OPEN_IDX] * - (1 + side_1 * abs(self.strategy.trailing_stop_positive_offset) + - abs(self.strategy.trailing_stop_positive / leverage))) + (1 + side_1 * abs(self.strategy.trailing_stop_positive_offset) - + side_1 * abs(self.strategy.trailing_stop_positive / leverage))) else: # Worst case: price ticks tiny bit above open and dives down. stop_rate = sell_row[OPEN_IDX] * (1 - From cbbdf00ddd745ef15c5887ce7d639561d399c4c2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 15 Mar 2022 06:39:07 +0100 Subject: [PATCH 16/18] Update comments in short backtest rates --- freqtrade/optimize/backtesting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c597c6748..ea9850624 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -444,22 +444,22 @@ class Backtesting: if (trade_dur == 0 and ( ( is_short - # Red candle (for longs), TODO: green candle (for shorts) + # Red candle (for longs) and sell_row[OPEN_IDX] < sell_row[CLOSE_IDX] # Red candle - and trade.open_rate > sell_row[OPEN_IDX] # trade-open below open_rate - and close_rate < sell_row[CLOSE_IDX] + and trade.open_rate > sell_row[OPEN_IDX] # trade-open above open_rate + and close_rate < sell_row[CLOSE_IDX] # closes below close ) or ( not is_short - # Red candle (for longs), TODO: green candle (for shorts) - and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # Red candle + # green candle (for shorts) + and sell_row[OPEN_IDX] > sell_row[CLOSE_IDX] # green candle and trade.open_rate < sell_row[OPEN_IDX] # trade-open below open_rate - and close_rate > sell_row[CLOSE_IDX] + and close_rate > sell_row[CLOSE_IDX] # closes above close ) )): # ROI on opening candles with custom pricing can only - # trigger if the entry was at Open or lower. + # trigger if the entry was at Open or lower wick. # details: https: // github.com/freqtrade/freqtrade/issues/6261 # If open_rate is < open, only allow sells below the close on red candles. raise ValueError("Opening candle ROI on red candles.") From ceba4d6e9b10627fbec55e9a165e59d448ca4ac4 Mon Sep 17 00:00:00 2001 From: adriance Date: Tue, 15 Mar 2022 14:03:06 +0800 Subject: [PATCH 17/18] Remove meaningless code --- freqtrade/optimize/backtesting.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ea9850624..ac5d4e652 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -467,10 +467,7 @@ class Backtesting: # Use the maximum between close_rate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - if is_short: - return max(min(close_rate, sell_row[HIGH_IDX]), sell_row[LOW_IDX]) - else: - return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) + return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) else: # This should not be reached... From 7c9d2dd20a9e040e7092e5ebc363a5ab188e5bd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 16 Mar 2022 07:00:50 +0100 Subject: [PATCH 18/18] Fix a few more short bugs in backtesting --- freqtrade/optimize/backtesting.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ac5d4e652..cf499d4e2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -575,8 +575,8 @@ class Backtesting: ft_pair=trade.pair, order_id=str(self.order_id_counter), symbol=trade.pair, - ft_order_side="sell", - side="sell", + ft_order_side=trade.exit_side, + side=trade.exit_side, order_type=order_type, status="open", price=closerate, @@ -756,8 +756,8 @@ class Backtesting: ft_pair=trade.pair, order_id=str(self.order_id_counter), symbol=trade.pair, - ft_order_side="buy", - side="buy", + ft_order_side=trade.enter_side, + side=trade.enter_side, order_type=order_type, status="open", order_date=current_time, @@ -839,17 +839,17 @@ class Backtesting: timedout = self.strategy.ft_check_timed_out(order.side, trade, order, current_time) if timedout: - if order.side == 'buy': + if order.side == trade.enter_side: self.timedout_entry_orders += 1 if trade.nr_of_successful_entries == 0: - # Remove trade due to buy timeout expiration. + # Remove trade due to entry timeout expiration. return True else: # Close additional buy order del trade.orders[trade.orders.index(order)] - if order.side == 'sell': + if order.side == trade.exit_side: self.timedout_exit_orders += 1 - # Close sell order and retry selling on next signal. + # Close exit order and retry exiting on next signal. del trade.orders[trade.orders.index(order)] return False @@ -945,8 +945,8 @@ class Backtesting: open_trades[pair].append(trade) for trade in list(open_trades[pair]): - # 2. Process buy orders. - order = trade.select_order('buy', is_open=True) + # 2. Process entry orders. + order = trade.select_order(trade.enter_side, is_open=True) if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time) trade.open_order_id = None @@ -958,7 +958,7 @@ class Backtesting: self._get_sell_trade_entry(trade, row) # Place sell order if necessary # 4. Process sell orders. - order = trade.select_order('sell', is_open=True) + order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): trade.open_order_id = None trade.close_date = current_time