From a11d579bc2338fa3087fc5f7d079fa43314cecd5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Oct 2022 16:22:55 +0200 Subject: [PATCH 1/7] Verify order fills on "detail" timeframe --- freqtrade/optimize/backtesting.py | 69 ++++++++++++++++--------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4d98f1f5a..b3395a2c3 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -688,10 +688,11 @@ class Backtesting: trade.orders.append(order) return trade - def _get_exit_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]: + def _get_exit_trade_entry( + self, trade: LocalTrade, row: Tuple, is_first: bool) -> Optional[LocalTrade]: exit_candle_time: datetime = row[DATE_IDX].to_pydatetime() - if self.trading_mode == TradingMode.FUTURES: + if is_first and self.trading_mode == TradingMode.FUTURES: trade.funding_fees = self.exchange.calculate_funding_fees( self.futures_data[trade.pair], amount=trade.amount, @@ -700,32 +701,7 @@ class Backtesting: close_date=exit_candle_time, ) - if self.timeframe_detail and trade.pair in self.detail_data: - exit_candle_end = exit_candle_time + timedelta(minutes=self.timeframe_min) - - detail_data = self.detail_data[trade.pair] - detail_data = detail_data.loc[ - (detail_data['date'] >= exit_candle_time) & - (detail_data['date'] < exit_candle_end) - ].copy() - if len(detail_data) == 0: - # Fall back to "regular" data if no detail data was found for this candle - return self._get_exit_trade_entry_for_candle(trade, row) - detail_data.loc[:, 'enter_long'] = row[LONG_IDX] - detail_data.loc[:, 'exit_long'] = row[ELONG_IDX] - detail_data.loc[:, 'enter_short'] = row[SHORT_IDX] - detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX] - detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX] - detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX] - for det_row in detail_data[HEADERS].values.tolist(): - res = self._get_exit_trade_entry_for_candle(trade, det_row) - if res: - return res - - return None - - else: - return self._get_exit_trade_entry_for_candle(trade, row) + return self._get_exit_trade_entry_for_candle(trade, row) def get_valid_price_and_stake( self, pair: str, row: Tuple, propose_rate: float, stake_amount: float, @@ -1070,7 +1046,7 @@ class Backtesting: def backtest_loop( self, row: Tuple, pair: str, current_time: datetime, end_date: datetime, - max_open_trades: int, open_trade_count_start: int) -> int: + max_open_trades: int, open_trade_count_start: int, is_first: bool = True) -> int: """ NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. @@ -1088,9 +1064,11 @@ class Backtesting: # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected # don't open on the last row + # We only open trades on the initial candle. trade_dir = self.check_for_trade_entry(row) if ( (self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0) + and is_first and self.trade_slot_available(max_open_trades, open_trade_count_start) and current_time != end_date and trade_dir is not None @@ -1116,7 +1094,7 @@ class Backtesting: # 4. Create exit orders (if any) if not trade.open_order_id: - self._get_exit_trade_entry(trade, row) # Place exit order if necessary + self._get_exit_trade_entry(trade, row, is_first) # Place exit order if necessary # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) @@ -1167,7 +1145,6 @@ class Backtesting: self.progress.init_step(BacktestState.BACKTEST, int( (end_date - start_date) / timedelta(minutes=self.timeframe_min))) - # Loop timerange and get candle for each pair at that point in time while current_time <= end_date: open_trade_count_start = LocalTrade.bt_open_open_trade_count @@ -1181,9 +1158,35 @@ class Backtesting: row_index += 1 indexes[pair] = row_index self.dataprovider._set_dataframe_max_index(row_index) + current_detail_time: datetime = row[DATE_IDX].to_pydatetime() + if self.timeframe_detail and pair in self.detail_data: + exit_candle_end = current_detail_time + timedelta(minutes=self.timeframe_min) - open_trade_count_start = self.backtest_loop( - row, pair, current_time, end_date, max_open_trades, open_trade_count_start) + detail_data = self.detail_data[pair] + detail_data = detail_data.loc[ + (detail_data['date'] >= current_detail_time) & + (detail_data['date'] < exit_candle_end) + ].copy() + if len(detail_data) == 0: + # Fall back to "regular" data if no detail data was found for this candle + open_trade_count_start = self.backtest_loop( + row, pair, current_time, end_date, max_open_trades, + open_trade_count_start) + detail_data.loc[:, 'enter_long'] = row[LONG_IDX] + detail_data.loc[:, 'exit_long'] = row[ELONG_IDX] + detail_data.loc[:, 'enter_short'] = row[SHORT_IDX] + detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX] + detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX] + detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX] + is_first = True + for det_row in detail_data[HEADERS].values.tolist(): + open_trade_count_start = self.backtest_loop( + det_row, pair, current_time, end_date, max_open_trades, + open_trade_count_start, is_first) + is_first = False + else: + open_trade_count_start = self.backtest_loop( + row, pair, current_time, end_date, max_open_trades, open_trade_count_start) # Move time one configured time_interval ahead. self.progress.increment() From 29ba263c3c19a96abecf50015dcc9f6017fa6ee5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Oct 2022 16:23:03 +0200 Subject: [PATCH 2/7] Update some test parameters --- tests/optimize/test_backtesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 140cc3394..21d9d25cc 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -686,7 +686,7 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: ) # No data available. - res = backtesting._get_exit_trade_entry(trade, row_sell) + res = backtesting._get_exit_trade_entry(trade, row_sell, True) assert res is not None assert res.exit_reason == ExitType.ROI.value assert res.close_date_utc == datetime(2020, 1, 1, 5, 0, tzinfo=timezone.utc) @@ -699,13 +699,13 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: [], columns=['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', 'enter_short', 'exit_short', 'long_tag', 'short_tag', 'exit_tag']) - res = backtesting._get_exit_trade_entry(trade, row) + res = backtesting._get_exit_trade_entry(trade, row, True) assert res is None # Assign backtest-detail data backtesting.detail_data[pair] = row_detail - res = backtesting._get_exit_trade_entry(trade, row_sell) + res = backtesting._get_exit_trade_entry(trade, row_sell, True) assert res is not None assert res.exit_reason == ExitType.ROI.value # Sell at minute 3 (not available above!) From 0888b53b5a080c069006dc30d479126b17e56979 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 4 Nov 2022 07:07:56 +0100 Subject: [PATCH 3/7] Udpate current_time handling for detail loop --- freqtrade/optimize/backtesting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b3395a2c3..fa45e9dd4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1179,9 +1179,11 @@ class Backtesting: detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX] detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX] is_first = True + current_time_det = current_time for det_row in detail_data[HEADERS].values.tolist(): + current_time_det += timedelta(minutes=self.timeframe_detail_min) open_trade_count_start = self.backtest_loop( - det_row, pair, current_time, end_date, max_open_trades, + det_row, pair, current_time_det, end_date, max_open_trades, open_trade_count_start, is_first) is_first = False else: From 5bd3e54b17424daec79c3208589b807f92890b7a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Nov 2022 20:01:05 +0100 Subject: [PATCH 4/7] Add test for detail backtesting --- tests/optimize/test_backtesting.py | 87 ++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 21d9d25cc..26c31efef 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -787,17 +787,98 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: for _, t in results.iterrows(): assert len(t['orders']) == 2 ln = data_pair.loc[data_pair["date"] == t["open_date"]] - # Check open trade rate alignes to open rate + # Check open trade rate aligns to open rate assert not ln.empty assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) - # check close trade rate alignes to close rate or is between high and low + # check close trade rate aligns to close rate or is between high and low ln1 = data_pair.loc[data_pair["date"] == t["close_date"]] - assert not ln1.empty assert (round(ln1.iloc[0]["open"], 6) == round(t["close_rate"], 6) or round(ln1.iloc[0]["low"], 6) < round( t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6)) +@pytest.mark.parametrize('use_detail', [True, False]) +def test_backtest_one_detail(default_conf_usdt, fee, mocker, testdatadir, use_detail) -> None: + default_conf_usdt['use_exit_signal'] = False + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) + mocker.patch("freqtrade.exchange.Exchange.get_max_pair_stake_amount", return_value=float('inf')) + if use_detail: + default_conf_usdt['timeframe_detail'] = '1m' + patch_exchange(mocker) + + def advise_entry(df, *args, **kwargs): + # Mock function to force several entries + df.loc[(df['rsi'] < 40), 'enter_long'] = 1 + return df + + def custom_entry_price(proposed_rate, **kwargs): + return proposed_rate * 0.997 + + backtesting = Backtesting(default_conf_usdt) + backtesting._set_strategy(backtesting.strategylist[0]) + backtesting.strategy.populate_entry_trend = advise_entry + backtesting.strategy.custom_entry_price = custom_entry_price + pair = 'XRP/ETH' + # Pick a timerange adapted to the pair we use to test + timerange = TimeRange.parse_timerange('20191010-20191013') + data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['XRP/ETH'], + timerange=timerange) + if use_detail: + data_1m = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['XRP/ETH'], + timerange=timerange) + backtesting.detail_data = data_1m + processed = backtesting.strategy.advise_all_indicators(data) + min_date, max_date = get_timerange(processed) + + result = backtesting.backtest( + processed=deepcopy(processed), + start_date=min_date, + end_date=max_date, + max_open_trades=10, + ) + results = result['results'] + assert not results.empty + # Timeout settings from default_conf = entry: 10, exit: 30 + assert len(results) == (2 if use_detail else 3) + + assert 'orders' in results.columns + data_pair = processed[pair] + + data_1m_pair = data_1m[pair] if use_detail else pd.DataFrame() + late_entry = 0 + for _, t in results.iterrows(): + assert len(t['orders']) == 2 + + entryo = t['orders'][0] + entry_ts = datetime.fromtimestamp(entryo['order_filled_timestamp'] // 1000, tz=timezone.utc) + if entry_ts > t['open_date']: + late_entry += 1 + + # Get "entry fill" candle + ln = (data_1m_pair.loc[data_1m_pair["date"] == entry_ts] + if use_detail else data_pair.loc[data_pair["date"] == entry_ts]) + # Check open trade rate aligns to open rate + assert not ln.empty + + # assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) + assert round(ln.iloc[0]["low"], 6) <= round( + t["open_rate"], 6) <= round(ln.iloc[0]["high"], 6) + # check close trade rate aligns to close rate or is between high and low + ln1 = data_pair.loc[data_pair["date"] == t["close_date"]] + if use_detail: + ln1_1m = data_1m_pair.loc[data_1m_pair["date"] == t["close_date"]] + assert not ln1.empty or not ln1_1m.empty + else: + assert not ln1.empty + ln2 = ln1_1m if ln1.empty else ln1 + + assert (round(ln2.iloc[0]["low"], 6) <= round( + t["close_rate"], 6) <= round(ln2.iloc[0]["high"], 6)) + + assert late_entry > 0 + + def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) -> None: # This strategy intentionally places unfillable orders. default_conf['strategy'] = 'StrategyTestV3CustomEntryPrice' From d089fdae34820726c4902911bf27fa96ea44e27b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Nov 2022 20:02:36 +0100 Subject: [PATCH 5/7] Fix current-time_det calculation --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index fa45e9dd4..8faeeb9fe 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1181,10 +1181,10 @@ class Backtesting: is_first = True current_time_det = current_time for det_row in detail_data[HEADERS].values.tolist(): - current_time_det += timedelta(minutes=self.timeframe_detail_min) open_trade_count_start = self.backtest_loop( det_row, pair, current_time_det, end_date, max_open_trades, open_trade_count_start, is_first) + current_time_det += timedelta(minutes=self.timeframe_detail_min) is_first = False else: open_trade_count_start = self.backtest_loop( From ded57fb3019e1e564cc9a6842c2183ae18de8951 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Nov 2022 20:03:20 +0100 Subject: [PATCH 6/7] Remove no longer valid test part --- freqtrade/optimize/backtesting.py | 2 +- tests/optimize/test_backtesting.py | 32 ------------------------------ 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8faeeb9fe..54312177c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1064,7 +1064,7 @@ class Backtesting: # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected # don't open on the last row - # We only open trades on the initial candle. + # We only open trades on the main candle, not on detail candles trade_dir = self.check_for_trade_entry(row) if ( (self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 26c31efef..9a91b0c6f 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -663,27 +663,6 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: '', # Exit Signal Name ] - row_detail = pd.DataFrame( - [ - [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0, tzinfo=timezone.utc), - 200, 200.1, 197, 199, 1, 0, 0, 0, '', '', '', - ], [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=1, tzinfo=timezone.utc), - 199, 199.7, 199, 199.5, 0, 0, 0, 0, '', '', '', - ], [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=2, tzinfo=timezone.utc), - 199.5, 200.8, 199, 200.9, 0, 0, 0, 0, '', '', '', - ], [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=3, tzinfo=timezone.utc), - 200.5, 210.5, 193, 210.5, 0, 0, 0, 0, '', '', '', # ROI sell (?) - ], [ - pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=4, tzinfo=timezone.utc), - 200, 200.1, 193, 199, 0, 0, 0, 0, '', '', '', - ], - ], columns=['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long', - 'enter_short', 'exit_short', 'long_tag', 'short_tag', 'exit_tag'] - ) # No data available. res = backtesting._get_exit_trade_entry(trade, row_sell, True) @@ -702,17 +681,6 @@ def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None: res = backtesting._get_exit_trade_entry(trade, row, True) assert res is None - # Assign backtest-detail data - backtesting.detail_data[pair] = row_detail - - res = backtesting._get_exit_trade_entry(trade, row_sell, True) - assert res is not None - assert res.exit_reason == ExitType.ROI.value - # Sell at minute 3 (not available above!) - assert res.close_date_utc == datetime(2020, 1, 1, 5, 3, tzinfo=timezone.utc) - sell_order = res.select_order('sell', True) - assert sell_order is not None - def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: default_conf['use_exit_signal'] = False From 2c1330a4e29abfbea95df7e7cfa994faffe3dd81 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Nov 2022 08:32:27 +0100 Subject: [PATCH 7/7] Update docs to new behavior --- docs/backtesting.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index e3cddb7a1..bfe0f4d07 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -583,7 +583,8 @@ To utilize this, you can append `--timeframe-detail 5m` to your regular backtest freqtrade backtesting --strategy AwesomeStrategy --timeframe 1h --timeframe-detail 5m ``` -This will load 1h data as well as 5m data for the timeframe. The strategy will be analyzed with the 1h timeframe - and for every "open trade candle" (candles where a trade is open) the 5m data will be used to simulate intra-candle movements. +This will load 1h data as well as 5m data for the timeframe. The strategy will be analyzed with the 1h timeframe, and Entry orders will only be placed at the main timeframe, however Order fills and exit signals will be evaluated at the 5m candle, simulating intra-candle movements. + All callback functions (`custom_exit()`, `custom_stoploss()`, ... ) will be running for each 5m candle once the trade is opened (so 12 times in the above example of 1h timeframe, and 5m detailed timeframe). `--timeframe-detail` must be smaller than the original timeframe, otherwise backtesting will fail to start.