From 16b6b08227b12290d613f140d4c23db9a7a50d3f Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 14:42:41 +0300 Subject: [PATCH 01/41] Update docs to include info on new functionality. --- docs/bot-basics.md | 4 ++- docs/strategy-callbacks.md | 73 +++++++++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index e45e3d9ca..0ee585a15 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -24,7 +24,7 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and * Fetch open trades from persistence. * Calculate current list of tradable pairs. -* Download OHLCV data for the pairlist including all [informative pairs](strategy-customization.md#get-data-for-non-tradeable-pairs) +* Download OHLCV data for the pairlist including all [informative pairs](strategy-customization.md#get-data-for-non-tradeable-pairs) This step is only executed once per Candle to avoid unnecessary network traffic. * Call `bot_loop_start()` strategy callback. * Analyze strategy per pair. @@ -34,6 +34,8 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and * Check timeouts for open orders. * Calls `check_entry_timeout()` strategy callback for open entry orders. * Calls `check_exit_timeout()` strategy callback for open exit orders. +* Check readjustment request for open orders. + * Calls `readjust_entry_price()` strategy callback for open entry orders. * Verifies existing positions and eventually places exit orders. * Considers stoploss, ROI and exit-signal, `custom_exit()` and `custom_stoploss()`. * Determine exit-price based on `exit_pricing` configuration setting or by using the `custom_exit_price()` callback. diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index bd32f41c3..94b1230b3 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -16,6 +16,7 @@ Currently available callbacks: * [`confirm_trade_entry()`](#trade-entry-buy-order-confirmation) * [`confirm_trade_exit()`](#trade-exit-sell-order-confirmation) * [`adjust_trade_position()`](#adjust-trade-position) +* [`readjust_entry_price()`](#readjust-entry-price) * [`leverage()`](#leverage-callback) !!! Tip "Callback calling sequence" @@ -365,13 +366,13 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) new_entryprice = dataframe['bollinger_10_lowerband'].iat[-1] - + return new_entryprice def custom_exit_price(self, pair: str, trade: Trade, @@ -381,14 +382,14 @@ class AwesomeStrategy(IStrategy): dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) new_exitprice = dataframe['bollinger_10_upperband'].iat[-1] - + return new_exitprice ``` !!! Warning - Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. - **Example**: + Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. + **Example**: If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98, which is 2% below the current (proposed) rate. !!! Warning "Backtesting" @@ -430,7 +431,7 @@ class AwesomeStrategy(IStrategy): 'exit': 60 * 25 } - def check_entry_timeout(self, pair: str, trade: 'Trade', order: dict, + def check_entry_timeout(self, pair: str, trade: 'Trade', order: dict, current_time: datetime, **kwargs) -> bool: if trade.open_rate > 100 and trade.open_date_utc < current_time - timedelta(minutes=5): return True @@ -508,7 +509,7 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, current_time: datetime, entry_tag: Optional[str], + time_in_force: str, current_time: datetime, entry_tag: Optional[str], side: str, **kwargs) -> bool: """ Called right before placing a entry order. @@ -616,35 +617,35 @@ from freqtrade.persistence import Trade class DigDeeperStrategy(IStrategy): - + position_adjustment_enable = True - + # Attempts to handle large drops with DCA. High stoploss is required. stoploss = -0.30 - + # ... populate_* methods - + # Example specific variables max_entry_position_adjustment = 3 # This number is explained a bit further down max_dca_multiplier = 5.5 - + # This is called when placing the initial order (opening trade) def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, proposed_stake: float, min_stake: float, max_stake: float, entry_tag: Optional[str], side: str, **kwargs) -> float: - + # We need to leave most of the funds for possible further DCA orders # This also applies to fixed stakes return proposed_stake / self.max_dca_multiplier - + def adjust_trade_position(self, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, min_stake: float, max_stake: float, **kwargs): """ Custom trade adjustment logic, returning the stake amount that a trade should be increased. This means extra buy orders with additional fees. - + :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Current buy rate. @@ -654,7 +655,7 @@ class DigDeeperStrategy(IStrategy): :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: Stake amount to adjust your trade """ - + if current_profit > -0.05: return None @@ -689,6 +690,46 @@ class DigDeeperStrategy(IStrategy): ``` +## Readjust Entry Price + +The `readjust_entry_price()` callback may be used by strategy developer to refresh/replace limit orders upon arrival of new candles. +Be aware that `custom_entry_price()` is still the one dictating initial entry limit order price target at the time of entry trigger. + +!!! Warning This mechanism will not trigger if previous orders were partially or fully filled. + +!!! Warning Entry `unfilledtimeout` mechanism takes precedence over this. Be sure to update timeout values to match your expectancy. + +```python +from freqtrade.persistence import Trade +from datetime import timedelta + +class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + def readjust_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: + """ + Entry price readjustment logic, returning the readjusted entry price. + + :param pair: Pair that's currently analyzed + :param trade: Trade object. + :param current_time: datetime object, containing the current datetime + :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. + :param side: 'long' or 'short' - indicating the direction of the proposed trade + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New entry price value if provided + + """ + # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair. + if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc: + dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) + current_candle = dataframe.iloc[-1].squeeze() + return current_candle['sma_200'] + return proposed_rate +``` + ## Leverage Callback When trading in markets that allow leverage, this method must return the desired Leverage (Defaults to 1 -> No leverage). From e5d4f7766e507dab3e0a5171d30cb965c1961452 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 14:44:41 +0300 Subject: [PATCH 02/41] Add new cancel reason for when replacing orders. --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c6a2ab5d3..cd04a71f1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -478,6 +478,7 @@ CANCEL_REASON = { "ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)", "CANCELLED_ON_EXCHANGE": "cancelled on exchange", "FORCE_EXIT": "forcesold", + "REPLACE": "cancelled to be replaced by new limit order", } # List of pairs with their timeframes From 76c545ba0d7098e5b084041655b1e5176645a546 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:03:09 +0300 Subject: [PATCH 03/41] Reorganize, rename, redescribe and add new functionality --- freqtrade/freqtradebot.py | 104 +++++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 29 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 57d7cac3c..cdb8a4bcf 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -22,6 +22,7 @@ from freqtrade.enums import (ExitCheckTuple, ExitType, RPCMessageType, RunMode, from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.mixins import LoggingMixin from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db @@ -188,8 +189,8 @@ class FreqtradeBot(LoggingMixin): self.strategy.analyze(self.active_pair_whitelist) with self._exit_lock: - # Check and handle any timed out open orders - self.check_handle_timedout() + # Check for exchange cancelations, timeouts and user requested replace + self.manage_open_orders() # Protect from collisions with force_exit. # Without this, freqtrade my try to recreate stoploss_on_exchange orders @@ -1123,13 +1124,13 @@ class FreqtradeBot(LoggingMixin): return True return False - def check_handle_timedout(self) -> None: + def manage_open_orders(self) -> None: """ - Check if any orders are timed out and cancel if necessary - :param timeoutvalue: Number of minutes until order is considered timed out + Management of open orders on exchange. Unfilled orders might be cancelled if timeout + was met or replaced if there's a new candle and user has requested it. + Timeout setting takes priority over limit order adjustment request. :return: None """ - for trade in Trade.get_open_order_trades(): try: if not trade.open_order_id: @@ -1140,33 +1141,78 @@ class FreqtradeBot(LoggingMixin): continue fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order) - is_entering = order['side'] == trade.entry_side not_closed = order['status'] == 'open' or fully_cancelled - max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) - order_obj = trade.select_order_by_order_id(trade.open_order_id) - if not_closed and (fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( - trade, order_obj, datetime.now(timezone.utc))) - ): - if is_entering: - self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) + if not_closed: + if fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( + trade, order_obj, datetime.now(timezone.utc))): + self.handle_timedout_orders(order, trade) else: - canceled = self.handle_cancel_exit( - trade, order, constants.CANCEL_REASON['TIMEOUT']) - canceled_count = trade.get_exit_order_count() - max_timeouts = self.config.get( - 'unfilledtimeout', {}).get('exit_timeout_count', 0) - if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: - logger.warning(f'Emergency exiting trade {trade}, as the exit order ' - f'timed out {max_timeouts} times.') - try: - self.execute_trade_exit( - trade, order.get('price'), - exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT)) - except DependencyException as exception: - logger.warning( - f'Unable to emergency sell trade {trade.pair}: {exception}') + self.replace_orders(order, order_obj, trade) + + def handle_timedout_orders(self, order: Dict, trade: Trade) -> None: + """ + Check if any orders are timed out and cancel if necessary. + :param order: Order dict grabbed with exchange.fetch_order() + :param trade: Trade object. + :return: None + """ + if order['side'] == trade.entry_side: + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) + else: + canceled = self.handle_cancel_exit( + trade, order, constants.CANCEL_REASON['TIMEOUT']) + canceled_count = trade.get_exit_order_count() + max_timeouts = self.config.get('unfilledtimeout', {}).get('exit_timeout_count', 0) + if canceled and max_timeouts > 0 and canceled_count >= max_timeouts: + logger.warning(f'Emergency exiting trade {trade}, as the exit order ' + f'timed out {max_timeouts} times.') + try: + self.execute_trade_exit( + trade, order['price'], + exit_check=ExitCheckTuple(exit_type=ExitType.EMERGENCY_EXIT)) + except DependencyException as exception: + logger.warning( + f'Unable to emergency sell trade {trade.pair}: {exception}') + + def replace_orders(self, order: Dict, order_obj: Optional[Order], trade: Trade) -> None: + """ + Check if any orders should be replaced and do so + :param order: Order dict grabbed with exchange.fetch_order() + :param order_obj: Order object. + :param trade: Trade object. + :return: None + """ + analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, + self.strategy.timeframe) + latest_candle_open_date = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None + latest_candle_close_date = timeframe_to_next_date(self.strategy.timeframe, + latest_candle_open_date) + # Check if new candle + if order_obj and latest_candle_close_date.replace(tzinfo=None) > order_obj.order_date: + # New candle + proposed_rate = self.exchange.get_rate( + trade.pair, side='entry', is_short=trade.is_short, refresh=True) + adjusted_entry_price = strategy_safe_wrapper(self.strategy.readjust_entry_price, + default_retval=proposed_rate)( + pair=trade.pair, current_time=datetime.now(timezone.utc), + proposed_rate=proposed_rate, entry_tag=trade.enter_tag, + side=trade.entry_side) + # check if user has requested entry limit adjustment + if proposed_rate != adjusted_entry_price: + # cancel existing order + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['REPLACE'], + allow_full_cancel=False) + stake = self.wallets.get_trade_stake_amount(trade.pair, self.edge) + # place new order with requested price + self.execute_entry( + pair=trade.pair, + stake_amount=stake, + price=adjusted_entry_price, + trade=trade, + is_short=trade.is_short + ) def cancel_all_open_orders(self) -> None: """ From 317c1e0746def78e37f895c1c3640f727f3f137c Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:03:44 +0300 Subject: [PATCH 04/41] Add option to handle_cancel_enter to prevent closing trade. --- freqtrade/freqtradebot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cdb8a4bcf..473ad9a8d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1234,7 +1234,10 @@ class FreqtradeBot(LoggingMixin): self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) Trade.commit() - def handle_cancel_enter(self, trade: Trade, order: Dict, reason: str) -> bool: + def handle_cancel_enter( + self, trade: Trade, order: Dict, reason: str, + allow_full_cancel: Optional[bool] = True + ) -> bool: """ Buy cancel - cancel order :return: True if order was fully cancelled @@ -1274,7 +1277,7 @@ class FreqtradeBot(LoggingMixin): if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): logger.info(f'{side} order fully cancelled. Removing {trade} from database.') # if trade is not partially completed and it's the only order, just delete the trade - if len(trade.orders) <= 1: + if len(trade.orders) <= 1 and allow_full_cancel: trade.delete() was_trade_fully_canceled = True reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" From f8a7fdd5edd7d590c348037d8f2376b1873cae3c Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:04:22 +0300 Subject: [PATCH 05/41] Add new callback to strategy interface. --- freqtrade/strategy/interface.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index ba2eb9636..aeda66d4e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -467,6 +467,28 @@ class IStrategy(ABC, HyperStrategyMixin): """ return None + def readjust_entry_price(self, trade: Trade, pair: str, current_time: datetime, + proposed_rate: float, entry_tag: Optional[str], + side: str, **kwargs) -> float: + """ + Entry price readjustment logic, returning the readjusted entry price. + This only executes when a order was already placed, open(unfilled) and not timed out on + subsequent candles. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + :param pair: Pair that's currently analyzed + :param trade: Trade object. + :param current_time: datetime object, containing the current datetime + :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. + :param side: 'long' or 'short' - indicating the direction of the proposed trade + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New entry price value if provided + + """ + return proposed_rate + def leverage(self, pair: str, current_time: datetime, current_rate: float, proposed_leverage: float, max_leverage: float, side: str, **kwargs) -> float: From bf5799ef9eff2b821e6600997f2bb97d0df1d72a Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:07:18 +0300 Subject: [PATCH 06/41] Add new functionality to backtesting. --- freqtrade/optimize/backtesting.py | 36 ++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbb220e45..f91013585 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -635,7 +635,7 @@ class Backtesting: def get_valid_price_and_stake( self, pair: str, row: Tuple, propose_rate: float, stake_amount: Optional[float], direction: LongShort, current_time: datetime, entry_tag: Optional[str], - trade: Optional[LocalTrade], order_type: str + trade: Optional[LocalTrade], order_type: str, readjust_req: Optional[bool] = False ) -> Tuple[float, float, float, float]: if order_type == 'limit': @@ -645,6 +645,14 @@ class Backtesting: proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate + if readjust_req: + propose_rate = strategy_safe_wrapper(self.strategy.readjust_entry_price, + default_retval=propose_rate)( + pair=pair, current_time=current_time, + proposed_rate=propose_rate, entry_tag=entry_tag, + side=direction + ) # default value is open rate or custom rate from before + # We can't place orders higher than current high (otherwise it'd be a stop limit buy) # which freqtrade does not support in live. if direction == "short": @@ -652,7 +660,7 @@ class Backtesting: else: propose_rate = min(propose_rate, row[HIGH_IDX]) - pos_adjust = trade is not None + pos_adjust = trade is not None and readjust_req is False leverage = trade.leverage if trade else 1.0 if not pos_adjust: try: @@ -697,17 +705,18 @@ class Backtesting: def _enter_trade(self, pair: str, row: Tuple, direction: LongShort, stake_amount: Optional[float] = None, - trade: Optional[LocalTrade] = None) -> Optional[LocalTrade]: + trade: Optional[LocalTrade] = None, + readjust_req: Optional[bool] = False) -> Optional[LocalTrade]: current_time = row[DATE_IDX].to_pydatetime() entry_tag = row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None # let's call the custom entry price, using the open price as default price order_type = self.strategy.order_types['entry'] - pos_adjust = trade is not None + pos_adjust = trade is not None and readjust_req is False propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake( pair, row, row[OPEN_IDX], stake_amount, direction, current_time, entry_tag, trade, - order_type + order_type, readjust_req ) if not stake_amount: @@ -850,6 +859,21 @@ class Backtesting: self.protections.stop_per_pair(pair, current_time) self.protections.global_stop(current_time) + def check_order_replace(self, trade: LocalTrade, current_time, row: Tuple) -> None: + """ + Check if an entry order has to be replaced and do so. + Returns None. + """ + for order in [o for o in trade.orders if o.ft_is_open]: + if order.side == trade.entry_side and current_time > order.order_date_utc: + # cancel existing order + del trade.orders[trade.orders.index(order)] + + # place new order + self._enter_trade(pair=trade.pair, row=row, trade=trade, + direction='short' if trade.is_short else 'long', + readjust_req=True) + def check_order_cancel(self, trade: LocalTrade, current_time) -> bool: """ Check if an order has been canceled. @@ -949,6 +973,8 @@ class Backtesting: open_trade_count -= 1 open_trades[pair].remove(t) self.wallets.update() + else: + self.check_order_replace(t, current_time, row) # 2. Process buys. # without positionstacking, we can only have one open trade per pair. From 452f44206a448363886f5cac4c04fcd674cf845d Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:08:09 +0300 Subject: [PATCH 07/41] Add new callback to advanced template. --- .../subtemplates/strategy_methods_advanced.j2 | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 17dfa0873..c7e69d3e4 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -30,6 +30,27 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: """ return proposed_rate +def readjust_entry_price(self, trade: Trade, pair: str, current_time: datetime, + proposed_rate: float, entry_tag: Optional[str], + side: str, **kwargs) -> float: + """ + Entry price readjustment logic, returning the readjusted entry price. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ + + When not implemented by a strategy, returns proposed_rate and orders are not replaced. + + :param pair: Pair that's currently analyzed + :param trade: Trade object. + :param current_time: datetime object, containing the current datetime + :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. + :param side: 'long' or 'short' - indicating the direction of the proposed trade + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New entry price value if provided + """ + return proposed_rate + def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, current_profit: float, **kwargs) -> float: From 237d116d8cd707a091f77aa43bfc17de88ba73f9 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:08:54 +0300 Subject: [PATCH 08/41] Update existing tests to use the new func name. --- tests/test_freqtradebot.py | 67 +++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3737c7c05..724b7fd56 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2363,7 +2363,7 @@ def test_bot_loop_start_called_once(mocker, default_conf_usdt, caplog): @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_entry_usercustom( +def test_manage_open_orders_entry_usercustom( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, limit_sell_order_old, fee, mocker, is_short ) -> None: @@ -2395,12 +2395,12 @@ def test_check_handle_timedout_entry_usercustom( Trade.query.session.add(open_trade) # Ensure default is to return empty (so not mocked yet) - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 # Return false - trade remains open freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() nb_trades = len(trades) @@ -2408,7 +2408,7 @@ def test_check_handle_timedout_entry_usercustom( assert freqtrade.strategy.check_entry_timeout.call_count == 1 freqtrade.strategy.check_entry_timeout = MagicMock(side_effect=KeyError) - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() nb_trades = len(trades) @@ -2417,7 +2417,7 @@ def test_check_handle_timedout_entry_usercustom( freqtrade.strategy.check_entry_timeout = MagicMock(return_value=True) # Trade should be closed since the function returns true - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_wr_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() @@ -2427,7 +2427,7 @@ def test_check_handle_timedout_entry_usercustom( @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_entry( +def test_manage_open_orders_entry( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, limit_sell_order_old, fee, mocker, is_short ) -> None: @@ -2452,7 +2452,7 @@ def test_check_handle_timedout_entry( freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() @@ -2461,7 +2461,6 @@ def test_check_handle_timedout_entry( # Custom user buy-timeout is never called assert freqtrade.strategy.check_entry_timeout.call_count == 0 - @pytest.mark.parametrize("is_short", [False, True]) def test_check_handle_cancelled_buy( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, @@ -2485,7 +2484,7 @@ def test_check_handle_cancelled_buy( Trade.query.session.add(open_trade) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() @@ -2496,7 +2495,7 @@ def test_check_handle_cancelled_buy( @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_buy_exception( +def test_manage_open_orders_buy_exception( default_conf_usdt, ticker_usdt, open_trade, is_short, fee, mocker ) -> None: rpc_mock = patch_RPCManager(mocker) @@ -2516,7 +2515,7 @@ def test_check_handle_timedout_buy_exception( Trade.query.session.add(open_trade) # check it does cancel buy orders over the time limit - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 0 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() @@ -2525,7 +2524,7 @@ def test_check_handle_timedout_buy_exception( @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_exit_usercustom( +def test_manage_open_orders_exit_usercustom( default_conf_usdt, ticker_usdt, limit_sell_order_old, mocker, is_short, open_trade_usdt, caplog ) -> None: @@ -2554,13 +2553,13 @@ def test_check_handle_timedout_exit_usercustom( Trade.query.session.add(open_trade_usdt) # Ensure default is false - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 freqtrade.strategy.check_exit_timeout = MagicMock(return_value=False) freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) # Return false - No impact - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 0 assert open_trade_usdt.is_open is False @@ -2570,7 +2569,7 @@ def test_check_handle_timedout_exit_usercustom( freqtrade.strategy.check_exit_timeout = MagicMock(side_effect=KeyError) freqtrade.strategy.check_entry_timeout = MagicMock(side_effect=KeyError) # Return Error - No impact - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 0 assert open_trade_usdt.is_open is False @@ -2580,7 +2579,7 @@ def test_check_handle_timedout_exit_usercustom( # Return True - sells! freqtrade.strategy.check_exit_timeout = MagicMock(return_value=True) freqtrade.strategy.check_entry_timeout = MagicMock(return_value=True) - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert open_trade_usdt.is_open is True @@ -2593,7 +2592,7 @@ def test_check_handle_timedout_exit_usercustom( mocker.patch('freqtrade.persistence.Trade.get_exit_order_count', return_value=1) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit', side_effect=DependencyException) - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert log_has_re('Unable to emergency sell .*', caplog) et_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit') @@ -2603,16 +2602,16 @@ def test_check_handle_timedout_exit_usercustom( # If cancelling fails - no emergency sell! with patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit', return_value=False): - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert et_mock.call_count == 0 - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert log_has_re('Emergency exiting trade.*', caplog) assert et_mock.call_count == 1 @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_exit( +def test_manage_open_orders_exit( default_conf_usdt, ticker_usdt, limit_sell_order_old, mocker, is_short, open_trade_usdt ) -> None: rpc_mock = patch_RPCManager(mocker) @@ -2639,7 +2638,7 @@ def test_check_handle_timedout_exit( freqtrade.strategy.check_exit_timeout = MagicMock(return_value=False) freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert open_trade_usdt.is_open is True @@ -2675,7 +2674,7 @@ def test_check_handle_cancelled_exit( Trade.query.session.add(open_trade_usdt) # check it does cancel sell orders over the time limit - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 assert open_trade_usdt.is_open is True @@ -2685,7 +2684,7 @@ def test_check_handle_cancelled_exit( @pytest.mark.parametrize("is_short", [False, True]) @pytest.mark.parametrize("leverage", [1, 3, 5, 10]) -def test_check_handle_timedout_partial( +def test_manage_open_orders_partial( default_conf_usdt, ticker_usdt, limit_buy_order_old_partial, is_short, leverage, open_trade, mocker ) -> None: @@ -2711,7 +2710,7 @@ def test_check_handle_timedout_partial( # check it does cancel buy orders over the time limit # note this is for a partially-complete buy order - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() @@ -2722,7 +2721,7 @@ def test_check_handle_timedout_partial( @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_partial_fee( +def test_manage_open_orders_partial_fee( default_conf_usdt, ticker_usdt, open_trade, caplog, fee, is_short, limit_buy_order_old_partial, trades_for_order, limit_buy_order_old_partial_canceled, mocker @@ -2754,7 +2753,7 @@ def test_check_handle_timedout_partial_fee( Trade.query.session.add(open_trade) # cancelling a half-filled order should update the amount to the bought amount # and apply fees if necessary. - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert log_has_re(r"Applying fee on amount for Trade.*", caplog) @@ -2771,7 +2770,7 @@ def test_check_handle_timedout_partial_fee( @pytest.mark.parametrize("is_short", [False, True]) -def test_check_handle_timedout_partial_except( +def test_manage_open_orders_partial_except( default_conf_usdt, ticker_usdt, open_trade, caplog, fee, is_short, limit_buy_order_old_partial, trades_for_order, limit_buy_order_old_partial_canceled, mocker @@ -2802,7 +2801,7 @@ def test_check_handle_timedout_partial_except( Trade.query.session.add(open_trade) # cancelling a half-filled order should update the amount to the bought amount # and apply fees if necessary. - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert log_has_re(r"Could not update trade amount: .*", caplog) @@ -2818,7 +2817,7 @@ def test_check_handle_timedout_partial_except( assert trades[0].fee_open == fee() -def test_check_handle_timedout_exception(default_conf_usdt, ticker_usdt, open_trade_usdt, mocker, +def test_manage_open_orders_exception(default_conf_usdt, ticker_usdt, open_trade_usdt, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2840,7 +2839,7 @@ def test_check_handle_timedout_exception(default_conf_usdt, ticker_usdt, open_tr Trade.query.session.add(open_trade_usdt) caplog.clear() - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() assert log_has_re(r"Cannot query order for Trade\(id=1, pair=ADA/USDT, amount=30.00000000, " r"is_short=False, leverage=1.0, " r"open_rate=2.00000000, open_since=" @@ -3397,7 +3396,7 @@ def test_execute_trade_exit_with_stoploss_on_exchange( assert trade trades = [trade] - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() freqtrade.exit_positions(trades) # Increase the price and sell it @@ -3449,7 +3448,7 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit( # Create some test data freqtrade.enter_positions() - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() trade = Trade.query.first() trades = [trade] assert trade.stoploss_order_id is None @@ -5212,7 +5211,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: assert trade.stake_amount == 110 assert not trade.fee_updated('buy') - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() trade = Trade.query.first() assert trade @@ -5318,7 +5317,7 @@ def test_position_adjust(mocker, default_conf_usdt, fee) -> None: MagicMock(return_value=closed_dca_order_1)) mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', MagicMock(return_value=closed_dca_order_1)) - freqtrade.check_handle_timedout() + freqtrade.manage_open_orders() # Assert trade is as expected (averaged dca) trade = Trade.query.first() From 698c25f133658ca88c5db7e00d8dbe665bb851f5 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 16 Apr 2022 15:44:07 +0300 Subject: [PATCH 09/41] Fix issues reported by flake. --- tests/test_freqtradebot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 724b7fd56..2026872de 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2461,6 +2461,7 @@ def test_manage_open_orders_entry( # Custom user buy-timeout is never called assert freqtrade.strategy.check_entry_timeout.call_count == 0 + @pytest.mark.parametrize("is_short", [False, True]) def test_check_handle_cancelled_buy( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, @@ -2818,7 +2819,7 @@ def test_manage_open_orders_partial_except( def test_manage_open_orders_exception(default_conf_usdt, ticker_usdt, open_trade_usdt, mocker, - caplog) -> None: + caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() From 17da4ca09939ae56256d7a5d71edf7ff1babbdcd Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 17 Apr 2022 12:11:30 +0300 Subject: [PATCH 10/41] Use order_date_utc --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 473ad9a8d..a020754eb 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1190,7 +1190,7 @@ class FreqtradeBot(LoggingMixin): latest_candle_close_date = timeframe_to_next_date(self.strategy.timeframe, latest_candle_open_date) # Check if new candle - if order_obj and latest_candle_close_date.replace(tzinfo=None) > order_obj.order_date: + if order_obj and latest_candle_close_date > order_obj.order_date_utc: # New candle proposed_rate = self.exchange.get_rate( trade.pair, side='entry', is_short=trade.is_short, refresh=True) From 541147c801ff48b4bcc5fbd0310bf0701219419e Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 18 Apr 2022 21:13:50 +0300 Subject: [PATCH 11/41] Update documentation to match feature changes. --- docs/bot-basics.md | 2 +- docs/strategy-callbacks.md | 34 ++++++++++++++++++++++------------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 0ee585a15..abc0e7b16 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -35,7 +35,7 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and * Calls `check_entry_timeout()` strategy callback for open entry orders. * Calls `check_exit_timeout()` strategy callback for open exit orders. * Check readjustment request for open orders. - * Calls `readjust_entry_price()` strategy callback for open entry orders. + * Calls `adjust_entry_price()` strategy callback for open entry orders. * Verifies existing positions and eventually places exit orders. * Considers stoploss, ROI and exit-signal, `custom_exit()` and `custom_stoploss()`. * Determine exit-price based on `exit_pricing` configuration setting or by using the `custom_exit_price()` callback. diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 5f3e46be9..c78a2c7e5 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -16,7 +16,7 @@ Currently available callbacks: * [`confirm_trade_entry()`](#trade-entry-buy-order-confirmation) * [`confirm_trade_exit()`](#trade-exit-sell-order-confirmation) * [`adjust_trade_position()`](#adjust-trade-position) -* [`readjust_entry_price()`](#readjust-entry-price) +* [`adjust_entry_price()`](#adjust-entry-price) * [`leverage()`](#leverage-callback) !!! Tip "Callback calling sequence" @@ -389,7 +389,7 @@ class AwesomeStrategy(IStrategy): !!! Warning Modifying entry and exit prices will only work for limit orders. Depending on the price chosen, this can result in a lot of unfilled orders. By default the maximum allowed distance between the current price and the custom price is 2%, this value can be changed in config with the `custom_price_max_distance_ratio` parameter. - **Example**: + **Example**: If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98, which is 2% below the current (proposed) rate. !!! Warning "Backtesting" @@ -690,14 +690,16 @@ class DigDeeperStrategy(IStrategy): ``` -## Readjust Entry Price +## Adjust Entry Price -The `readjust_entry_price()` callback may be used by strategy developer to refresh/replace limit orders upon arrival of new candles. +The `adjust_entry_price()` callback may be used by strategy developer to refresh/replace limit orders upon arrival of new candles. Be aware that `custom_entry_price()` is still the one dictating initial entry limit order price target at the time of entry trigger. -!!! Warning This mechanism will not trigger if previous orders were partially or fully filled. +!!! Note "Simple Order Cancelation" + This also allows simple cancelation without an replacement order. This behavior occurs when `None` is returned. -!!! Warning Entry `unfilledtimeout` mechanism takes precedence over this. Be sure to update timeout values to match your expectancy. +!!! Warning + Entry `unfilledtimeout` mechanism takes precedence over this. Be sure to update timeout values to match your expectancy. ```python from freqtrade.persistence import Trade @@ -707,13 +709,17 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def readjust_entry_price(self, pair: str, current_time: datetime, proposed_rate: float, - entry_tag: Optional[str], side: str, **kwargs) -> float: + def adjust_entry_price(self, trade: Trade, order: Order, pair: str, + current_time: datetime, proposed_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: """ - Entry price readjustment logic, returning the readjusted entry price. + Entry price re-adjustment logic, returning the user desired limit price. + This only executes when a order was already placed, still open(unfilled fully or partially) + and not timed out on subsequent candles after entry trigger. :param pair: Pair that's currently analyzed :param trade: Trade object. + :param order: Order object :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. @@ -724,9 +730,13 @@ class AwesomeStrategy(IStrategy): """ # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair. if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc: - dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) - current_candle = dataframe.iloc[-1].squeeze() - return current_candle['sma_200'] + # just cancel the order if it has been filled more than half of the ammount + if order.filled > order.remaining: + return None + else: + dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) + current_candle = dataframe.iloc[-1].squeeze() + return current_candle['sma_200'] return proposed_rate ``` From 2cac1b7dcc1e026a1529787ed7889d24167a9dde Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 18 Apr 2022 21:14:35 +0300 Subject: [PATCH 12/41] Add new (user cancellation) reason. --- freqtrade/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index cd04a71f1..fba6c968d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -479,6 +479,7 @@ CANCEL_REASON = { "CANCELLED_ON_EXCHANGE": "cancelled on exchange", "FORCE_EXIT": "forcesold", "REPLACE": "cancelled to be replaced by new limit order", + "USER_CANCEL": "user requested order cancel" } # List of pairs with their timeframes From 95e009b9cbbe872333474fd39cd900fb95bb4247 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 18 Apr 2022 21:16:45 +0300 Subject: [PATCH 13/41] Update adjustment functionality and add cancelation option --- freqtrade/freqtradebot.py | 53 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a020754eb..8d710a760 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1147,13 +1147,13 @@ class FreqtradeBot(LoggingMixin): if not_closed: if fully_cancelled or (order_obj and self.strategy.ft_check_timed_out( trade, order_obj, datetime.now(timezone.utc))): - self.handle_timedout_orders(order, trade) + self.handle_timedout_order(order, trade) else: - self.replace_orders(order, order_obj, trade) + self.replace_order(order, order_obj, trade) - def handle_timedout_orders(self, order: Dict, trade: Trade) -> None: + def handle_timedout_order(self, order: Dict, trade: Trade) -> None: """ - Check if any orders are timed out and cancel if necessary. + Check if current analyzed order timed out and cancel if necessary. :param order: Order dict grabbed with exchange.fetch_order() :param trade: Trade object. :return: None @@ -1176,9 +1176,11 @@ class FreqtradeBot(LoggingMixin): logger.warning( f'Unable to emergency sell trade {trade.pair}: {exception}') - def replace_orders(self, order: Dict, order_obj: Optional[Order], trade: Trade) -> None: + def replace_order(self, order: Dict, order_obj: Optional[Order], trade: Trade) -> None: """ - Check if any orders should be replaced and do so + Check if current analyzed entry order should be replaced. Analyzed order is canceled + if adjust_entry_price() returned price differs from proposed_rate. + New order is only placed if adjust_entry_price() returned price is not None. :param order: Order dict grabbed with exchange.fetch_order() :param order_obj: Order object. :param trade: Trade object. @@ -1194,25 +1196,30 @@ class FreqtradeBot(LoggingMixin): # New candle proposed_rate = self.exchange.get_rate( trade.pair, side='entry', is_short=trade.is_short, refresh=True) - adjusted_entry_price = strategy_safe_wrapper(self.strategy.readjust_entry_price, + adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price, default_retval=proposed_rate)( - pair=trade.pair, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_rate, entry_tag=trade.enter_tag, - side=trade.entry_side) - # check if user has requested entry limit adjustment + trade=trade, order=order_obj, pair=trade.pair, + current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, + entry_tag=trade.enter_tag, side=trade.entry_side) + + full_cancel = False + cancel_reason = constants.CANCEL_REASON['REPLACE'] + if not adjusted_entry_price: + full_cancel = True + cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] if proposed_rate != adjusted_entry_price: - # cancel existing order - self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['REPLACE'], - allow_full_cancel=False) - stake = self.wallets.get_trade_stake_amount(trade.pair, self.edge) - # place new order with requested price - self.execute_entry( - pair=trade.pair, - stake_amount=stake, - price=adjusted_entry_price, - trade=trade, - is_short=trade.is_short - ) + # cancel existing order if new price is supplied or None + self.handle_cancel_enter(trade, order, cancel_reason, + allow_full_cancel=full_cancel) + if adjusted_entry_price: + # place new order only if new price is supplied + self.execute_entry( + pair=trade.pair, + stake_amount=(order_obj.remaining * order_obj.price), + price=adjusted_entry_price, + trade=trade, + is_short=trade.is_short + ) def cancel_all_open_orders(self) -> None: """ From 3166739ec9a1f211c72f922b8157619d56538148 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 18 Apr 2022 21:17:39 +0300 Subject: [PATCH 14/41] Update strategy callback params and description. --- freqtrade/strategy/interface.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 57fd07042..01473391a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -467,18 +467,22 @@ class IStrategy(ABC, HyperStrategyMixin): """ return None - def readjust_entry_price(self, trade: Trade, pair: str, current_time: datetime, - proposed_rate: float, entry_tag: Optional[str], - side: str, **kwargs) -> float: + def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, + current_time: datetime, proposed_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: """ - Entry price readjustment logic, returning the readjusted entry price. - This only executes when a order was already placed, open(unfilled) and not timed out on - subsequent candles. + Entry price re-adjustment logic, returning the user desired limit price. + This only executes when a order was already placed, still open(unfilled fully or partially) + and not timed out on subsequent candles after entry trigger. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + When not implemented by a strategy, returns proposed_stake. + If None is returned then order gets canceled but not replaced by a new one. + :param pair: Pair that's currently analyzed :param trade: Trade object. + :param order: Order object :param current_time: datetime object, containing the current datetime :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. From d9f838a65f23f39ea3d4b8c6ffc34d8d5e478422 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 18 Apr 2022 21:20:50 +0300 Subject: [PATCH 15/41] Update template usage to reflect changes. --- .../subtemplates/strategy_methods_advanced.j2 | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 90dbade91..db64b3e07 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -30,26 +30,31 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: """ return proposed_rate -def readjust_entry_price(self, trade: Trade, pair: str, current_time: datetime, - proposed_rate: float, entry_tag: Optional[str], - side: str, **kwargs) -> float: - """ - Entry price readjustment logic, returning the readjusted entry price. + def adjust_entry_price(self, trade: Trade, order: Order, pair: str, + current_time: datetime, proposed_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: + """ + Entry price re-adjustment logic, returning the user desired limit price. + This only executes when a order was already placed, still open(unfilled fully or partially) + and not timed out on subsequent candles after entry trigger. - For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ - When not implemented by a strategy, returns proposed_rate and orders are not replaced. + When not implemented by a strategy, returns proposed_stake. + If None is returned then order gets canceled but not replaced by a new one. - :param pair: Pair that's currently analyzed - :param trade: Trade object. - :param current_time: datetime object, containing the current datetime - :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. - :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. - :param side: 'long' or 'short' - indicating the direction of the proposed trade - :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New entry price value if provided - """ - return proposed_rate + :param pair: Pair that's currently analyzed + :param trade: Trade object. + :param order: Order object + :param current_time: datetime object, containing the current datetime + :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. + :param side: 'long' or 'short' - indicating the direction of the proposed trade + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New entry price value if provided + + """ + return proposed_rate def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, From d24ee9032a46e59f57fdb256637a02c813ef98cd Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 18 Apr 2022 21:21:38 +0300 Subject: [PATCH 16/41] Update usage in backtest. No functional update. --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f91013585..bf666abf2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -646,7 +646,7 @@ class Backtesting: side=direction, ) # default value is the open rate if readjust_req: - propose_rate = strategy_safe_wrapper(self.strategy.readjust_entry_price, + propose_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price, default_retval=propose_rate)( pair=pair, current_time=current_time, proposed_rate=propose_rate, entry_tag=entry_tag, From 76558f284f15c50264dba74f83bc58ff756d37c8 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Tue, 19 Apr 2022 13:33:37 +0300 Subject: [PATCH 17/41] Fix user cancellation functionality. --- freqtrade/freqtradebot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8d710a760..9febe64fe 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1284,7 +1284,8 @@ class FreqtradeBot(LoggingMixin): if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): logger.info(f'{side} order fully cancelled. Removing {trade} from database.') # if trade is not partially completed and it's the only order, just delete the trade - if len(trade.orders) <= 1 and allow_full_cancel: + open_order_count = len([order for order in trade.orders if order.status == 'open']) + if open_order_count <= 1 and allow_full_cancel: trade.delete() was_trade_fully_canceled = True reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" From 17650d7e6025385f8d9dbd3dbe44d097b50ce9eb Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Fri, 29 Apr 2022 00:10:17 +0300 Subject: [PATCH 18/41] Maintain existing order. Update functionality and documentation --- docs/strategy-callbacks.md | 20 ++++++--- freqtrade/freqtradebot.py | 14 +++--- freqtrade/strategy/interface.py | 12 ++--- .../subtemplates/strategy_methods_advanced.j2 | 44 ++++++++++--------- 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 8da8bab0f..7f86f2610 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -698,6 +698,9 @@ Be aware that `custom_entry_price()` is still the one dictating initial entry li !!! Note "Simple Order Cancelation" This also allows simple cancelation without an replacement order. This behavior occurs when `None` is returned. +!!! Note "Maintaining Order" + Maintaining existing order on exchange is facilitated. This behavior occurs when `order.price` is returned. + !!! Warning Entry `unfilledtimeout` mechanism takes precedence over this. Be sure to update timeout values to match your expectancy. @@ -709,19 +712,24 @@ class AwesomeStrategy(IStrategy): # ... populate_* methods - def adjust_entry_price(self, trade: Trade, order: Order, pair: str, - current_time: datetime, proposed_rate: float, - entry_tag: Optional[str], side: str, **kwargs) -> float: + def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, + current_time: datetime, proposed_rate: float, current_order_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: """ Entry price re-adjustment logic, returning the user desired limit price. This only executes when a order was already placed, still open(unfilled fully or partially) and not timed out on subsequent candles after entry trigger. + When not implemented by a strategy, returns current_order_rate as default. + If current_order_rate is returned then the existing order is maintained. + If None is returned then order gets canceled but not replaced by a new one. + :param pair: Pair that's currently analyzed :param trade: Trade object. :param order: Order object :param current_time: datetime object, containing the current datetime - :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. + :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. + :param current_order_rate: Rate of the existing order in place. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. :param side: 'long' or 'short' - indicating the direction of the proposed trade :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. @@ -736,8 +744,10 @@ class AwesomeStrategy(IStrategy): else: dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) current_candle = dataframe.iloc[-1].squeeze() + # desired price return current_candle['sma_200'] - return proposed_rate + # default: maintain existing order + return current_order_rate ``` ## Leverage Callback diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b55fee35f..330bfcdf0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1165,9 +1165,10 @@ class FreqtradeBot(LoggingMixin): def replace_order(self, order: Dict, order_obj: Optional[Order], trade: Trade) -> None: """ - Check if current analyzed entry order should be replaced. Analyzed order is canceled - if adjust_entry_price() returned price differs from proposed_rate. - New order is only placed if adjust_entry_price() returned price is not None. + Check if current analyzed entry order should be replaced or simply cancelled. + To simply cancel the existing order(no replacement) adjust_entry_price() should return None + To maintain existing order adjust_entry_price() should return order_obj.price + To replace existing order adjust_entry_price() should return desired price for limit order :param order: Order dict grabbed with exchange.fetch_order() :param order_obj: Order object. :param trade: Trade object. @@ -1184,17 +1185,18 @@ class FreqtradeBot(LoggingMixin): proposed_rate = self.exchange.get_rate( trade.pair, side='entry', is_short=trade.is_short, refresh=True) adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price, - default_retval=proposed_rate)( + default_retval=order_obj.price)( trade=trade, order=order_obj, pair=trade.pair, current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, - entry_tag=trade.enter_tag, side=trade.entry_side) + current_order_rate=order_obj.price, entry_tag=trade.enter_tag, + side=trade.entry_side) full_cancel = False cancel_reason = constants.CANCEL_REASON['REPLACE'] if not adjusted_entry_price: full_cancel = True cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] - if proposed_rate != adjusted_entry_price: + if order_obj.price != adjusted_entry_price: # cancel existing order if new price is supplied or None self.handle_cancel_enter(trade, order, cancel_reason, allow_full_cancel=full_cancel) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 0a7580b6f..a472a6943 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -465,30 +465,32 @@ class IStrategy(ABC, HyperStrategyMixin): return None def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, - current_time: datetime, proposed_rate: float, + current_time: datetime, proposed_rate: float, current_order_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ Entry price re-adjustment logic, returning the user desired limit price. This only executes when a order was already placed, still open(unfilled fully or partially) and not timed out on subsequent candles after entry trigger. - For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ - When not implemented by a strategy, returns proposed_stake. + When not implemented by a strategy, returns current_order_rate as default. + If current_order_rate is returned then the existing order is maintained. If None is returned then order gets canceled but not replaced by a new one. :param pair: Pair that's currently analyzed :param trade: Trade object. :param order: Order object :param current_time: datetime object, containing the current datetime - :param proposed_rate: Rate, calculated based on pricing settings in exit_pricing. + :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. + :param current_order_rate: Rate of the existing order in place. :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. :param side: 'long' or 'short' - indicating the direction of the proposed trade :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New entry price value if provided """ - return proposed_rate + return current_order_rate def leverage(self, pair: str, current_time: datetime, current_rate: float, proposed_leverage: float, max_leverage: float, side: str, diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 176f567c7..7f9671bb1 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -30,31 +30,33 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: """ return proposed_rate - def adjust_entry_price(self, trade: Trade, order: Order, pair: str, - current_time: datetime, proposed_rate: float, - entry_tag: Optional[str], side: str, **kwargs) -> float: - """ - Entry price re-adjustment logic, returning the user desired limit price. - This only executes when a order was already placed, still open(unfilled fully or partially) - and not timed out on subsequent candles after entry trigger. +def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, + current_time: datetime, proposed_rate: float, current_order_rate: float, + entry_tag: Optional[str], side: str, **kwargs) -> float: + """ + Entry price re-adjustment logic, returning the user desired limit price. + This only executes when a order was already placed, still open(unfilled fully or partially) + and not timed out on subsequent candles after entry trigger. - For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ - When not implemented by a strategy, returns proposed_stake. - If None is returned then order gets canceled but not replaced by a new one. + When not implemented by a strategy, returns current_order_rate as default. + If current_order_rate is returned then the existing order is maintained. + If None is returned then order gets canceled but not replaced by a new one. - :param pair: Pair that's currently analyzed - :param trade: Trade object. - :param order: Order object - :param current_time: datetime object, containing the current datetime - :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. - :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. - :param side: 'long' or 'short' - indicating the direction of the proposed trade - :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New entry price value if provided + :param pair: Pair that's currently analyzed + :param trade: Trade object. + :param order: Order object + :param current_time: datetime object, containing the current datetime + :param proposed_rate: Rate, calculated based on pricing settings in entry_pricing. + :param current_order_rate: Rate of the existing order in place. + :param entry_tag: Optional entry_tag (buy_tag) if provided with the buy signal. + :param side: 'long' or 'short' - indicating the direction of the proposed trade + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return float: New entry price value if provided - """ - return proposed_rate + """ + return current_order_rate def custom_exit_price(self, pair: str, trade: 'Trade', current_time: 'datetime', proposed_rate: float, From f9977c26e7f682346f875f30fd8b11a9f19cc867 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 30 Apr 2022 12:55:03 +0300 Subject: [PATCH 19/41] Full cancel only for non DCA trades. --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 330bfcdf0..49c7050c9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1194,7 +1194,7 @@ class FreqtradeBot(LoggingMixin): full_cancel = False cancel_reason = constants.CANCEL_REASON['REPLACE'] if not adjusted_entry_price: - full_cancel = True + full_cancel = True if trade.nr_of_successful_entries == 0 else False cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] if order_obj.price != adjusted_entry_price: # cancel existing order if new price is supplied or None From ad0c5d944034228f8e17ba658b3b3e1d8ff4af72 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 30 Apr 2022 13:38:17 +0300 Subject: [PATCH 20/41] Refactor entry adjustment for backtesting. --- freqtrade/optimize/backtesting.py | 73 +++++++++++++++++++------------ 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 02867d157..d567e1159 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -650,7 +650,7 @@ class Backtesting: def get_valid_price_and_stake( self, pair: str, row: Tuple, propose_rate: float, stake_amount: Optional[float], direction: LongShort, current_time: datetime, entry_tag: Optional[str], - trade: Optional[LocalTrade], order_type: str, readjust_req: Optional[bool] = False + trade: Optional[LocalTrade], order_type: str ) -> Tuple[float, float, float, float]: if order_type == 'limit': @@ -660,13 +660,6 @@ class Backtesting: proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate - if readjust_req: - propose_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price, - default_retval=propose_rate)( - pair=pair, current_time=current_time, - proposed_rate=propose_rate, entry_tag=entry_tag, - side=direction - ) # default value is open rate or custom rate from before # We can't place orders higher than current high (otherwise it'd be a stop limit buy) # which freqtrade does not support in live. @@ -675,7 +668,7 @@ class Backtesting: else: propose_rate = min(propose_rate, row[HIGH_IDX]) - pos_adjust = trade is not None and readjust_req is False + pos_adjust = trade is not None leverage = trade.leverage if trade else 1.0 if not pos_adjust: try: @@ -721,19 +714,24 @@ class Backtesting: def _enter_trade(self, pair: str, row: Tuple, direction: LongShort, stake_amount: Optional[float] = None, trade: Optional[LocalTrade] = None, - readjust_req: Optional[bool] = False) -> Optional[LocalTrade]: + requested_rate: Optional[float] = None, + requested_stake: Optional[float] = None) -> Optional[LocalTrade]: current_time = row[DATE_IDX].to_pydatetime() entry_tag = row[ENTER_TAG_IDX] if len(row) >= ENTER_TAG_IDX + 1 else None # let's call the custom entry price, using the open price as default price order_type = self.strategy.order_types['entry'] - pos_adjust = trade is not None and readjust_req is False + pos_adjust = trade is not None and requested_rate is None propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake( pair, row, row[OPEN_IDX], stake_amount, direction, current_time, entry_tag, trade, - order_type, readjust_req + order_type ) + # replace proposed rate if another rate was requested + propose_rate = requested_rate if requested_rate else propose_rate + stake_amount = requested_stake if requested_stake else stake_amount + if not stake_amount: # In case of pos adjust, still return the original trade # If not pos adjust, trade is None @@ -874,20 +872,36 @@ class Backtesting: self.protections.stop_per_pair(pair, current_time) self.protections.global_stop(current_time) - def check_order_replace(self, trade: LocalTrade, current_time, row: Tuple) -> None: + def check_order_replace(self, trade: LocalTrade, current_time, row: Tuple) -> bool: """ - Check if an entry order has to be replaced and do so. - Returns None. + Check if an entry order has to be replaced and do so. If user requested cancellation + and there are no filled orders in the trade will instruct caller to delete the trade. + Returns True if the trade should be deleted. """ for order in [o for o in trade.orders if o.ft_is_open]: + # only check on new candles for open entry orders if order.side == trade.entry_side and current_time > order.order_date_utc: - # cancel existing order - del trade.orders[trade.orders.index(order)] + requested_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price, + default_retval=order.price)( + trade=trade, order=order, pair=trade.pair, current_time=current_time, + proposed_rate=row[OPEN_IDX], current_order_rate=order.price, + entry_tag=trade.enter_tag, side=trade.trade_direction + ) # default value is current order price - # place new order - self._enter_trade(pair=trade.pair, row=row, trade=trade, - direction='short' if trade.is_short else 'long', - readjust_req=True) + # cancel existing order whenever a new rate is requested (or None) + if requested_rate != order.price: + del trade.orders[trade.orders.index(order)] + + # place new order if None was not returned + if requested_rate: + self._enter_trade(pair=trade.pair, row=row, trade=trade, + requested_rate=requested_rate, + requested_stake=(order.remaining * order.price), + direction='short' if trade.is_short else 'long') + else: + # assumption: there can't be multiple open entry orders at any given time + return (trade.nr_of_successful_entries == 0) + return False def check_order_cancel(self, trade: LocalTrade, current_time) -> bool: """ @@ -983,15 +997,16 @@ class Backtesting: for t in list(open_trades[pair]): # 1. Cancel expired entry/exit orders. - if self.check_order_cancel(t, current_time): - # Close trade due to entry timeout expiration. + order_cancel = self.check_order_cancel(t, current_time) + # 2. Replace/cancel (user requested) entry orders. + order_replace = self.check_order_replace(t, current_time, row) + if order_cancel or order_replace: + # Close trade due to entry timeout expiration or cancellation. open_trade_count -= 1 open_trades[pair].remove(t) self.wallets.update() - else: - self.check_order_replace(t, current_time, row) - # 2. Process entries. + # 3. Process entries. # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected # don't open on the last row @@ -1014,7 +1029,7 @@ class Backtesting: open_trades[pair].append(trade) for trade in list(open_trades[pair]): - # 3. Process entry orders. + # 4. Process entry orders. order = trade.select_order(trade.entry_side, is_open=True) if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time) @@ -1022,11 +1037,11 @@ class Backtesting: LocalTrade.add_bt_trade(trade) self.wallets.update() - # 4. Create exit orders (if any) + # 5. Create exit orders (if any) if not trade.open_order_id: self._get_exit_trade_entry(trade, row) # Place exit order if necessary - # 5. Process exit orders. + # 6. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): trade.open_order_id = None From 8c19953cdd3c1aab5143d0ef8e5c6d2a442cf382 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 1 May 2022 12:08:19 +0300 Subject: [PATCH 21/41] Quick exit when order should be maintained. --- freqtrade/optimize/backtesting.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4485d3da3..e42bfd2ea 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -890,7 +890,10 @@ class Backtesting: ) # default value is current order price # cancel existing order whenever a new rate is requested (or None) - if requested_rate != order.price: + if requested_rate == order.price: + # assumption: there can't be multiple open entry orders at any given time + return False + else: del trade.orders[trade.orders.index(order)] # place new order if None was not returned From 9d205132d0fb54df99a7f2b5d0a59520e07eab1b Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 1 May 2022 12:10:11 +0300 Subject: [PATCH 22/41] Revert unintended comment change. --- freqtrade/optimize/backtesting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e42bfd2ea..1ee96d015 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -661,8 +661,7 @@ class Backtesting: proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate - - # We can't place orders higher than current high (otherwise it'd be a stop limit buy) + # We can't place orders higher than current high (otherwise it'd be a stop limit entry) # which freqtrade does not support in live. if direction == "short": propose_rate = max(propose_rate, row[LOW_IDX]) From 4e43194dfeefb135e3a567659dba605bb501b10d Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sun, 1 May 2022 18:06:20 +0300 Subject: [PATCH 23/41] BT: Refactor open order management. --- freqtrade/optimize/backtesting.py | 121 ++++++++++++++++-------------- 1 file changed, 66 insertions(+), 55 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1ee96d015..d54c0e5a9 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -872,64 +872,78 @@ class Backtesting: self.protections.stop_per_pair(pair, current_time) self.protections.global_stop(current_time) - def check_order_replace(self, trade: LocalTrade, current_time, row: Tuple) -> bool: + def manage_open_orders(self, trade: LocalTrade, current_time, row: Tuple) -> bool: """ - Check if an entry order has to be replaced and do so. If user requested cancellation - and there are no filled orders in the trade will instruct caller to delete the trade. + Check if any open order needs to be cancelled or replaced. Returns True if the trade should be deleted. """ for order in [o for o in trade.orders if o.ft_is_open]: - # only check on new candles for open entry orders - if order.side == trade.entry_side and current_time > order.order_date_utc: - requested_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price, - default_retval=order.price)( - trade=trade, order=order, pair=trade.pair, current_time=current_time, - proposed_rate=row[OPEN_IDX], current_order_rate=order.price, - entry_tag=trade.enter_tag, side=trade.trade_direction - ) # default value is current order price - - # cancel existing order whenever a new rate is requested (or None) - if requested_rate == order.price: - # assumption: there can't be multiple open entry orders at any given time - return False - else: - del trade.orders[trade.orders.index(order)] - - # place new order if None was not returned - if requested_rate: - self._enter_trade(pair=trade.pair, row=row, trade=trade, - requested_rate=requested_rate, - requested_stake=(order.remaining * order.price), - direction='short' if trade.is_short else 'long') - else: - # assumption: there can't be multiple open entry orders at any given time - return (trade.nr_of_successful_entries == 0) + if self.check_order_cancel(trade, order, current_time): + # delete trade due to order timeout + return True + elif self.check_order_replace(trade, order, current_time, row): + # delete trade due to user request + return True + # default maintain trade return False - def check_order_cancel(self, trade: LocalTrade, current_time) -> bool: + def check_order_cancel(self, trade: LocalTrade, order: Order, current_time) -> bool: """ - Check if an order has been canceled. + Check if current analyzed order has to be canceled. Returns True if the trade should be Deleted (initial order was canceled). """ - for order in [o for o in trade.orders if o.ft_is_open]: - - timedout = self.strategy.ft_check_timed_out(trade, order, current_time) - if timedout: - if order.side == trade.entry_side: - self.timedout_entry_orders += 1 - if trade.nr_of_successful_entries == 0: - # Remove trade due to entry timeout expiration. - return True - else: - # Close additional entry order - del trade.orders[trade.orders.index(order)] - if order.side == trade.exit_side: - self.timedout_exit_orders += 1 - # Close exit order and retry exiting on next signal. + timedout = self.strategy.ft_check_timed_out(trade, order, current_time) + if timedout: + if order.side == trade.entry_side: + self.timedout_entry_orders += 1 + if trade.nr_of_successful_entries == 0: + # Remove trade due to entry timeout expiration. + return True + else: + # Close additional entry order del trade.orders[trade.orders.index(order)] + if order.side == trade.exit_side: + self.timedout_exit_orders += 1 + # Close exit order and retry exiting on next signal. + del trade.orders[trade.orders.index(order)] return False + def check_order_replace(self, trade: LocalTrade, order: Order, current_time, + row: Tuple) -> bool: + """ + Check if current analyzed entry order has to be replaced and do so. + If user requested cancellation and there are no filled orders in the trade will + instruct caller to delete the trade. + Returns True if the trade should be deleted. + """ + # only check on new candles for open entry orders + if order.side == trade.entry_side and current_time > order.order_date_utc: + requested_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price, + default_retval=order.price)( + trade=trade, order=order, pair=trade.pair, current_time=current_time, + proposed_rate=row[OPEN_IDX], current_order_rate=order.price, + entry_tag=trade.enter_tag, side=trade.trade_direction + ) # default value is current order price + + # cancel existing order whenever a new rate is requested (or None) + if requested_rate == order.price: + # assumption: there can't be multiple open entry orders at any given time + return False + else: + del trade.orders[trade.orders.index(order)] + + # place new order if None was not returned + if requested_rate: + self._enter_trade(pair=trade.pair, row=row, trade=trade, + requested_rate=requested_rate, + requested_stake=(order.remaining * order.price), + direction='short' if trade.is_short else 'long') + else: + # assumption: there can't be multiple open entry orders at any given time + return (trade.nr_of_successful_entries == 0) + return False + def validate_row( self, data: Dict, pair: str, row_index: int, current_time: datetime) -> Optional[Tuple]: try: @@ -999,17 +1013,14 @@ class Backtesting: self.dataprovider._set_dataframe_max_index(row_index) for t in list(open_trades[pair]): - # 1. Cancel expired entry/exit orders. - order_cancel = self.check_order_cancel(t, current_time) - # 2. Replace/cancel (user requested) entry orders. - order_replace = self.check_order_replace(t, current_time, row) - if order_cancel or order_replace: - # Close trade due to entry timeout expiration or cancellation. + # 1. Manage currently open orders of active trades + if self.manage_open_orders(t, current_time, row): + # Close trade open_trade_count -= 1 open_trades[pair].remove(t) self.wallets.update() - # 3. Process entries. + # 2. Process entries. # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected # don't open on the last row @@ -1032,7 +1043,7 @@ class Backtesting: open_trades[pair].append(trade) for trade in list(open_trades[pair]): - # 4. Process entry orders. + # 3. Process entry orders. order = trade.select_order(trade.entry_side, is_open=True) if order and self._get_order_filled(order.price, row): order.close_bt_order(current_time) @@ -1040,11 +1051,11 @@ class Backtesting: LocalTrade.add_bt_trade(trade) self.wallets.update() - # 5. Create exit orders (if any) + # 4. Create exit orders (if any) if not trade.open_order_id: self._get_exit_trade_entry(trade, row) # Place exit order if necessary - # 6. Process exit orders. + # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): trade.open_order_id = None From b83cd95a026b759814eba0ff98455d0b3ec86c0f Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 2 May 2022 18:07:48 +0300 Subject: [PATCH 24/41] Tests: add basic testcases for entry adjustment. --- tests/test_freqtradebot.py | 102 +++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e3785e67e..e58619bc0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2450,6 +2450,7 @@ def test_manage_open_orders_entry( Trade.query.session.add(open_trade) freqtrade.strategy.check_entry_timeout = MagicMock(return_value=False) + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) # check it does cancel buy orders over the time limit freqtrade.manage_open_orders() assert cancel_order_mock.call_count == 1 @@ -2459,6 +2460,107 @@ def test_manage_open_orders_entry( assert nb_trades == 0 # Custom user buy-timeout is never called assert freqtrade.strategy.check_entry_timeout.call_count == 0 + # Entry adjustment is never called + assert freqtrade.strategy.adjust_entry_price.call_count == 0 + + +@pytest.mark.parametrize("is_short", [False, True]) +def test_adjust_entry_cancel( + default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, + limit_sell_order_old, fee, mocker, caplog, is_short +) -> None: + old_order = limit_sell_order_old if is_short else limit_buy_order_old + old_order['id'] = open_trade.open_order_id + limit_buy_cancel = deepcopy(old_order) + limit_buy_cancel['status'] = 'canceled' + cancel_order_mock = MagicMock(return_value=limit_buy_cancel) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker_usdt, + fetch_order=MagicMock(return_value=old_order), + cancel_order_with_result=cancel_order_mock, + get_fee=fee + ) + freqtrade = FreqtradeBot(default_conf_usdt) + + open_trade.is_short = is_short + Trade.query.session.add(open_trade) + + # Timeout to not interfere + freqtrade.strategy.ft_check_timed_out = MagicMock(return_value=False) + + # check that order is cancelled + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=None) + freqtrade.manage_open_orders() + trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 0 + nb_all_orders = len(Order.query.all()) + assert nb_all_orders == 0 + assert log_has_re( + f"{'Sell' if is_short else 'Buy'} order user requested order cancel*", caplog) + assert log_has_re( + f"{'Sell' if is_short else 'Buy'} order fully cancelled.*", caplog) + + # Entry adjustment is called + assert freqtrade.strategy.adjust_entry_price.call_count == 1 + + +@pytest.mark.parametrize("is_short", [False, True]) +def test_adjust_entry_maintain_replace( + default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, + limit_sell_order_old, fee, mocker, caplog, is_short +) -> None: + old_order = limit_sell_order_old if is_short else limit_buy_order_old + old_order['id'] = open_trade.open_order_id + limit_buy_cancel = deepcopy(old_order) + limit_buy_cancel['status'] = 'canceled' + cancel_order_mock = MagicMock(return_value=limit_buy_cancel) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker_usdt, + fetch_order=MagicMock(return_value=old_order), + cancel_order_with_result=cancel_order_mock, + get_fee=fee + ) + freqtrade = FreqtradeBot(default_conf_usdt) + + open_trade.is_short = is_short + Trade.query.session.add(open_trade) + + # Timeout to not interfere + freqtrade.strategy.ft_check_timed_out = MagicMock(return_value=False) + + # Check that order is maintained + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=old_order['price']) + freqtrade.manage_open_orders() + trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + nb_orders = len(Order.get_open_orders()) + assert nb_orders == 1 + # Entry adjustment is called + assert freqtrade.strategy.adjust_entry_price.call_count == 1 + + # Check that order is replaced + freqtrade.get_valid_enter_price_and_stake = MagicMock(return_value={100, 10, 1}) + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) + freqtrade.manage_open_orders() + trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + nb_all_orders = len(Order.query.all()) + freqtrade.logger.warning(Order.query.all()) + assert nb_all_orders == 2 + # New order seems to be in closed status? + # nb_open_orders = len(Order.get_open_orders()) + # assert nb_open_orders == 1 + assert log_has_re( + f"{'Sell' if is_short else 'Buy'} order cancelled to be replaced*", caplog) + # Entry adjustment is called + assert freqtrade.strategy.adjust_entry_price.call_count == 1 @pytest.mark.parametrize("is_short", [False, True]) From 59397cdd19ea23171e91145b70b0fcfb7b5efc0c Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 2 May 2022 18:09:28 +0300 Subject: [PATCH 25/41] Freqtradebot: Fix full cancel logging location. --- freqtrade/freqtradebot.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9258b7ca2..c0ccf2688 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1123,6 +1123,7 @@ class FreqtradeBot(LoggingMixin): Timeout setting takes priority over limit order adjustment request. :return: None """ + logger.warning(Order.query.all()) for trade in Trade.get_open_order_trades(): try: if not trade.open_order_id: @@ -1150,6 +1151,7 @@ class FreqtradeBot(LoggingMixin): :param trade: Trade object. :return: None """ + logger.warning("handle_timedout_order") if order['side'] == trade.entry_side: self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) else: @@ -1179,6 +1181,8 @@ class FreqtradeBot(LoggingMixin): :param trade: Trade object. :return: None """ + logger.warning("replace_order") + logger.warning(f"Order: {order}, Trade:{trade}") analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, self.strategy.timeframe) latest_candle_open_date = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None @@ -1195,6 +1199,7 @@ class FreqtradeBot(LoggingMixin): current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, current_order_rate=order_obj.price, entry_tag=trade.enter_tag, side=trade.entry_side) + logger.warning(f"adjusted_entry_price: {adjusted_entry_price}") full_cancel = False cancel_reason = constants.CANCEL_REASON['REPLACE'] @@ -1276,10 +1281,10 @@ class FreqtradeBot(LoggingMixin): # Using filled to determine the filled amount filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled') if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): - logger.info(f'{side} order fully cancelled. Removing {trade} from database.') # if trade is not partially completed and it's the only order, just delete the trade open_order_count = len([order for order in trade.orders if order.status == 'open']) if open_order_count <= 1 and allow_full_cancel: + logger.info(f'{side} order fully cancelled. Removing {trade} from database.') trade.delete() was_trade_fully_canceled = True reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" From 4c7460107381dffba06004e0cba24e1df5dad00f Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Mon, 2 May 2022 18:22:41 +0300 Subject: [PATCH 26/41] Freqtradebot: Cleanup stray debug messages. --- freqtrade/freqtradebot.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c0ccf2688..c3ddccdd8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1123,7 +1123,6 @@ class FreqtradeBot(LoggingMixin): Timeout setting takes priority over limit order adjustment request. :return: None """ - logger.warning(Order.query.all()) for trade in Trade.get_open_order_trades(): try: if not trade.open_order_id: @@ -1151,7 +1150,6 @@ class FreqtradeBot(LoggingMixin): :param trade: Trade object. :return: None """ - logger.warning("handle_timedout_order") if order['side'] == trade.entry_side: self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) else: @@ -1181,8 +1179,6 @@ class FreqtradeBot(LoggingMixin): :param trade: Trade object. :return: None """ - logger.warning("replace_order") - logger.warning(f"Order: {order}, Trade:{trade}") analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, self.strategy.timeframe) latest_candle_open_date = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None @@ -1199,7 +1195,6 @@ class FreqtradeBot(LoggingMixin): current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, current_order_rate=order_obj.price, entry_tag=trade.enter_tag, side=trade.entry_side) - logger.warning(f"adjusted_entry_price: {adjusted_entry_price}") full_cancel = False cancel_reason = constants.CANCEL_REASON['REPLACE'] From 5c82cce06c8c9ccaefae297001b0405f7bcaed47 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 May 2022 06:40:12 +0200 Subject: [PATCH 27/41] Fix new test failures --- tests/test_freqtradebot.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e58619bc0..f6d03db6b 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2469,12 +2469,12 @@ def test_adjust_entry_cancel( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, limit_sell_order_old, fee, mocker, caplog, is_short ) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) old_order = limit_sell_order_old if is_short else limit_buy_order_old old_order['id'] = open_trade.open_order_id limit_buy_cancel = deepcopy(old_order) limit_buy_cancel['status'] = 'canceled' cancel_order_mock = MagicMock(return_value=limit_buy_cancel) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, @@ -2482,7 +2482,6 @@ def test_adjust_entry_cancel( cancel_order_with_result=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf_usdt) open_trade.is_short = is_short Trade.query.session.add(open_trade) @@ -2512,12 +2511,12 @@ def test_adjust_entry_maintain_replace( default_conf_usdt, ticker_usdt, limit_buy_order_old, open_trade, limit_sell_order_old, fee, mocker, caplog, is_short ) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) old_order = limit_sell_order_old if is_short else limit_buy_order_old old_order['id'] = open_trade.open_order_id limit_buy_cancel = deepcopy(old_order) limit_buy_cancel['status'] = 'canceled' cancel_order_mock = MagicMock(return_value=limit_buy_cancel) - patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker_usdt, @@ -2525,7 +2524,6 @@ def test_adjust_entry_maintain_replace( cancel_order_with_result=cancel_order_mock, get_fee=fee ) - freqtrade = FreqtradeBot(default_conf_usdt) open_trade.is_short = is_short Trade.query.session.add(open_trade) From b2f33944eccbece5dd13600a380c72ab1de4255b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 4 May 2022 07:13:02 +0200 Subject: [PATCH 28/41] Add preliminary backtesting test --- tests/optimize/__init__.py | 2 ++ tests/optimize/test_backtest_detail.py | 24 ++++++++++++++++++++++++ tests/test_freqtradebot.py | 19 ++++++------------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index fc4125a42..a3dd59004 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -40,6 +40,8 @@ class BTContainer(NamedTuple): custom_entry_price: Optional[float] = None custom_exit_price: Optional[float] = None leverage: float = 1.0 + timeout: Optional[int] = None + adjust_entry_price: Optional[float] = None def _get_frame_time_from_offset(offset): diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index ea13de4c8..f2e2c89ad 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -754,6 +754,21 @@ tc47 = BTContainer(data=[ trades=[] ) +# Test 48: Custom-entry-price below all candles - readjust order +tc48 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # timeout + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.1, + timeout=1000, + custom_entry_price=4200, + adjust_entry_price=5200, + trades=[BTrade(exit_reason=ExitType.ROI, open_tick=1, close_tick=2, is_short=False)] +) + TESTS = [ tc0, @@ -804,6 +819,7 @@ TESTS = [ tc45, tc46, tc47, + tc48, ] @@ -817,6 +833,11 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) default_conf["timeframe"] = tests_timeframe default_conf["trailing_stop"] = data.trailing_stop default_conf["trailing_only_offset_is_reached"] = data.trailing_only_offset_is_reached + if data.timeout: + default_conf['unfilledtimeout'].update({ + 'entry': data.timeout, + 'exit': data.timeout, + }) # Only add this to configuration If it's necessary if data.trailing_stop_positive is not None: default_conf["trailing_stop_positive"] = data.trailing_stop_positive @@ -840,6 +861,9 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) backtesting.strategy.custom_entry_price = MagicMock(return_value=data.custom_entry_price) if data.custom_exit_price: backtesting.strategy.custom_exit_price = MagicMock(return_value=data.custom_exit_price) + if data.adjust_entry_price: + backtesting.strategy.adjust_entry_price = MagicMock(return_value=data.adjust_entry_price) + backtesting.strategy.use_custom_stoploss = data.use_custom_stoploss backtesting.strategy.leverage = lambda **kwargs: data.leverage caplog.set_level(logging.DEBUG) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index f6d03db6b..e19d5f36a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2493,10 +2493,8 @@ def test_adjust_entry_cancel( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=None) freqtrade.manage_open_orders() trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() - nb_trades = len(trades) - assert nb_trades == 0 - nb_all_orders = len(Order.query.all()) - assert nb_all_orders == 0 + assert len(trades) == 0 + assert len(Order.query.all()) == 0 assert log_has_re( f"{'Sell' if is_short else 'Buy'} order user requested order cancel*", caplog) assert log_has_re( @@ -2535,10 +2533,8 @@ def test_adjust_entry_maintain_replace( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=old_order['price']) freqtrade.manage_open_orders() trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() - nb_trades = len(trades) - assert nb_trades == 1 - nb_orders = len(Order.get_open_orders()) - assert nb_orders == 1 + assert len(trades) == 1 + assert len(Order.get_open_orders()) == 1 # Entry adjustment is called assert freqtrade.strategy.adjust_entry_price.call_count == 1 @@ -2547,10 +2543,8 @@ def test_adjust_entry_maintain_replace( freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1234) freqtrade.manage_open_orders() trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() - nb_trades = len(trades) - assert nb_trades == 1 + assert len(trades) == 1 nb_all_orders = len(Order.query.all()) - freqtrade.logger.warning(Order.query.all()) assert nb_all_orders == 2 # New order seems to be in closed status? # nb_open_orders = len(Order.get_open_orders()) @@ -2588,8 +2582,7 @@ def test_check_handle_cancelled_buy( assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() - nb_trades = len(trades) - assert nb_trades == 0 + assert len(trades) == 0 assert log_has_re( f"{'Sell' if is_short else 'Buy'} order cancelled on exchange for Trade.*", caplog) From dbecc097dffdb54b8b78c10ce52c99283c00d4b0 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Wed, 4 May 2022 21:34:45 +0300 Subject: [PATCH 29/41] Models:Trade: Update trade open_rate based on lastest order. --- freqtrade/persistence/models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 299032bb4..4ed651e20 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -866,9 +866,17 @@ class LocalTrade(): return float(f"{profit_ratio:.8f}") def recalc_trade_from_orders(self): + filled_orders_count = len(self.select_filled_orders(self.entry_side)) + latest_order_in_trade = self.select_order(self.entry_side, True) + # No fills but newer order + if (filled_orders_count == 0 and latest_order_in_trade is not None and + latest_order_in_trade.id is not None): + # after ensuring there is a populated order + if latest_order_in_trade.id > 1: + self.open_rate = latest_order_in_trade.price # We need at least 2 entry orders for averaging amounts and rates. # TODO: this condition could probably be removed - if len(self.select_filled_orders(self.entry_side)) < 2: + if filled_orders_count < 2: self.stake_amount = self.amount * self.open_rate / self.leverage # Just in case, still recalc open trade value From ae01afdd0f3bd9f63b3b39d8bb22ae8a3a38830b Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Wed, 4 May 2022 22:05:53 +0300 Subject: [PATCH 30/41] Models:Trade: Fix open_rate updates. --- freqtrade/persistence/models.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 4ed651e20..62f3d7d55 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -868,12 +868,11 @@ class LocalTrade(): def recalc_trade_from_orders(self): filled_orders_count = len(self.select_filled_orders(self.entry_side)) latest_order_in_trade = self.select_order(self.entry_side, True) - # No fills but newer order + # No fills - update open_rate in case order was replaced if (filled_orders_count == 0 and latest_order_in_trade is not None and - latest_order_in_trade.id is not None): - # after ensuring there is a populated order - if latest_order_in_trade.id > 1: - self.open_rate = latest_order_in_trade.price + latest_order_in_trade.price is not None): + # after ensuring there is a populated order price + self.open_rate = latest_order_in_trade.price # We need at least 2 entry orders for averaging amounts and rates. # TODO: this condition could probably be removed if filled_orders_count < 2: From 25c74e26d1a59699e8ae5e54f94d5fe0cb47a3d4 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Thu, 5 May 2022 12:18:19 +0300 Subject: [PATCH 31/41] Models:Trade: Revert trade open_rate update. --- freqtrade/persistence/models.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 62f3d7d55..d1846f9ef 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -866,16 +866,10 @@ class LocalTrade(): return float(f"{profit_ratio:.8f}") def recalc_trade_from_orders(self): - filled_orders_count = len(self.select_filled_orders(self.entry_side)) - latest_order_in_trade = self.select_order(self.entry_side, True) - # No fills - update open_rate in case order was replaced - if (filled_orders_count == 0 and latest_order_in_trade is not None and - latest_order_in_trade.price is not None): - # after ensuring there is a populated order price - self.open_rate = latest_order_in_trade.price + # We need at least 2 entry orders for averaging amounts and rates. # TODO: this condition could probably be removed - if filled_orders_count < 2: + if len(self.select_filled_orders(self.entry_side)) < 2: self.stake_amount = self.amount * self.open_rate / self.leverage # Just in case, still recalc open trade value From 2bed0eab0cc5433800ac8de8b68bb0ad2d7ca1f3 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Thu, 5 May 2022 12:19:05 +0300 Subject: [PATCH 32/41] BT: Update trade open_rate on first filled order. --- freqtrade/optimize/backtesting.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 376236747..35761c54c 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -813,6 +813,11 @@ class Backtesting: cost=stake_amount + trade.fee_open, ) if pos_adjust and self._get_order_filled(order.price, row): + # Update trade open_rate on first filled order + # this is for cases where adjust_entry_order might have replaced the + # initial order from trade opening + if len(trade.select_filled_orders(trade.entry_side)) == 1: + trade.open_rate = order.price order.close_bt_order(current_time) else: trade.open_order_id = str(self.order_id_counter) From 29f1edbde70a77c5d8bb1f367464e514cc04da37 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Thu, 5 May 2022 12:24:32 +0300 Subject: [PATCH 33/41] Cleanup. Remove stray new line. --- freqtrade/persistence/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index d1846f9ef..299032bb4 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -866,7 +866,6 @@ class LocalTrade(): return float(f"{profit_ratio:.8f}") def recalc_trade_from_orders(self): - # We need at least 2 entry orders for averaging amounts and rates. # TODO: this condition could probably be removed if len(self.select_filled_orders(self.entry_side)) < 2: From 2d9be6dacee665803b704d7dffd9814396930e4b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 5 May 2022 19:50:16 +0200 Subject: [PATCH 34/41] move open_rate updating to close_bt_order --- freqtrade/optimize/backtesting.py | 6 ++---- freqtrade/persistence/models.py | 6 +++++- tests/optimize/test_backtest_detail.py | 26 ++++++++++++++++++++------ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 35761c54c..aadda6dbd 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -816,9 +816,7 @@ class Backtesting: # Update trade open_rate on first filled order # this is for cases where adjust_entry_order might have replaced the # initial order from trade opening - if len(trade.select_filled_orders(trade.entry_side)) == 1: - trade.open_rate = order.price - order.close_bt_order(current_time) + order.close_bt_order(current_time, trade) else: trade.open_order_id = str(self.order_id_counter) trade.orders.append(order) @@ -1052,7 +1050,7 @@ class Backtesting: # 3. Process entry orders. order = trade.select_order(trade.entry_side, is_open=True) if order and self._get_order_filled(order.price, row): - order.close_bt_order(current_time) + order.close_bt_order(current_time, trade) trade.open_order_id = None LocalTrade.add_bt_trade(trade) self.wallets.update() diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 299032bb4..c5ea34a30 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -219,11 +219,15 @@ class Order(_DECL_BASE): 'remaining': self.remaining, } - def close_bt_order(self, close_date: datetime): + def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): self.order_filled_date = close_date self.filled = self.amount self.status = 'closed' self.ft_is_open = False + if (self.ft_order_side == trade.entry_side + and len(trade.select_filled_orders(trade.entry_side)) == 1): + trade.open_rate = self.price + trade.recalc_open_trade_value() @staticmethod def update_orders(orders: List['Order'], order: Dict[str, Any]): diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index f2e2c89ad..aab864431 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -760,16 +760,29 @@ tc48 = BTContainer(data=[ [0, 5000, 5050, 4950, 5000, 6172, 1, 0], [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # timeout [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust - [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 1], [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], - stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.1, - timeout=1000, - custom_entry_price=4200, - adjust_entry_price=5200, - trades=[BTrade(exit_reason=ExitType.ROI, open_tick=1, close_tick=2, is_short=False)] + stop_loss=-0.01, roi={"0": 0.10}, profit_perc=-0.087, + use_exit_signal=True, timeout=1000, + custom_entry_price=4200, adjust_entry_price=5200, + trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=False)] ) +# Test 49: Custom-entry-price short above all candles - readjust order +tc49 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 0, 0, 1, 0], + [1, 5000, 5200, 4951, 5000, 6172, 0, 0, 0, 0], # timeout + [2, 4900, 5250, 4900, 5100, 6172, 0, 0, 0, 0], # Order readjust + [3, 5100, 5100, 4650, 4750, 6172, 0, 0, 0, 1], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0, 0, 0]], + stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.05, + use_exit_signal=True, timeout=1000, + custom_entry_price=5300, adjust_entry_price=5000, + trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=True)] +) + TESTS = [ tc0, tc1, @@ -820,6 +833,7 @@ TESTS = [ tc46, tc47, tc48, + tc49, ] From d11c44940eebc4ed413fd858b4bdb9096f2730af Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 6 May 2022 06:23:06 +0200 Subject: [PATCH 35/41] Slightly reword docs remove some Note-boxes - people tend to skip these. --- docs/bot-basics.md | 2 +- docs/strategy-callbacks.md | 17 ++++++++++------- freqtrade/optimize/backtesting.py | 3 --- freqtrade/strategy/interface.py | 2 +- .../subtemplates/strategy_methods_advanced.j2 | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index abc0e7b16..9fdbdc8a8 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -24,7 +24,7 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and * Fetch open trades from persistence. * Calculate current list of tradable pairs. -* Download OHLCV data for the pairlist including all [informative pairs](strategy-customization.md#get-data-for-non-tradeable-pairs) +* Download OHLCV data for the pairlist including all [informative pairs](strategy-customization.md#get-data-for-non-tradeable-pairs) This step is only executed once per Candle to avoid unnecessary network traffic. * Call `bot_loop_start()` strategy callback. * Analyze strategy per pair. diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 462cf604f..a58878ee7 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -719,14 +719,17 @@ class DigDeeperStrategy(IStrategy): The `adjust_entry_price()` callback may be used by strategy developer to refresh/replace limit orders upon arrival of new candles. Be aware that `custom_entry_price()` is still the one dictating initial entry limit order price target at the time of entry trigger. -!!! Note "Simple Order Cancelation" - This also allows simple cancelation without an replacement order. This behavior occurs when `None` is returned. +Orders can ba cancelled out of this callback by returning `None`. -!!! Note "Maintaining Order" - Maintaining existing order on exchange is facilitated. This behavior occurs when `order.price` is returned. +Returning `current_order_rate` will keep the order on the exchange "as is". +Returning any other price will cancel the existing order, and replace it with a new order. -!!! Warning - Entry `unfilledtimeout` mechanism takes precedence over this. Be sure to update timeout values to match your expectancy. +The trade open-date (`trade.open_date_utc`) will remain at the time of the very first order placed. +Please makes sure to be aware of this - and eventually adjust your logic in other callbacks to account for this, and use the date of the first filled order instead. + +!!! Warning "Regular timeout" + Entry `unfilledtimeout` mechanism (as well as `check_entry_timeout()`) takes precedence over this. + Entry Orders that are cancelled via the above methods will not have this callback called. Be sure to update timeout values to match your expectations. ```python from freqtrade.persistence import Trade @@ -741,7 +744,7 @@ class AwesomeStrategy(IStrategy): entry_tag: Optional[str], side: str, **kwargs) -> float: """ Entry price re-adjustment logic, returning the user desired limit price. - This only executes when a order was already placed, still open(unfilled fully or partially) + This only executes when a order was already placed, still open (unfilled fully or partially) and not timed out on subsequent candles after entry trigger. When not implemented by a strategy, returns current_order_rate as default. diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index aadda6dbd..86dcb1094 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -813,9 +813,6 @@ class Backtesting: cost=stake_amount + trade.fee_open, ) if pos_adjust and self._get_order_filled(order.price, row): - # Update trade open_rate on first filled order - # this is for cases where adjust_entry_order might have replaced the - # initial order from trade opening order.close_bt_order(current_time, trade) else: trade.open_order_id = str(self.order_id_counter) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 367097d71..26efd74a9 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -476,7 +476,7 @@ class IStrategy(ABC, HyperStrategyMixin): entry_tag: Optional[str], side: str, **kwargs) -> float: """ Entry price re-adjustment logic, returning the user desired limit price. - This only executes when a order was already placed, still open(unfilled fully or partially) + This only executes when a order was already placed, still open (unfilled fully or partially) and not timed out on subsequent candles after entry trigger. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 7f9671bb1..014e97cc0 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -35,7 +35,7 @@ def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, entry_tag: Optional[str], side: str, **kwargs) -> float: """ Entry price re-adjustment logic, returning the user desired limit price. - This only executes when a order was already placed, still open(unfilled fully or partially) + This only executes when a order was already placed, still open (unfilled fully or partially) and not timed out on subsequent candles after entry trigger. For full documentation please go to https://www.freqtrade.io/en/latest/strategy-callbacks/ From 5b3eaa3003ce70dfeb3d23b47ef91605742462fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 6 May 2022 06:30:35 +0200 Subject: [PATCH 36/41] Ensure advanced strategy template is runnable --- freqtrade/templates/base_strategy.py.j2 | 4 +++- .../subtemplates/strategy_methods_advanced.j2 | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 53237f67d..9e7e1fe50 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -4,7 +4,9 @@ # --- Do not remove these libs --- import numpy as np # noqa import pandas as pd # noqa -from pandas import DataFrame +from pandas import DataFrame # noqa +from datetime import datetime # noqa +from typing import Optional # noqa from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, IStrategy, IntParameter) diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 014e97cc0..317602da9 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -13,7 +13,7 @@ def bot_loop_start(self, **kwargs) -> None: pass def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: float, - entry_tag: 'Optional[str]', **kwargs) -> float: + entry_tag: Optional[str], **kwargs) -> float: """ Custom entry price logic, returning the new entry price. @@ -30,7 +30,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate: """ return proposed_rate -def adjust_entry_price(self, trade: Trade, order: Optional[Order], pair: str, +def adjust_entry_price(self, trade: 'Trade', order: 'Optional[Order]', pair: str, current_time: datetime, proposed_rate: float, current_order_rate: float, entry_tag: Optional[str], side: str, **kwargs) -> float: """ @@ -81,7 +81,7 @@ def custom_exit_price(self, pair: str, trade: 'Trade', def custom_stake_amount(self, pair: str, current_time: 'datetime', current_rate: float, proposed_stake: float, min_stake: float, max_stake: float, - side: str, entry_tag: 'Optional[str]', **kwargs) -> float: + side: str, entry_tag: Optional[str], **kwargs) -> float: """ Customize stake size for each new trade. @@ -146,7 +146,7 @@ def custom_exit(self, pair: str, trade: 'Trade', current_time: 'datetime', curre return None def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, current_time: datetime, entry_tag: 'Optional[str]', + time_in_force: str, current_time: datetime, entry_tag: Optional[str], side: str, **kwargs) -> bool: """ Called right before placing a entry order. @@ -245,7 +245,7 @@ def check_exit_timeout(self, pair: str, trade: 'Trade', order: 'Order', def adjust_trade_position(self, trade: 'Trade', current_time: 'datetime', current_rate: float, current_profit: float, min_stake: float, - max_stake: float, **kwargs) -> 'Optional[float]': + max_stake: float, **kwargs) -> Optional[float]: """ Custom trade adjustment logic, returning the stake amount that a trade should be increased. This means extra buy orders with additional fees. From 182a6f475d6c0a05fd475d7b40f83afe7e7c733b Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Fri, 6 May 2022 10:13:29 +0300 Subject: [PATCH 37/41] Minor typos. --- docs/strategy-callbacks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index a58878ee7..750d5fbd0 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -719,13 +719,13 @@ class DigDeeperStrategy(IStrategy): The `adjust_entry_price()` callback may be used by strategy developer to refresh/replace limit orders upon arrival of new candles. Be aware that `custom_entry_price()` is still the one dictating initial entry limit order price target at the time of entry trigger. -Orders can ba cancelled out of this callback by returning `None`. +Orders can be cancelled out of this callback by returning `None`. Returning `current_order_rate` will keep the order on the exchange "as is". Returning any other price will cancel the existing order, and replace it with a new order. The trade open-date (`trade.open_date_utc`) will remain at the time of the very first order placed. -Please makes sure to be aware of this - and eventually adjust your logic in other callbacks to account for this, and use the date of the first filled order instead. +Please make sure to be aware of this - and eventually adjust your logic in other callbacks to account for this, and use the date of the first filled order instead. !!! Warning "Regular timeout" Entry `unfilledtimeout` mechanism (as well as `check_entry_timeout()`) takes precedence over this. From 70bac41d89a25f1cccffe730340cee5df5126e4a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 6 May 2022 06:45:22 +0200 Subject: [PATCH 38/41] Add more backtest test scenarios --- tests/optimize/test_backtest_detail.py | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index aab864431..c98330e6c 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -783,6 +783,34 @@ tc49 = BTContainer(data=[ trades=[BTrade(exit_reason=ExitType.EXIT_SIGNAL, open_tick=1, close_tick=4, is_short=True)] ) +# Test 50: Custom-entry-price below all candles - readjust order cancels order +tc50 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # timeout + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust - cancel order + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.0, + use_exit_signal=True, timeout=1000, + custom_entry_price=4200, adjust_entry_price=None, + trades=[] +) + +# Test 51: Custom-entry-price below all candles - readjust order leaves order in place and timeout. +tc51 = BTContainer(data=[ + # D O H L C V EL XL ES Xs BT + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # timeout + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust - cancel order + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.0, + use_exit_signal=True, timeout=1000, + custom_entry_price=4200, adjust_entry_price=4200, + trades=[] +) + TESTS = [ tc0, tc1, @@ -834,6 +862,8 @@ TESTS = [ tc47, tc48, tc49, + tc50, + tc51, ] From 108903f7f0c968f88a3b2520a8cc8e7753c4c2e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 6 May 2022 19:49:39 +0200 Subject: [PATCH 39/41] Add DCA order adjust test --- tests/test_integration.py | 92 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 8f56c1fea..020f77fed 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -351,3 +351,95 @@ def test_dca_short(default_conf_usdt, ticker_usdt, fee, mocker) -> None: assert trade.nr_of_successful_entries == 2 assert trade.nr_of_successful_exits == 1 + + +def test_dca_order_adjust(default_conf_usdt, ticker_usdt, fee, mocker) -> None: + default_conf_usdt['position_adjustment_enable'] = True + + freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker_usdt, + get_fee=fee, + amount_to_precision=lambda s, x, y: y, + price_to_precision=lambda s, x, y: y, + ) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=False) + + patch_get_signal(freqtrade) + freqtrade.strategy.custom_entry_price = lambda **kwargs: ticker_usdt['ask'] * 0.96 + + freqtrade.enter_positions() + + assert len(Trade.get_trades().all()) == 1 + trade = Trade.get_trades().first() + assert len(trade.orders) == 1 + assert trade.open_order_id is not None + assert pytest.approx(trade.stake_amount) == 60 + assert trade.open_rate == 1.96 + # No adjustment + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 1 + assert trade.open_order_id is not None + assert pytest.approx(trade.stake_amount) == 60 + + # Cancel order and place new one + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1.99) + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 2 + assert trade.open_order_id is not None + # Open rate is not adjusted yet + assert trade.open_rate == 1.96 + + # Fill order + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 2 + assert trade.open_order_id is None + # Open rate is not adjusted yet + assert trade.open_rate == 1.99 + + # 2nd order - not filling + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=120) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=False) + + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 3 + assert trade.open_order_id is not None + assert trade.open_rate == 1.99 + assert trade.orders[-1].price == 1.96 + assert trade.orders[-1].cost == 120 + + # Replace new order with diff. order at a lower price + freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1.95) + + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 4 + assert trade.open_order_id is not None + assert trade.open_rate == 1.99 + assert trade.orders[-1].price == 1.95 + assert pytest.approx(trade.orders[-1].cost) == 120 + + # Fill DCA order + freqtrade.strategy.adjust_trade_position = MagicMock(return_value=None) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) + freqtrade.strategy.adjust_entry_price = MagicMock(side_effect=ValueError) + + freqtrade.process() + trade = Trade.get_trades().first() + assert len(trade.orders) == 4 + assert trade.open_order_id is None + assert pytest.approx(trade.open_rate) == 1.963153456 + assert trade.orders[-1].price == 1.95 + assert pytest.approx(trade.orders[-1].cost) == 120 + assert trade.orders[-1].status == 'closed' + + assert pytest.approx(trade.amount) == 91.689215 + # Check the 2 filled orders equal the above amount + assert pytest.approx(trade.orders[1].amount) == 30.150753768 + assert pytest.approx(trade.orders[-1].amount) == 61.538461232 From eca8d16c61d70e896f2f9a30049314b8df849419 Mon Sep 17 00:00:00 2001 From: eSeR1805 Date: Sat, 7 May 2022 17:31:56 +0300 Subject: [PATCH 40/41] Minor fix and enhancement for TC51. --- tests/optimize/test_backtest_detail.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index c98330e6c..18b4c3621 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -786,9 +786,9 @@ tc49 = BTContainer(data=[ # Test 50: Custom-entry-price below all candles - readjust order cancels order tc50 = BTContainer(data=[ # D O H L C V EL XL ES Xs BT - [0, 5000, 5050, 4950, 5000, 6172, 1, 0], - [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # timeout - [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust - cancel order + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], # Enter long - place order + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # Order readjust - cancel order + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], [3, 5100, 5100, 4650, 4750, 6172, 0, 0], [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.0, @@ -800,14 +800,14 @@ tc50 = BTContainer(data=[ # Test 51: Custom-entry-price below all candles - readjust order leaves order in place and timeout. tc51 = BTContainer(data=[ # D O H L C V EL XL ES Xs BT - [0, 5000, 5050, 4950, 5000, 6172, 1, 0], - [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # timeout - [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust - cancel order - [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], # Enter long - place order + [1, 5000, 5500, 4951, 5000, 6172, 0, 0], # Order readjust - replace order + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Order readjust - maintain order + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], # Timeout [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], stop_loss=-0.01, roi={"0": 0.10}, profit_perc=0.0, - use_exit_signal=True, timeout=1000, - custom_entry_price=4200, adjust_entry_price=4200, + use_exit_signal=True, timeout=60, + custom_entry_price=4200, adjust_entry_price=4100, trades=[] ) @@ -905,8 +905,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) backtesting.strategy.custom_entry_price = MagicMock(return_value=data.custom_entry_price) if data.custom_exit_price: backtesting.strategy.custom_exit_price = MagicMock(return_value=data.custom_exit_price) - if data.adjust_entry_price: - backtesting.strategy.adjust_entry_price = MagicMock(return_value=data.adjust_entry_price) + backtesting.strategy.adjust_entry_price = MagicMock(return_value=data.adjust_entry_price) backtesting.strategy.use_custom_stoploss = data.use_custom_stoploss backtesting.strategy.leverage = lambda **kwargs: data.leverage From 277e07589e5be31640411ef1a38fcb598f5cbd14 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 7 May 2022 17:47:37 +0200 Subject: [PATCH 41/41] update/fix some comments and docs --- docs/bot-basics.md | 2 +- docs/strategy-callbacks.md | 2 +- freqtrade/optimize/backtesting.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 9fdbdc8a8..1acbca565 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -34,7 +34,6 @@ By default, loop runs every few seconds (`internals.process_throttle_secs`) and * Check timeouts for open orders. * Calls `check_entry_timeout()` strategy callback for open entry orders. * Calls `check_exit_timeout()` strategy callback for open exit orders. -* Check readjustment request for open orders. * Calls `adjust_entry_price()` strategy callback for open entry orders. * Verifies existing positions and eventually places exit orders. * Considers stoploss, ROI and exit-signal, `custom_exit()` and `custom_stoploss()`. @@ -60,6 +59,7 @@ This loop will be repeated again and again until the bot is stopped. * Calculate entry / exit signals (calls `populate_entry_trend()` and `populate_exit_trend()` once per pair). * Loops per candle simulating entry and exit points. * Check for Order timeouts, either via the `unfilledtimeout` configuration, or via `check_entry_timeout()` / `check_exit_timeout()` strategy callbacks. + * Calls `adjust_entry_price()` strategy callback for open entry orders. * Check for trade entry signals (`enter_long` / `enter_short` columns). * Confirm trade entry / exits (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy). * Call `custom_entry_price()` (if implemented in the strategy) to determine entry price (Prices are moved to be within the opening candle). diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 750d5fbd0..ab67a3c26 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -765,7 +765,7 @@ class AwesomeStrategy(IStrategy): """ # Limit orders to use and follow SMA200 as price target for the first 10 minutes since entry trigger for BTC/USDT pair. if pair == 'BTC/USDT' and entry_tag == 'long_sma200' and side == 'long' and (current_time - timedelta(minutes=10) > trade.open_date_utc: - # just cancel the order if it has been filled more than half of the ammount + # just cancel the order if it has been filled more than half of the amount if order.filled > order.remaining: return None else: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 86dcb1094..45300b744 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -934,7 +934,7 @@ class Backtesting: else: del trade.orders[trade.orders.index(order)] - # place new order if None was not returned + # place new order if result was not None if requested_rate: self._enter_trade(pair=trade.pair, row=row, trade=trade, requested_rate=requested_rate,