diff --git a/docs/configuration.md b/docs/configuration.md index 7e697cb83..97895e52f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -503,7 +503,8 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo * API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in dry-run mode. * Wallets (`/balance`) are simulated based on `dry_run_wallet`. * Orders are simulated, and will not be posted to the exchange. -* Orders are assumed to fill immediately, and will never time out. +* Market orders fill based on orderbook volume the moment the order is placed. +* Limit orders fill once price reaches the defined level - or time out based on `unfilledtimeout` settings. * In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. * Open orders (not trades, which are stored in the database) are reset on bot restart. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 07ac337fc..595fca106 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -567,7 +567,7 @@ class Exchange: rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) - dry_order = { + dry_order: Dict[str, Any] = { 'id': order_id, 'symbol': pair, 'price': rate, @@ -583,26 +583,94 @@ class Exchange: 'fee': None, 'info': {} } - self._store_dry_order(dry_order, pair) + if dry_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: + dry_order["info"] = {"stopPrice": dry_order["price"]} + + if dry_order["type"] == "market": + # Update market order pricing + average = self.get_dry_market_fill_price(pair, side, amount, rate) + dry_order.update({ + 'average': average, + 'cost': dry_order['amount'] * average, + }) + dry_order = self.add_dry_order_fee(pair, dry_order) + + dry_order = self.check_dry_limit_order_filled(dry_order) + + self._dry_run_open_orders[dry_order["id"]] = dry_order # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def _store_dry_order(self, dry_order: Dict, pair: str) -> None: - closed_order = dry_order.copy() - if closed_order['type'] in ["market", "limit"]: - closed_order.update({ - 'status': 'closed', - 'filled': closed_order['amount'], - 'remaining': 0, - 'fee': { - 'currency': self.get_pair_quote_currency(pair), - 'cost': dry_order['cost'] * self.get_fee(pair), - 'rate': self.get_fee(pair) - } - }) - if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: - closed_order["info"].update({"stopPrice": closed_order["price"]}) - self._dry_run_open_orders[closed_order["id"]] = closed_order + def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]: + dry_order.update({ + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } + }) + return dry_order + + def get_dry_market_fill_price(self, pair: str, side: str, amount: float, rate: float) -> float: + """ + Get the market order fill price based on orderbook interpolation + """ + if self.exchange_has('fetchL2OrderBook'): + ob = self.fetch_l2_order_book(pair, 20) + ob_type = 'asks' if side == 'buy' else 'bids' + + remaining_amount = amount + filled_amount = 0 + for book_entry in ob[ob_type]: + book_entry_price = book_entry[0] + book_entry_coin_volume = book_entry[1] + if remaining_amount > 0: + if remaining_amount < book_entry_coin_volume: + filled_amount += remaining_amount * book_entry_price + else: + filled_amount += book_entry_coin_volume * book_entry_price + remaining_amount -= book_entry_coin_volume + else: + break + else: + # If remaining_amount wasn't consumed completely (break was not called) + filled_amount += remaining_amount * book_entry_price + forecast_avg_filled_price = filled_amount / amount + return self.price_to_precision(pair, forecast_avg_filled_price) + + return rate + + def _is_dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool: + if not self.exchange_has('fetchL2OrderBook'): + return True + ob = self.fetch_l2_order_book(pair, 1) + if side == 'buy': + price = ob['asks'][0][0] + logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") + if limit >= price: + return True + else: + price = ob['bids'][0][0] + logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") + if limit <= price: + return True + return False + + def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: + """ + Check dry-run limit order fill and update fee (if it filled). + """ + if order['status'] != "closed" and order['type'] in ["limit"]: + pair = order['symbol'] + if self._is_dry_limit_order_filled(pair, order['side'], order['price']): + order.update({ + 'status': 'closed', + 'filled': order['amount'], + 'remaining': 0, + }) + self.add_dry_order_fee(pair, order) + + return order def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: """ @@ -611,6 +679,7 @@ class Exchange: """ try: order = self._dry_run_open_orders[order_id] + order = self.check_dry_limit_order_filled(order) return order except KeyError as e: # Gracefully handle errors with dry-run orders. diff --git a/tests/conftest.py b/tests/conftest.py index c6a0dfcfd..6e00db274 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1088,6 +1088,40 @@ def order_book_l2(): }) +@pytest.fixture +def order_book_l2_usd(): + return MagicMock(return_value={ + 'symbol': 'LTC/USDT', + 'bids': [ + [25.563, 49.269], + [25.562, 83.0], + [25.56, 106.0], + [25.559, 15.381], + [25.558, 29.299], + [25.557, 34.624], + [25.556, 10.0], + [25.555, 14.684], + [25.554, 45.91], + [25.553, 50.0] + ], + 'asks': [ + [25.566, 14.27], + [25.567, 48.484], + [25.568, 92.349], + [25.572, 31.48], + [25.573, 23.0], + [25.574, 20.0], + [25.575, 89.606], + [25.576, 262.016], + [25.577, 178.557], + [25.578, 78.614] + ], + 'timestamp': None, + 'datetime': None, + 'nonce': 2372149736 + }) + + @pytest.fixture def ohlcv_history_list(): return [ diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f5becc274..ba9ec8bc7 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -947,6 +947,72 @@ def test_create_dry_run_order(default_conf, mocker, side, exchange_name): assert order["symbol"] == "ETH/BTC" +@pytest.mark.parametrize("side,startprice,endprice", [ + ("buy", 25.563, 25.566), + ("sell", 25.566, 25.563) +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, + exchange_name, order_book_l2_usd): + default_conf['dry_run'] = True + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + fetch_l2_order_book=order_book_l2_usd, + ) + + order = exchange.create_dry_run_order( + pair='LTC/USDT', ordertype='limit', side=side, amount=1, rate=startprice) + assert order_book_l2_usd.call_count == 1 + assert 'id' in order + assert f'dry_run_{side}_' in order["id"] + assert order["side"] == side + assert order["type"] == "limit" + assert order["symbol"] == "LTC/USDT" + order_book_l2_usd.reset_mock() + + order_closed = exchange.fetch_dry_run_order(order['id']) + assert order_book_l2_usd.call_count == 1 + assert order_closed['status'] == 'open' + assert not order['fee'] + + order_book_l2_usd.reset_mock() + order_closed['price'] = endprice + + order_closed = exchange.fetch_dry_run_order(order['id']) + assert order_closed['status'] == 'closed' + assert order['fee'] + + +@pytest.mark.parametrize("side,amount,endprice", [ + ("buy", 1, 25.566), + ("buy", 100, 25.5672), # Requires interpolation + ("buy", 1000, 25.575), # More than orderbook return + ("sell", 1, 25.563), + ("sell", 100, 25.5625), # Requires interpolation + ("sell", 1000, 25.5555), # More than orderbook return +]) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_create_dry_run_order_market_fill(default_conf, mocker, side, amount, endprice, + exchange_name, order_book_l2_usd): + default_conf['dry_run'] = True + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + fetch_l2_order_book=order_book_l2_usd, + ) + + order = exchange.create_dry_run_order( + pair='LTC/USDT', ordertype='market', side=side, amount=amount, rate=25.5) + assert 'id' in order + assert f'dry_run_{side}_' in order["id"] + assert order["side"] == side + assert order["type"] == "market" + assert order["symbol"] == "LTC/USDT" + assert order['status'] == 'closed' + assert round(order["average"], 4) == round(endprice, 4) + + @pytest.mark.parametrize("side", [ ("buy"), ("sell") @@ -2117,6 +2183,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange def test_cancel_order_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True) assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {} diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 8a41099f4..cc29dc157 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -679,6 +679,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'filled': 0.0, } ), + _is_dry_limit_order_filled=MagicMock(return_value=True), get_fee=fee, ) mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=1000) @@ -703,8 +704,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert msg == {'result': 'Created sell orders for all open trades.'} freqtradebot.enter_positions() - msg = rpc._rpc_forcesell('1') - assert msg == {'result': 'Created sell order for trade 1.'} + msg = rpc._rpc_forcesell('2') + assert msg == {'result': 'Created sell order for trade 2.'} freqtradebot.state = State.STOPPED with pytest.raises(RPCException, match=r'.*trader is not running*'): @@ -715,9 +716,11 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 + mocker.patch( + 'freqtrade.exchange.Exchange._is_dry_limit_order_filled', MagicMock(return_value=False)) freqtradebot.enter_positions() # make an limit-buy open trade - trade = Trade.query.filter(Trade.id == '1').first() + trade = Trade.query.filter(Trade.id == '3').first() filled_amount = trade.amount / 2 # Fetch order - it's open first, and closed after cancel_order is called. mocker.patch( @@ -738,7 +741,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated - rpc._rpc_forcesell('1') + rpc._rpc_forcesell('3') assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount @@ -766,8 +769,8 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: } ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called - msg = rpc._rpc_forcesell('2') - assert msg == {'result': 'Created sell order for trade 2.'} + msg = rpc._rpc_forcesell('4') + assert msg == {'result': 'Created sell order for trade 4.'} assert cancel_order_mock.call_count == 2 assert trade.amount == amount diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 8d48e2286..b8dd112c9 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -996,7 +996,8 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): get_balances=MagicMock(return_value=ticker), fetch_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) + markets=PropertyMock(return_value=markets), + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_get_signal(ftbot, (True, False)) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 39ef6a1ab..782ae69c6 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -218,6 +218,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) status_table = MagicMock() mocker.patch.multiple( @@ -666,6 +667,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -724,6 +726,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) freqtradebot = FreqtradeBot(default_conf) @@ -784,6 +787,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=True), ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) @@ -800,9 +804,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None context.args = ["all"] telegram._forcesell(update=update, context=context) - # Called for each trade 4 times - assert msg_mock.call_count == 12 - msg = msg_mock.call_args_list[2][0][0] + # Called for each trade 2 times + assert msg_mock.call_count == 8 + msg = msg_mock.call_args_list[1][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 66f87f7c9..aac8d0c4c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -304,6 +304,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -333,6 +334,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) # Save state of current whitelist @@ -2532,6 +2534,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2595,6 +2598,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2647,6 +2651,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -2749,6 +2754,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke price_to_precision=lambda s, x, y: y, stoploss=stoploss, cancel_stoploss_order=cancel_order, + _is_dry_limit_order_filled=MagicMock(side_effect=[True, False]), ) freqtrade = FreqtradeBot(default_conf) @@ -2791,6 +2797,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f get_fee=fee, amount_to_precision=lambda s, x, y: y, price_to_precision=lambda s, x, y: y, + _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]), ) stoploss = MagicMock(return_value={ @@ -2859,6 +2866,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) patch_whitelist(mocker, default_conf) freqtrade = FreqtradeBot(default_conf) @@ -3464,6 +3472,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_b }), buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, + _is_dry_limit_order_filled=MagicMock(return_value=False), ) default_conf['ask_strategy'] = { 'ignore_roi_if_buy_signal': False