added partial fills for dry run

If we buy 2 coins at $2.5 and in order book only 1 coin is left for $2.5 and then the order should be partially filled. Previously it would be fully filled.

freqtrade/exchange/exchange.py

@@ -694,42 +694,66 @@ class Exchange:
 
         return rate
 
-    def _is_dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool:
+    def _fill_dry_limit_order(self, pair: str, side: str,  # noqa: max-complexity: 13
+                                    limit: float, given_amount: float) -> Tuple[Optional[float], float]:
+        """
+        Returns average price and filled amount
+        """
         if not self.exchange_has('fetchL2OrderBook'):
-            return True
+            return limit, given_amount
-        ob = self.fetch_l2_order_book(pair, 1)
+        for order_book_top in (None, 1000):
-        try:
+            ob = self.fetch_l2_order_book(pair, order_book_top)
             if side == 'buy':
-                price = ob['asks'][0][0]
+                obd = ob['asks']
-                logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}")
-                if limit >= price:
-                    return True
             else:
-                price = ob['bids'][0][0]
+                obd = ob['bids']
-                logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}")
+            try:
-                if limit <= price:
+                logger.debug(
-                    return True
+                    f"{pair} checking dry {side}-order: "
-        except IndexError:
+                    f"{limit=}, {given_amount=}, {order_book_top=}")
-            # Ignore empty orderbooks when filling - can be filled with the next iteration.
+                cost = 0
-            pass
+                total_filled_amount = 0
-        return False
+                remaining_amount = given_amount
+                for price, amount in obd:
+                    if (side == 'buy' and limit < price
+                            or side == 'sell' and limit > price):
+                        break
+                    filled_amount = min(amount, remaining_amount)
+                    cost += price * filled_amount
+                    total_filled_amount += filled_amount
+                    remaining_amount -= filled_amount
+                    if total_filled_amount == given_amount:
+                        break
+                else:
+                    continue
+                break
+            except IndexError:
+                # Ignore empty orderbooks when filling - can be filled with the next iteration.
+                pass
+        average_price = cost / total_filled_amount if total_filled_amount else None
+        return average_price, total_filled_amount
 
     def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]:
         """
-        Check dry-run limit order fill and update fee (if it filled).
+        Check dry-run limit order fill and update fee (if it is filled).
         """
         if (order['status'] != "closed"
                 and order['type'] in ["limit"]
                 and not order.get('ft_order_type')):
             pair = order['symbol']
-            if self._is_dry_limit_order_filled(pair, order['side'], order['price']):
+            average_price, filled_amount = self._fill_dry_limit_order(
-                order.update({
+                pair, order['side'], order['price'], order['remaining'])
-                    'status': 'closed',
-                    'filled': order['amount'],
-                    'remaining': 0,
-                })
-                self.add_dry_order_fee(pair, order)
 
+            if filled_amount:
+                order['remaining'] -= filled_amount
+                order_cost = order['average'] * order['filled'] + average_price * filled_amount
+                order['filled'] += filled_amount
+                order['average'] = order_cost / order['filled']
+                order['cost'] = order_cost
+
+            if order['remaining'] == 0:
+                order['status'] = 'closed'
+                self.add_dry_order_fee(pair, order)
         return order
 
     def fetch_dry_run_order(self, order_id) -> Dict[str, Any]:
@@ -740,6 +764,7 @@ class Exchange:
         try:
             order = self._dry_run_open_orders[order_id]
             order = self.check_dry_limit_order_filled(order)
+            logger.debug(order)
             return order
         except KeyError as e:
             # Gracefully handle errors with dry-run orders.
@@ -1528,7 +1553,7 @@ class Exchange:
             else:
                 logger.debug(
                     "Fetching trades for pair %s, since %s %s...",
-                    pair,  since,
+                    pair, since,
                     '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
                 )
                 trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)

tests/exchange/test_exchange.py

@@ -996,17 +996,20 @@ def test_create_dry_run_order(default_conf, mocker, side, exchange_name):
 ])
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
 def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice,
-                                         exchange_name, order_book_l2_usd):
+                                         exchange_name, order_book_l2_usd, caplog):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
     mocker.patch.multiple('freqtrade.exchange.Exchange',
                           exchange_has=MagicMock(return_value=True),
                           fetch_l2_order_book=order_book_l2_usd,
                           )
-
+    pair = 'LTC/USDT'
+    caplog.set_level(logging.DEBUG)
     order = exchange.create_dry_run_order(
-        pair='LTC/USDT', ordertype='limit', side=side, amount=1, rate=startprice)
+        pair=pair, ordertype='limit', side=side, amount=1, rate=startprice)
-    assert order_book_l2_usd.call_count == 1
+
+    extra_check = log_has_re(rf"{pair} checking dry {side}-order: .* order_book_top=1000", caplog)
+    assert order_book_l2_usd.call_count == 1 + int(extra_check)
     assert 'id' in order
     assert f'dry_run_{side}_' in order["id"]
     assert order["side"] == side
@@ -1014,14 +1017,29 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
     assert order["symbol"] == "LTC/USDT"
     order_book_l2_usd.reset_mock()
 
-    order_closed = exchange.fetch_dry_run_order(order['id'])
+    order = exchange.fetch_dry_run_order(order['id'])
-    assert order_book_l2_usd.call_count == 1
+    extra_check = log_has_re(rf"{pair} checking dry {side}-order: .* order_book_top=1000", caplog)
-    assert order_closed['status'] == 'open'
+    assert order_book_l2_usd.call_count == 1 + int(extra_check)
+    assert order['status'] == 'open'
     assert not order['fee']
-    assert order_closed['filled'] == 0
+    ob_side = 'asks' if side == 'buy' else 'bids'
+    assert order['filled'] == 0
 
     order_book_l2_usd.reset_mock()
-    order_closed['price'] = endprice
+    order['price'] = endprice
+    order['amount'] = 50
+    order['remaining'] = 50
+
+    order = exchange.fetch_dry_run_order(order['id'])
+    assert order['status'] == 'open'
+    assert not order['fee']
+    assert order['filled'] == min(order_book_l2_usd.return_value[ob_side][0][1], order['amount'])
+
+    order_book_l2_usd.reset_mock()
+    order['price'] = endprice
+    order['amount'] = 1
+    order['remaining'] = 1
+    order['filled'] = 0
 
     order_closed = exchange.fetch_dry_run_order(order['id'])
     assert order_closed['status'] == 'closed'
@@ -2359,7 +2377,7 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange
 def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
-    mocker.patch('freqtrade.exchange.Exchange._is_dry_limit_order_filled', return_value=True)
+    mocker.patch('freqtrade.exchange.Exchange._fill_dry_limit_order', side_effect=lambda *_: _[-2:])
     assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {}
     assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {}
 

tests/rpc/test_rpc.py

@@ -713,7 +713,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
                 'filled': 0.0,
             }
         ),
-        _is_dry_limit_order_filled=MagicMock(return_value=True),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_: _[-2:]),
         get_fee=fee,
     )
     mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=1000)
@@ -751,7 +751,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
     freqtradebot.state = State.RUNNING
     assert cancel_order_mock.call_count == 0
     mocker.patch(
-        'freqtrade.exchange.Exchange._is_dry_limit_order_filled', MagicMock(return_value=False))
+        'freqtrade.exchange.Exchange._fill_dry_limit_order', MagicMock(side_effect=lambda *_: (None, 0)),)
     freqtradebot.enter_positions()
     # make an limit-buy open trade
     trade = Trade.query.filter(Trade.id == '3').first()

tests/rpc/test_rpc_apiserver.py

@@ -1103,7 +1103,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets):
         fetch_ticker=ticker,
         get_fee=fee,
         markets=PropertyMock(return_value=markets),
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     patch_get_signal(ftbot)
 

tests/rpc/test_rpc_telegram.py

@@ -301,7 +301,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None:
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=True),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_: _[-2:]),
     )
     status_table = MagicMock()
     mocker.patch.multiple(
@@ -1004,7 +1004,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=True),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_: _[-2:]),
     )
 
     freqtradebot = FreqtradeBot(default_conf)
@@ -1065,7 +1065,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=True),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_: _[-2:]),
     )
 
     freqtradebot = FreqtradeBot(default_conf)
@@ -1128,7 +1128,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=True),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_: _[-2:]),
     )
     default_conf['max_open_trades'] = 4
     freqtradebot = FreqtradeBot(default_conf)

tests/test_freqtradebot.py

@@ -29,6 +29,16 @@ from tests.conftest_trades import (MOCK_TRADE_COUNT, mock_order_1, mock_order_2,
                                    mock_order_5_stoploss, mock_order_6_sell)
 
 
+class SideEffect:
+    # https://stackoverflow.com/a/64992350
+    def __init__(self, *fns):
+        self.fs = iter(fns)
+
+    def __call__(self, *args, **kwargs):
+        f = next(self.fs)
+        return f(*args, **kwargs)
+
+
 def patch_RPCManager(mocker) -> MagicMock:
     """
     This function mock RPC manager to avoid repeating this code in almost every tests
@@ -246,7 +256,7 @@ def test_total_open_trades_stakes(mocker, default_conf_usdt, ticker_usdt, fee) -
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     freqtrade = FreqtradeBot(default_conf_usdt)
     patch_get_signal(freqtrade)
@@ -276,7 +286,7 @@ def test_create_trade(default_conf_usdt, ticker_usdt, limit_buy_order_usdt, fee,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
 
     # Save state of current whitelist
@@ -2665,7 +2675,7 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     patch_whitelist(mocker, default_conf_usdt)
     freqtrade = FreqtradeBot(default_conf_usdt)
@@ -2733,7 +2743,7 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     patch_whitelist(mocker, default_conf_usdt)
     freqtrade = FreqtradeBot(default_conf_usdt)
@@ -2787,7 +2797,7 @@ def test_execute_trade_exit_custom_exit_price(default_conf_usdt, ticker_usdt, fe
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     config = deepcopy(default_conf_usdt)
     config['custom_price_max_distance_ratio'] = 0.1
@@ -2855,7 +2865,7 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     patch_whitelist(mocker, default_conf_usdt)
     freqtrade = FreqtradeBot(default_conf_usdt)
@@ -2963,7 +2973,7 @@ def test_execute_trade_exit_with_stoploss_on_exchange(default_conf_usdt, ticker_
         price_to_precision=lambda s, x, y: y,
         stoploss=stoploss,
         cancel_stoploss_order=cancel_order,
-        _is_dry_limit_order_filled=MagicMock(side_effect=[True, False]),
+        _fill_dry_limit_order=MagicMock(side_effect=SideEffect(lambda *_: _[-2:], lambda *_: (None, 0))),
     )
 
     freqtrade = FreqtradeBot(default_conf_usdt)
@@ -3006,7 +3016,7 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit(default_conf_usdt
         get_fee=fee,
         amount_to_precision=lambda s, x, y: y,
         price_to_precision=lambda s, x, y: y,
-        _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]),
+        _fill_dry_limit_order=MagicMock(side_effect=SideEffect(lambda *_: (None, 0), lambda *_: _[-2:])),
     )
 
     stoploss = MagicMock(return_value={
@@ -3075,7 +3085,7 @@ def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_usdt,
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     patch_whitelist(mocker, default_conf_usdt)
     freqtrade = FreqtradeBot(default_conf_usdt)
@@ -3530,7 +3540,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usd
             {'id': 1234553383}
         ]),
         get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(return_value=False),
+        _fill_dry_limit_order=MagicMock(side_effect=lambda *_ :(None, 0)),
     )
     default_conf_usdt['ask_strategy'] = {
         'ignore_roi_if_buy_signal': False