Merge branch 'develop' into feat/short

freqtrade/freqtradebot.py

@@ -1578,9 +1578,14 @@ class FreqtradeBot(LoggingMixin):
             # Handling of this will happen in check_handle_timedout.
             return True
 
-        order = self.handle_order_fee(trade, order)
+        order_obj = trade.select_order_by_order_id(order_id)
+        if not order_obj:
+            raise DependencyException(
+                f"Order_obj not found for {order_id}. This should not have happened.")
+        self.handle_order_fee(trade, order_obj, order)
 
-        trade.update(order)
+        trade.update_trade(order_obj)
+        # TODO: is the below necessary? it's already done in update_trade for filled buys
         trade.recalc_trade_from_orders()
         Trade.commit()
 
@@ -1634,17 +1639,15 @@ class FreqtradeBot(LoggingMixin):
             return real_amount
         return amount
 
-    def handle_order_fee(self, trade: Trade, order: Dict[str, Any]) -> Dict[str, Any]:
+    def handle_order_fee(self, trade: Trade, order_obj: Order, order: Dict[str, Any]) -> None:
         # Try update amount (binance-fix)
         try:
             new_amount = self.get_real_amount(trade, order)
             if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount,
                            abs_tol=constants.MATH_CLOSE_PREC):
-                order['amount'] = new_amount
+                order_obj.ft_fee_base = trade.amount - new_amount
-                order.pop('filled', None)
         except DependencyException as exception:
             logger.warning("Could not update trade amount: %s", exception)
-        return order
 
     def get_real_amount(self, trade: Trade, order: Dict) -> float:
         """

freqtrade/optimize/backtesting.py

@@ -139,7 +139,8 @@ class Backtesting:
     def __del__(self):
         self.cleanup()
 
-    def cleanup(self):
+    @staticmethod
+    def cleanup():
         LoggingMixin.show_output = True
         PairLocks.use_db = True
         Trade.use_db = True

freqtrade/persistence/migrations.py

@@ -191,19 +191,22 @@ def drop_orders_table(engine, table_back_name: str):
         connection.execute(text("drop table orders"))
 
 
-def migrate_orders_table(engine, table_back_name: str, cols: List):
+def migrate_orders_table(engine, table_back_name: str, cols_order: List):
+
+    ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null')
 
     # let SQLAlchemy create the schema as required
-    leverage = get_column_def(cols, 'leverage', '1.0')
+    leverage = get_column_def(cols_order, 'leverage', '1.0')
     # sqlite does not support literals for booleans
     with engine.begin() as connection:
         connection.execute(text(f"""
             insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
-            order_date, order_filled_date, order_update_date, leverage)
+            order_date, order_filled_date, order_update_date, ft_fee_base, leverage)
             select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, null average, remaining, cost,
-            order_date, order_filled_date, order_update_date, {leverage} leverage
+            order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base,
+            {leverage} leverage
             from {table_back_name}
             """))
 
@@ -222,22 +225,22 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     inspector = inspect(engine)
 
     cols = inspector.get_columns('trades')
+    cols_orders = inspector.get_columns('orders')
     tabs = get_table_names_for_table(inspector, 'trades')
-    cols_order = inspector.get_columns('orders')
     table_back_name = get_backup_name(tabs, 'trades_bak')
     order_tabs = get_table_names_for_table(inspector, 'orders')
     order_table_bak_name = get_backup_name(order_tabs, 'orders_bak')
 
     # Check if migration necessary
     # Migrates both trades and orders table!
-    if not has_column(cols, 'enter_tag'):
+    # if not has_column(cols, 'buy_tag'):
+    if ('orders' not in previous_tables
+            or not has_column(cols_orders, 'ft_fee_base')
+            or not has_column(cols_orders, 'leverage')):
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")
         migrate_trades_and_orders_table(
-            decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_order)
+            decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_orders)
-        # Reread columns - the above recreated the table!
-        inspector = inspect(engine)
-        cols = inspector.get_columns('trades')
 
     if 'orders' not in previous_tables and 'trades' in previous_tables:
         logger.info('Moving open orders to Orders table.')

freqtrade/persistence/models.py

@@ -17,7 +17,6 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
 from freqtrade.enums import SellType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.leverage import interest
-from freqtrade.misc import safe_value_fallback
 from freqtrade.persistence.migrations import check_migrate
 
 
@@ -117,14 +116,15 @@ class Order(_DECL_BASE):
 
     trade = relationship("Trade", back_populates="orders")
 
-    ft_order_side = Column(String(25), nullable=False)
+    # order_side can only be 'buy', 'sell' or 'stoploss'
-    ft_pair = Column(String(25), nullable=False)
+    ft_order_side: str = Column(String(25), nullable=False)
+    ft_pair: str = Column(String(25), nullable=False)
     ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
 
     order_id = Column(String(255), nullable=False, index=True)
     status = Column(String(255), nullable=True)
     symbol = Column(String(25), nullable=True)
-    order_type = Column(String(50), nullable=True)
+    order_type: str = Column(String(50), nullable=True)
     side = Column(String(25), nullable=True)
     price = Column(Float, nullable=True)
     average = Column(Float, nullable=True)
@@ -137,10 +137,29 @@ class Order(_DECL_BASE):
     order_update_date = Column(DateTime, nullable=True)
     leverage = Column(Float, nullable=True, default=1.0)
 
+    ft_fee_base = Column(Float, nullable=True)
+
     @property
-    def order_date_utc(self):
+    def order_date_utc(self) -> datetime:
+        """ Order-date with UTC timezoneinfo"""
         return self.order_date.replace(tzinfo=timezone.utc)
 
+    @property
+    def safe_price(self) -> float:
+        return self.average or self.price
+
+    @property
+    def safe_filled(self) -> float:
+        return self.filled or self.amount or 0.0
+
+    @property
+    def safe_fee_base(self) -> float:
+        return self.ft_fee_base or 0.0
+
+    @property
+    def safe_amount_after_fee(self) -> float:
+        return self.safe_filled - self.safe_fee_base
+
     def __repr__(self):
 
         return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
@@ -584,50 +603,46 @@ class LocalTrade():
             f"Trailing stoploss saved us: "
             f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.")
 
-    def update(self, order: Dict) -> None:
+    def update_trade(self, order: Order) -> None:
         """
         Updates this entity with amount and actual open/close rates.
         :param order: order retrieved by exchange.fetch_order()
         :return: None
         """
-        order_type = order['type']
-
-        if 'is_short' in order and order['side'] == 'sell':
-            # Only set's is_short on opening trades, ignores non-shorts
-            self.is_short = order['is_short']
 
         # Ignore open and cancelled orders
-        if order['status'] == 'open' or safe_value_fallback(order, 'average', 'price') is None:
+        if order.status == 'open' or order.safe_price is None:
             return
 
-        logger.info('Updating trade (id=%s) ...', self.id)
+        logger.info(f'Updating trade (id={self.id}) ...')
 
-        if order_type in ('market', 'limit') and self.enter_side == order['side']:
+        if order.ft_order_side == self.enter_side:
             # Update open rate and actual amount
-            self.open_rate = float(safe_value_fallback(order, 'average', 'price'))
+            self.open_rate = order.safe_price
-            self.amount = float(safe_value_fallback(order, 'filled', 'amount'))
+            self.amount = order.safe_amount_after_fee
-            if 'leverage' in order:
+            # if 'leverage' in order:
-                self.leverage = order['leverage']
+            # TODO-lev: order.leverage is not properly filled on the order object!
+            # self.leverage = order.leverage
             if self.is_open:
                 payment = "SELL" if self.is_short else "BUY"
-                logger.info(f'{order_type.upper()}_{payment} has been fulfilled for {self}.')
+                logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
             self.open_order_id = None
             self.recalc_trade_from_orders()
-        elif order_type in ('market', 'limit') and self.exit_side == order['side']:
+        elif order.ft_order_side == self.exit_side:
             if self.is_open:
                 payment = "BUY" if self.is_short else "SELL"
                 # * On margin shorts, you buy a little bit more than the amount (amount + interest)
-                logger.info(f'{order_type.upper()}_{payment} has been fulfilled for {self}.')
+                logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
-            self.close(safe_value_fallback(order, 'average', 'price'))
+            self.close(order.safe_price)
-        elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'):
+        elif order.ft_order_side == 'stoploss':
             self.stoploss_order_id = None
             self.close_rate_requested = self.stop_loss
             self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
             if self.is_open:
-                logger.info(f'{order_type.upper()} is hit for {self}.')
+                logger.info(f'{order.order_type.upper()} is hit for {self}.')
-            self.close(safe_value_fallback(order, 'average', 'price'))
+            self.close(order.safe_price)
         else:
-            raise ValueError(f'Unknown order type: {order_type}')
+            raise ValueError(f'Unknown order type: {order.order_type}')
         Trade.commit()
 
     def close(self, rate: float, *, show_msg: bool = True) -> None:
@@ -857,7 +872,7 @@ class LocalTrade():
                     (o.status not in NON_OPEN_EXCHANGE_STATES)):
                 continue
 
-            tmp_amount = o.amount
+            tmp_amount = o.safe_amount_after_fee
             tmp_price = o.average or o.price
             if o.filled is not None:
                 tmp_amount = o.filled

requirements.txt

@@ -32,7 +32,7 @@ sdnotify==0.3.2
 
 # API Server
 fastapi==0.74.0
-uvicorn==0.17.4
+uvicorn==0.17.5
 pyjwt==2.3.0
 aiofiles==0.8.0
 psutil==5.9.0

tests/conftest.py

@@ -285,6 +285,9 @@ def create_mock_trades_with_leverage(fee, use_db: bool = True):
     """
     Create some fake trades ...
     """
+    if use_db:
+        Trade.query.session.rollback()
+
     def add_trade(trade):
         if use_db:
             Trade.query.session.add(trade)
@@ -1818,7 +1821,7 @@ def limit_sell_order_open():
         'id': 'mocked_limit_sell',
         'type': 'limit',
         'side': 'sell',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': arrow.utcnow().isoformat(),
         'timestamp': arrow.utcnow().int_timestamp,
         'price': 0.00001173,
@@ -2825,7 +2828,7 @@ def limit_sell_order_usdt_open():
         'id': 'mocked_limit_sell_usdt',
         'type': 'limit',
         'side': 'sell',
-        'pair': 'mocked',
+        'symbol': 'mocked',
         'datetime': arrow.utcnow().isoformat(),
         'timestamp': arrow.utcnow().int_timestamp,
         'price': 2.20,

tests/optimize/test_backtesting.py

@@ -52,6 +52,13 @@ def trim_dictlist(dict_list, num):
     return new
 
 
+@pytest.fixture(autouse=True)
+def backtesting_cleanup() -> None:
+    yield None
+
+    Backtesting.cleanup()
+
+
 def load_data_test(what, testdatadir):
     timerange = TimeRange.parse_timerange('1510694220-1510700340')
     data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir,
@@ -568,8 +575,6 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None:
     trade = backtesting._enter_trade(pair, row=row, direction='long')
     assert trade is None
 
-    backtesting.cleanup()
-
 
 def test_backtest__get_sell_trade_entry(default_conf, fee, mocker) -> None:
     default_conf['use_sell_signal'] = False

tests/rpc/test_rpc.py

@@ -12,6 +12,7 @@ from freqtrade.enums import State, TradingMode
 from freqtrade.enums.signaltype import SignalDirection
 from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError
 from freqtrade.persistence import Trade
+from freqtrade.persistence.models import Order
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.rpc import RPC, RPCException
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -292,8 +293,10 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee,
     assert trade
 
     # Simulate buy & sell
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
-    trade.update(limit_sell_order)
+    trade.update_trade(oobj)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
     trade.close_date = datetime.utcnow()
     trade.is_open = False
 
@@ -432,28 +435,32 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee,
     freqtradebot.enter_positions()
     trade = Trade.query.first()
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     # Update the ticker with a market going up
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_sell_up
     )
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
     trade.close_date = datetime.utcnow()
     trade.is_open = False
 
     freqtradebot.enter_positions()
     trade = Trade.query.first()
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Update the ticker with a market going up
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_sell_up
     )
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
     trade.close_date = datetime.utcnow()
     trade.is_open = False
 
@@ -512,14 +519,16 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee,
     freqtradebot.enter_positions()
     trade = Trade.query.first()
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
     # Update the ticker with a market going up
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_sell_up,
         get_fee=fee
     )
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
     trade.close_date = datetime.utcnow()
     trade.is_open = False
 
@@ -771,13 +780,13 @@ def test_rpc_forceexit(default_conf, ticker, fee, mocker) -> None:
     mocker.patch(
         'freqtrade.exchange.Exchange.fetch_order',
         side_effect=[{
-            'id': '1234',
+            'id': trade.orders[0].order_id,
             'status': 'open',
             'type': 'limit',
             'side': 'buy',
             'filled': filled_amount
         }, {
-            'id': '1234',
+            'id': trade.orders[0].order_id,
             'status': 'closed',
             'type': 'limit',
             'side': 'buy',
@@ -857,10 +866,12 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -891,10 +902,12 @@ def test_enter_tag_performance_handle(default_conf, ticker, limit_buy_order, fee
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -963,10 +976,12 @@ def test_sell_reason_performance_handle(default_conf, ticker, limit_buy_order, f
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -1035,10 +1050,12 @@ def test_mix_tag_performance_handle(default_conf, ticker, limit_buy_order, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False

tests/rpc/test_rpc_apiserver.py

@@ -1189,6 +1189,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets):
                      data='{"tradeid": "1"}')
     assert_response(rc, 502)
     assert rc.json() == {"error": "Error querying /api/v1/forcesell: invalid argument"}
+    Trade.query.session.rollback()
 
     ftbot.enter_positions()
 

tests/rpc/test_rpc_telegram.py

@@ -424,10 +424,12 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobjs)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -467,8 +469,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee,
 
     trades = Trade.query.all()
     for trade in trades:
-        trade.update(limit_buy_order)
+        trade.update_trade(oobj)
-        trade.update(limit_sell_order)
+        trade.update_trade(oobjs)
         trade.close_date = datetime.utcnow()
         trade.is_open = False
 
@@ -533,10 +535,12 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobjs)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -580,8 +584,8 @@ def test_weekly_handle(default_conf, update, ticker, limit_buy_order, fee,
 
     trades = Trade.query.all()
     for trade in trades:
-        trade.update(limit_buy_order)
+        trade.update_trade(oobj)
-        trade.update(limit_sell_order)
+        trade.update_trade(oobjs)
         trade.close_date = datetime.utcnow()
         trade.is_open = False
 
@@ -649,10 +653,12 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobjs = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobjs)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -696,8 +702,8 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee,
 
     trades = Trade.query.all()
     for trade in trades:
-        trade.update(limit_buy_order)
+        trade.update_trade(oobj)
-        trade.update(limit_sell_order)
+        trade.update_trade(oobjs)
         trade.close_date = datetime.utcnow()
         trade.is_open = False
 
@@ -767,7 +773,9 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee,
     trade = Trade.query.first()
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
+
     context = MagicMock()
     # Test with invalid 2nd argument (should silently pass)
     context.args = ["aaa"]
@@ -782,7 +790,9 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee,
 
     # Update the ticker with a market going up
     mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up)
-    trade.update(limit_sell_order)
+    # Simulate fulfilled LIMIT_SELL order for trade
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.now(timezone.utc)
     trade.is_open = False
@@ -1303,10 +1313,12 @@ def test_telegram_performance_handle(default_conf, update, ticker, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -1332,11 +1344,13 @@ def test_telegram_buy_tag_performance_handle(default_conf, update, ticker, fee,
     assert trade
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
 
     trade.enter_tag = "TESTBUY"
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -1373,13 +1387,14 @@ def test_telegram_sell_reason_performance_handle(default_conf, update, ticker, f
     freqtradebot.enter_positions()
     trade = Trade.query.first()
     assert trade
-
-    # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
-
     trade.sell_reason = 'TESTSELL'
+    # Simulate fulfilled LIMIT_BUY order for trade
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
+
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False
@@ -1417,14 +1432,16 @@ def test_telegram_mix_tag_performance_handle(default_conf, update, ticker, fee,
     trade = Trade.query.first()
     assert trade
 
-    # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_buy_order)
-
     trade.enter_tag = "TESTBUY"
     trade.sell_reason = "TESTSELL"
 
+    # Simulate fulfilled LIMIT_BUY order for trade
+    oobj = Order.parse_from_ccxt_object(limit_buy_order, limit_buy_order['symbol'], 'buy')
+    trade.update_trade(oobj)
+
     # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(limit_sell_order)
+    oobj = Order.parse_from_ccxt_object(limit_sell_order, limit_sell_order['symbol'], 'sell')
+    trade.update_trade(oobj)
 
     trade.close_date = datetime.utcnow()
     trade.is_open = False

tests/test_freqtradebot.py

@@ -228,7 +228,8 @@ def test_edge_overrides_stoploss(limit_order, fee, caplog, mocker,
     freqtrade.enter_positions()
     trade = Trade.query.first()
     caplog.clear()
-    trade.update(limit_order['buy'])
+    oobj = Order.parse_from_ccxt_object(limit_order['buy'], 'ADA/USDT', 'buy')
+    trade.update_trade(oobj)
     #############################################
 
     # stoploss shoud be hit
@@ -299,7 +300,9 @@ def test_create_trade(default_conf_usdt, ticker_usdt, limit_order,
     assert trade.exchange == 'binance'
 
     # Simulate fulfilled LIMIT_BUY order for trade
-    trade.update(limit_order[enter_side(is_short)])
+    oobj = Order.parse_from_ccxt_object(
+        limit_order[enter_side(is_short)], 'ADA/USDT', enter_side(is_short))
+    trade.update_trade(oobj)
 
     assert trade.open_rate == open_rate
     assert trade.amount == 30.0
@@ -1101,11 +1104,17 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
     trade.is_short = is_short
     trade.is_open = True
     trade.open_order_id = None
-    trade.stoploss_order_id = 100
+    trade.stoploss_order_id = "100"
+    trade.orders.append(Order(
+        ft_order_side='stoploss',
+        order_id='100',
+        ft_pair=trade.pair,
+        ft_is_open=True,
+    ))
     assert trade
 
     stoploss_order_hit = MagicMock(return_value={
-        'id': 100,
+        'id': "100",
         'status': 'closed',
         'type': 'stop_loss_limit',
         'price': 3,
@@ -1832,9 +1841,10 @@ def test_update_trade_state(
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[])
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
                  return_value=order['amount'])
+    order_id = order['id']
 
     trade = Trade(
-        open_order_id=123,
+        open_order_id=order_id,
         fee_open=0.001,
         fee_close=0.001,
         open_rate=0.01,
@@ -1843,29 +1853,35 @@ def test_update_trade_state(
         exchange="binance",
         is_short=is_short
     )
+    trade.orders.append(Order(
+        ft_order_side=enter_side(is_short),
+        price=0.01,
+        order_id=order_id,
+
+    ))
     assert not freqtrade.update_trade_state(trade, None)
     assert log_has_re(r'Orderid for trade .* is empty.', caplog)
     caplog.clear()
     # Add datetime explicitly since sqlalchemy defaults apply only once written to database
-    freqtrade.update_trade_state(trade, '123')
+    freqtrade.update_trade_state(trade, order_id)
     # Test amount not modified by fee-logic
     assert not log_has_re(r'Applying fee to .*', caplog)
     caplog.clear()
     assert trade.open_order_id is None
     assert trade.amount == order['amount']
 
-    trade.open_order_id = '123'
+    trade.open_order_id = order_id
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81)
     assert trade.amount != 90.81
     # test amount modified by fee-logic
-    freqtrade.update_trade_state(trade, '123')
+    freqtrade.update_trade_state(trade, order_id)
     assert trade.amount == 90.81
     assert trade.open_order_id is None
 
     trade.is_open = True
     trade.open_order_id = None
     # Assert we call handle_trade() if trade is feasible for execution
-    freqtrade.update_trade_state(trade, '123')
+    freqtrade.update_trade_state(trade, order_id)
 
     assert log_has_re('Found open order for.*', caplog)
     limit_buy_order_usdt_new = deepcopy(limit_order)
@@ -1874,7 +1890,7 @@ def test_update_trade_state(
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', side_effect=ValueError)
     mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order_usdt_new)
-    res = freqtrade.update_trade_state(trade, '123')
+    res = freqtrade.update_trade_state(trade, order_id)
     # Cancelled empty
     assert res is True
 
@@ -1890,6 +1906,8 @@ def test_update_trade_state_withorderdict(
 ):
     order = limit_order[enter_side(is_short)]
     trades_for_order[0]['amount'] = initial_amount
+    order_id = "oid_123456"
+    order['id'] = order_id
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
     # fetch_order should not be called!!
     mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError))
@@ -1905,15 +1923,27 @@ def test_update_trade_state_withorderdict(
         open_date=arrow.utcnow().datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        open_order_id="123456",
+        open_order_id=order_id,
         is_open=True,
         is_short=is_short
     )
-    freqtrade.update_trade_state(trade, '123456', order)
+    trade.orders.append(
+        Order(
+            ft_order_side=enter_side(is_short),
+            ft_pair=trade.pair,
+            ft_is_open=True,
+            order_id=order_id,
+        )
+    )
+    log_text = r'Applying fee on amount for .*'
+    freqtrade.update_trade_state(trade, order_id, order)
     assert trade.amount != amount
-    assert trade.amount == order['amount']
     if has_rounding_fee:
-        assert log_has_re(r'Applying fee on amount for .*', caplog)
+        assert pytest.approx(trade.amount) == 29.992
+        assert log_has_re(log_text, caplog)
+    else:
+        assert pytest.approx(trade.amount) == order['amount']
+        assert not log_has_re(log_text, caplog)
 
 
 @pytest.mark.parametrize("is_short", [False, True])
@@ -1974,12 +2004,12 @@ def test_update_trade_state_sell(
         fee_open=0.0025,
         fee_close=0.0025,
         open_date=arrow.utcnow().datetime,
-        open_order_id="123456",
+        open_order_id=open_order['id'],
         is_open=True,
         interest_rate=0.0005,
         is_short=is_short
     )
-    order = Order.parse_from_ccxt_object(open_order, 'LTC/ETH', (enter_side(is_short)))
+    order = Order.parse_from_ccxt_object(open_order, 'LTC/ETH', exit_side(is_short))
     trade.orders.append(order)
     assert order.status == 'open'
     freqtrade.update_trade_state(trade, trade.open_order_id, l_order)
@@ -2026,7 +2056,8 @@ def test_handle_trade(
     assert trade
 
     time.sleep(0.01)  # Race condition fix
-    trade.update(enter_order)
+    oobj = Order.parse_from_ccxt_object(enter_order, enter_order['symbol'], enter_side(is_short))
+    trade.update_trade(oobj)
     assert trade.is_open is True
     freqtrade.wallets.update()
 
@@ -2035,8 +2066,9 @@ def test_handle_trade(
     assert freqtrade.handle_trade(trade) is True
     assert trade.open_order_id == exit_order['id']
 
-    # Simulate fulfilled LIMIT order for trade
+    # Simulate fulfilled LIMIT_SELL order for trade
-    trade.update(exit_order)
+    oobj = Order.parse_from_ccxt_object(exit_order, exit_order['symbol'], exit_side(is_short))
+    trade.update_trade(oobj)
 
     assert trade.close_rate == 2.0 if is_short else 2.2
     assert trade.close_profit == close_profit
@@ -2230,8 +2262,10 @@ def test_close_trade(
     trade.is_short = is_short
     assert trade
 
-    trade.update(enter_order)
+    oobj = Order.parse_from_ccxt_object(enter_order, enter_order['symbol'], 'buy')
-    trade.update(exit_order)
+    trade.update_trade(oobj)
+    oobj = Order.parse_from_ccxt_object(exit_order, exit_order['symbol'], 'sell')
+    trade.update_trade(oobj)
     assert trade.is_open is False
 
     with pytest.raises(DependencyException, match=r'.*closed trade.*'):
@@ -2258,6 +2292,8 @@ def test_check_handle_timedout_buy_usercustom(
 ) -> None:
 
     old_order = limit_sell_order_old if is_short else limit_buy_order_old
+    old_order['id'] = open_trade.open_order_id
+
     default_conf_usdt["unfilledtimeout"] = {"buy": 30,
                                             "sell": 1400} if is_short else {"buy": 1400, "sell": 30}
 
@@ -2617,6 +2653,7 @@ def test_check_handle_timedout_partial_fee(
     # TODO-lev: use is_short or remove it
     rpc_mock = patch_RPCManager(mocker)
     limit_buy_order_old_partial['id'] = open_trade.open_order_id
+    limit_buy_order_old_partial_canceled['id'] = open_trade.open_order_id
     cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled)
     mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=0))
     patch_exchange(mocker)
@@ -3511,6 +3548,7 @@ def test_sell_profit_only(
         fee, mocker, profit_only, bid, ask, handle_first, handle_second, sell_type) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
+    eside = enter_side(is_short)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
@@ -3519,7 +3557,7 @@ def test_sell_profit_only(
             'last': bid
         }),
         create_order=MagicMock(side_effect=[
-            limit_order_open[enter_side(is_short)],
+            limit_order_open[eside],
             {'id': 1234553382},
         ]),
         get_fee=fee,
@@ -3540,7 +3578,8 @@ def test_sell_profit_only(
 
     trade = Trade.query.first()
     trade.is_short = is_short
-    trade.update(limit_order[enter_side(is_short)])
+    oobj = Order.parse_from_ccxt_object(limit_order[eside], limit_order[eside]['symbol'], eside)
+    trade.update_trade(oobj)
     freqtrade.wallets.update()
     patch_get_signal(freqtrade, enter_long=False, exit_short=is_short, exit_long=not is_short)
     assert freqtrade.handle_trade(trade) is handle_first
@@ -3576,7 +3615,9 @@ def test_sell_not_enough_balance(default_conf_usdt, limit_order, limit_order_ope
 
     trade = Trade.query.first()
     amnt = trade.amount
-    trade.update(limit_order['buy'])
+
+    oobj = Order.parse_from_ccxt_object(limit_order['buy'], limit_order['buy']['symbol'], 'buy')
+    trade.update_trade(oobj)
     patch_get_signal(freqtrade, enter_long=False, exit_long=True)
     mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=trade.amount * 0.985))
 
@@ -3668,6 +3709,7 @@ def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_order_op
                                   fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
+    eside = enter_side(is_short)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
@@ -3676,7 +3718,7 @@ def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_order_op
             'last': 2.19
         }),
         create_order=MagicMock(side_effect=[
-            limit_order_open[enter_side(is_short)],
+            limit_order_open[eside],
             {'id': 1234553382},
         ]),
         get_fee=fee,
@@ -3691,7 +3733,9 @@ def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_order_op
 
     trade = Trade.query.first()
     trade.is_short = is_short
-    trade.update(limit_order[enter_side(is_short)])
+    oobj = Order.parse_from_ccxt_object(
+        limit_order[eside], limit_order[eside]['symbol'], eside)
+    trade.update_trade(oobj)
     freqtrade.wallets.update()
     if is_short:
         patch_get_signal(freqtrade, enter_long=False, enter_short=True, exit_short=True)
@@ -3786,6 +3830,7 @@ def test_trailing_stop_loss_positive(
     enter_price = limit_order[enter_side(is_short)]['price']
     patch_RPCManager(mocker)
     patch_exchange(mocker)
+    eside = enter_side(is_short)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
@@ -3794,7 +3839,7 @@ def test_trailing_stop_loss_positive(
             'last': enter_price - (-0.01 if is_short else 0.01),
         }),
         create_order=MagicMock(side_effect=[
-            limit_order_open[enter_side(is_short)],
+            limit_order_open[eside],
             {'id': 1234553382},
         ]),
         get_fee=fee,
@@ -3813,7 +3858,8 @@ def test_trailing_stop_loss_positive(
 
     trade = Trade.query.first()
     trade.is_short = is_short
-    trade.update(limit_order[enter_side(is_short)])
+    oobj = Order.parse_from_ccxt_object(limit_order[eside], limit_order[eside]['symbol'], eside)
+    trade.update_trade(oobj)
     caplog.set_level(logging.DEBUG)
     # stop-loss not reached
     assert freqtrade.handle_trade(trade) is False
@@ -3880,6 +3926,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_
                                           is_short, fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
+    eside = enter_side(is_short)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
@@ -3888,7 +3935,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_
             'last': 2.0
         }),
         create_order=MagicMock(side_effect=[
-            limit_order_open[enter_side(is_short)],
+            limit_order_open[eside],
             {'id': 1234553382},
             {'id': 1234553383}
         ]),
@@ -3906,7 +3953,10 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_
 
     trade = Trade.query.first()
     trade.is_short = is_short
-    trade.update(limit_order[enter_side(is_short)])
+
+    oobj = Order.parse_from_ccxt_object(
+        limit_order[eside], limit_order[eside]['symbol'], eside)
+    trade.update_trade(oobj)
     # Sell due to min_roi_reached
     patch_get_signal(freqtrade, enter_long=not is_short, enter_short=is_short, exit_short=is_short)
     assert freqtrade.handle_trade(trade) is True
@@ -4294,7 +4344,9 @@ def test_order_book_depth_of_market(
         assert len(Trade.query.all()) == 1
 
         # Simulate fulfilled LIMIT_BUY order for trade
-        trade.update(limit_order_open[enter_side(is_short)])
+        oobj = Order.parse_from_ccxt_object(
+            limit_order_open[enter_side(is_short)], 'ADA/USDT', enter_side(is_short))
+        trade.update_trade(oobj)
 
         assert trade.open_rate == ticker_usdt.return_value[ticker_side]
         assert whitelist == default_conf_usdt['exchange']['pair_whitelist']
@@ -4389,7 +4441,8 @@ def test_order_book_ask_strategy(
     assert trade
 
     time.sleep(0.01)  # Race condition fix
-    trade.update(limit_buy_order_usdt)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, limit_buy_order_usdt['symbol'], 'buy')
+    trade.update_trade(oobj)
     freqtrade.wallets.update()
     assert trade.is_open is True
 

tests/test_integration.py

@@ -4,6 +4,7 @@ import pytest
 
 from freqtrade.enums import SellType
 from freqtrade.persistence import Trade
+from freqtrade.persistence.models import Order
 from freqtrade.rpc.rpc import RPC
 from freqtrade.strategy.interface import SellCheckTuple
 from tests.conftest import get_patched_freqtradebot, patch_get_signal
@@ -94,7 +95,11 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee,
     trades = Trade.query.all()
     # Make sure stoploss-order is open and trade is bought (since we mock update_trade_state)
     for trade in trades:
-        trade.stoploss_order_id = 3
+        stoploss_order_closed['id'] = '3'
+        oobj = Order.parse_from_ccxt_object(stoploss_order_closed, trade.pair, 'stoploss')
+
+        trade.orders.append(oobj)
+        trade.stoploss_order_id = '3'
         trade.open_order_id = None
 
     n = freqtrade.exit_positions(trades)

tests/test_persistence.py

@@ -474,7 +474,8 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_
     assert trade.close_date is None
 
     trade.open_order_id = 'something'
-    trade.update(enter_order)
+    oobj = Order.parse_from_ccxt_object(enter_order, 'ADA/USDT', enter_side)
+    trade.update_trade(oobj)
     assert trade.open_order_id is None
     assert trade.open_rate == open_rate
     assert trade.close_profit is None
@@ -487,7 +488,8 @@ def test_update_limit_order(fee, caplog, limit_buy_order_usdt, limit_sell_order_
 
     caplog.clear()
     trade.open_order_id = 'something'
-    trade.update(exit_order)
+    oobj = Order.parse_from_ccxt_object(exit_order, 'ADA/USDT', exit_side)
+    trade.update_trade(oobj)
     assert trade.open_order_id is None
     assert trade.close_rate == close_rate
     assert trade.close_profit == profit
@@ -517,7 +519,8 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee,
     )
 
     trade.open_order_id = 'something'
-    trade.update(market_buy_order_usdt)
+    oobj = Order.parse_from_ccxt_object(market_buy_order_usdt, 'ADA/USDT', 'buy')
+    trade.update_trade(oobj)
     assert trade.open_order_id is None
     assert trade.open_rate == 2.0
     assert trade.close_profit is None
@@ -530,7 +533,8 @@ def test_update_market_order(market_buy_order_usdt, market_sell_order_usdt, fee,
     caplog.clear()
     trade.is_open = True
     trade.open_order_id = 'something'
-    trade.update(market_sell_order_usdt)
+    oobj = Order.parse_from_ccxt_object(market_sell_order_usdt, 'ADA/USDT', 'sell')
+    trade.update_trade(oobj)
     assert trade.open_order_id is None
     assert trade.close_rate == 2.2
     assert trade.close_profit == round(0.0945137157107232, 8)
@@ -583,8 +587,12 @@ def test_calc_open_close_trade_price(
 
     trade.open_order_id = f'something-{is_short}-{lev}-{exchange}'
 
-    trade.update(limit_buy_order_usdt)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy')
-    trade.update(limit_sell_order_usdt)
+    trade.update_trade(oobj)
+
+    oobj = Order.parse_from_ccxt_object(limit_sell_order_usdt, 'ADA/USDT', 'sell')
+    trade.update_trade(oobj)
+
     trade.open_rate = 2.0
     trade.close_rate = 2.2
     trade.recalc_open_trade_value()
@@ -640,7 +648,8 @@ def test_calc_close_trade_price_exception(limit_buy_order_usdt, fee):
     )
 
     trade.open_order_id = 'something'
-    trade.update(limit_buy_order_usdt)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy')
+    trade.update_trade(oobj)
     assert trade.calc_close_trade_value() == 0.0
 
 
@@ -662,7 +671,8 @@ def test_update_open_order(limit_buy_order_usdt):
     assert trade.close_date is None
 
     limit_buy_order_usdt['status'] = 'open'
-    trade.update(limit_buy_order_usdt)
+    oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'buy')
+    trade.update_trade(oobj)
 
     assert trade.open_order_id is None
     assert trade.close_profit is None
@@ -682,8 +692,9 @@ def test_update_invalid_order(limit_buy_order_usdt):
         trading_mode=margin
     )
     limit_buy_order_usdt['type'] = 'invalid'
+    oobj = Order.parse_from_ccxt_object(limit_buy_order_usdt, 'ADA/USDT', 'meep')
     with pytest.raises(ValueError, match=r'Unknown order type'):
-        trade.update(limit_buy_order_usdt)
+        trade.update_trade(oobj)
 
 
 @pytest.mark.parametrize('exchange', ['binance', 'kraken'])
@@ -737,6 +748,9 @@ def test_calc_open_trade_value(
         trading_mode=trading_mode
     )
     trade.open_order_id = 'open_trade'
+    oobj = Order.parse_from_ccxt_object(
+        limit_buy_order_usdt, 'ADA/USDT', 'sell' if is_short else 'buy')
+    trade.update_trade(oobj)  # Buy @ 2.0
 
     # Get the open rate price with the standard fee rate
     assert trade._calc_open_trade_value() == result