Merge pull request #8264 from xmatthias/sqlalchemy_2

Sqlalchemy 2

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
         # stages: [push]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v0.991"
+    rev: "v1.0.1"
     hooks:
       - id: mypy
         exclude: build_helpers
@@ -18,6 +18,7 @@ repos:
           - types-requests==2.28.11.15
           - types-tabulate==0.9.0.1
           - types-python-dateutil==2.8.19.9
+          - SQLAlchemy==2.0.4
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

build_helpers/pre_commit_update.py

@@ -8,12 +8,17 @@ import yaml
 
 pre_commit_file = Path('.pre-commit-config.yaml')
 require_dev = Path('requirements-dev.txt')
+require = Path('requirements.txt')
 
 with require_dev.open('r') as rfile:
     requirements = rfile.readlines()
 
+with require.open('r') as rfile:
+    requirements.extend(rfile.readlines())
+
 # Extract types only
-type_reqs = [r.strip('\n') for r in requirements if r.startswith('types-')]
+type_reqs = [r.strip('\n') for r in requirements if r.startswith(
+    'types-') or r.startswith('SQLAlchemy')]
 
 with pre_commit_file.open('r') as file:
     f = yaml.load(file, Loader=yaml.FullLoader)

freqtrade/data/btanalysis.py

@@ -346,7 +346,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     return df_final[df_final['open_trades'] > max_open_trades]
 
 
-def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame:
+def trade_list_to_dataframe(trades: Union[List[Trade], List[LocalTrade]]) -> pd.DataFrame:
     """
     Convert list of Trade objects to pandas Dataframe
     :param trades: List of trade objects

freqtrade/freqtradebot.py

@@ -633,7 +633,7 @@ class FreqtradeBot(LoggingMixin):
                 return
 
             remaining = (trade.amount - amount) * current_exit_rate
-            if remaining < min_exit_stake:
+            if min_exit_stake and remaining < min_exit_stake:
                 logger.info(f"Remaining amount of {remaining} would be smaller "
                             f"than the minimum of {min_exit_stake}.")
                 return
@@ -1314,7 +1314,7 @@ class FreqtradeBot(LoggingMixin):
                                                          default_retval=order_obj.price)(
                 trade=trade, order=order_obj, pair=trade.pair,
                 current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate,
-                current_order_rate=order_obj.price, entry_tag=trade.enter_tag,
+                current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag,
                 side=trade.entry_side)
 
             replacing = True
@@ -1330,7 +1330,8 @@ class FreqtradeBot(LoggingMixin):
                     # place new order only if new price is supplied
                     self.execute_entry(
                         pair=trade.pair,
-                        stake_amount=(order_obj.remaining * order_obj.price / trade.leverage),
+                        stake_amount=(
+                            order_obj.safe_remaining * order_obj.safe_price / trade.leverage),
                         price=adjusted_entry_price,
                         trade=trade,
                         is_short=trade.is_short,
@@ -1344,6 +1345,8 @@ class FreqtradeBot(LoggingMixin):
         """
 
         for trade in Trade.get_open_order_trades():
+            if not trade.open_order_id:
+                continue
             try:
                 order = self.exchange.fetch_order(trade.open_order_id, trade.pair)
             except (ExchangeError):
@@ -1368,6 +1371,9 @@ class FreqtradeBot(LoggingMixin):
         """
         was_trade_fully_canceled = False
         side = trade.entry_side.capitalize()
+        if not trade.open_order_id:
+            logger.warning(f"No open order for {trade}.")
+            return False
 
         # Cancelled orders may have the status of 'canceled' or 'closed'
         if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
@@ -1454,7 +1460,7 @@ class FreqtradeBot(LoggingMixin):
                     return False
 
             try:
-                co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
+                co = self.exchange.cancel_order_with_result(order['id'], trade.pair,
                                                             trade.amount)
             except InvalidOrderException:
                 logger.exception(
@@ -1639,7 +1645,7 @@ class FreqtradeBot(LoggingMixin):
             profit = trade.calc_profit(rate=order_rate, amount=amount, open_rate=trade.open_rate)
             profit_ratio = trade.calc_profit_ratio(order_rate, amount, trade.open_rate)
         else:
-            order_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
+            order_rate = trade.safe_close_rate
             profit = trade.calc_profit(rate=order_rate) + (0.0 if fill else trade.realized_profit)
             profit_ratio = trade.calc_profit_ratio(order_rate)
             amount = trade.amount
@@ -1694,7 +1700,7 @@ class FreqtradeBot(LoggingMixin):
             raise DependencyException(
                 f"Order_obj not found for {order_id}. This should not have happened.")
 
-        profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
+        profit_rate: float = trade.safe_close_rate
         profit_trade = trade.calc_profit(rate=profit_rate)
         current_rate = self.exchange.get_rate(
             trade.pair, side='exit', is_short=trade.is_short, refresh=False)
@@ -1737,7 +1743,8 @@ class FreqtradeBot(LoggingMixin):
 #
 
     def update_trade_state(
-            self, trade: Trade, order_id: str, action_order: Optional[Dict[str, Any]] = None,
+            self, trade: Trade, order_id: Optional[str],
+            action_order: Optional[Dict[str, Any]] = None,
             stoploss_order: bool = False, send_msg: bool = True) -> bool:
         """
         Checks trades with open orders and updates the amount if necessary

freqtrade/optimize/backtesting.py

@@ -440,7 +440,8 @@ class Backtesting:
                               side_1 * abs(self.strategy.trailing_stop_positive / leverage)))
             else:
                 # Worst case: price ticks tiny bit above open and dives down.
-                stop_rate = row[OPEN_IDX] * (1 - side_1 * abs(trade.stop_loss_pct / leverage))
+                stop_rate = row[OPEN_IDX] * (1 - side_1 * abs(
+                    (trade.stop_loss_pct or 0.0) / leverage))
                 if is_short:
                     assert stop_rate > row[LOW_IDX]
                 else:
@@ -472,7 +473,7 @@ class Backtesting:
             # - (Expected abs profit - open_rate - open_fee) / (fee_close -1)
             roi_rate = trade.open_rate * roi / leverage
             open_fee_rate = side_1 * trade.open_rate * (1 + side_1 * trade.fee_open)
-            close_rate = -(roi_rate + open_fee_rate) / (trade.fee_close - side_1 * 1)
+            close_rate = -(roi_rate + open_fee_rate) / ((trade.fee_close or 0.0) - side_1 * 1)
             if is_short:
                 is_new_roi = row[OPEN_IDX] < close_rate
             else:
@@ -563,7 +564,7 @@ class Backtesting:
             pos_trade = self._get_exit_for_signal(trade, row, exit_, amount)
             if pos_trade is not None:
                 order = pos_trade.orders[-1]
-                if self._get_order_filled(order.price, row):
+                if self._get_order_filled(order.ft_price, row):
                     order.close_bt_order(current_date, trade)
                     trade.recalc_trade_from_orders()
                 self.wallets.update()
@@ -664,6 +665,7 @@ class Backtesting:
             side=trade.exit_side,
             order_type=order_type,
             status="open",
+            ft_price=close_rate,
             price=close_rate,
             average=close_rate,
             amount=amount,
@@ -887,6 +889,7 @@ class Backtesting:
                 order_date=current_time,
                 order_filled_date=current_time,
                 order_update_date=current_time,
+                ft_price=propose_rate,
                 price=propose_rate,
                 average=propose_rate,
                 amount=amount,
@@ -895,7 +898,7 @@ class Backtesting:
                 cost=stake_amount + trade.fee_open,
             )
             trade.orders.append(order)
-            if pos_adjust and self._get_order_filled(order.price, row):
+            if pos_adjust and self._get_order_filled(order.ft_price, row):
                 order.close_bt_order(current_time, trade)
             else:
                 trade.open_order_id = str(self.order_id_counter)
@@ -1008,15 +1011,15 @@ class Backtesting:
         # only check on new candles for open entry orders
         if order.side == trade.entry_side and current_time > order.order_date_utc:
             requested_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price,
-                                                   default_retval=order.price)(
+                                                   default_retval=order.ft_price)(
                 trade=trade,  # type: ignore[arg-type]
                 order=order, pair=trade.pair, current_time=current_time,
-                proposed_rate=row[OPEN_IDX], current_order_rate=order.price,
+                proposed_rate=row[OPEN_IDX], current_order_rate=order.ft_price,
                 entry_tag=trade.enter_tag, side=trade.trade_direction
             )  # default value is current order price
 
             # cancel existing order whenever a new rate is requested (or None)
-            if requested_rate == order.price:
+            if requested_rate == order.ft_price:
                 # assumption: there can't be multiple open entry orders at any given time
                 return False
             else:
@@ -1028,7 +1031,8 @@ class Backtesting:
             if requested_rate:
                 self._enter_trade(pair=trade.pair, row=row, trade=trade,
                                   requested_rate=requested_rate,
-                                  requested_stake=(order.remaining * order.price / trade.leverage),
+                                  requested_stake=(
+                                    order.safe_remaining * order.ft_price / trade.leverage),
                                   direction='short' if trade.is_short else 'long')
                 self.replaced_entry_orders += 1
             else:
@@ -1095,7 +1099,7 @@ class Backtesting:
         for trade in list(LocalTrade.bt_trades_open_pp[pair]):
             # 3. Process entry orders.
             order = trade.select_order(trade.entry_side, is_open=True)
-            if order and self._get_order_filled(order.price, row):
+            if order and self._get_order_filled(order.ft_price, row):
                 order.close_bt_order(current_time, trade)
                 trade.open_order_id = None
                 self.wallets.update()
@@ -1106,7 +1110,7 @@ class Backtesting:
 
                 # 5. Process exit orders.
             order = trade.select_order(trade.exit_side, is_open=True)
-            if order and self._get_order_filled(order.price, row):
+            if order and self._get_order_filled(order.ft_price, row):
                 order.close_bt_order(current_time, trade)
                 trade.open_order_id = None
                 sub_trade = order.safe_amount_after_fee != trade.amount
@@ -1115,7 +1119,7 @@ class Backtesting:
                     trade.recalc_trade_from_orders()
                 else:
                     trade.close_date = current_time
-                    trade.close(order.price, show_msg=False)
+                    trade.close(order.ft_price, show_msg=False)
 
                     # logger.debug(f"{pair} - Backtesting exit {trade}")
                     LocalTrade.close_bt_trade(trade)

freqtrade/persistence/base.py

@@ -1,7 +1,9 @@
 
-from typing import Any
+from sqlalchemy.orm import DeclarativeBase, Session, scoped_session
-
-from sqlalchemy.orm import declarative_base
 
 
-_DECL_BASE: Any = declarative_base()
+SessionType = scoped_session[Session]
+
+
+class ModelBase(DeclarativeBase):
+    pass

freqtrade/persistence/models.py

@@ -2,6 +2,7 @@
 This module contains the class to persist trades into SQLite
 """
 import logging
+from typing import Any, Dict
 
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.exc import NoSuchModuleError
@@ -9,7 +10,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.pool import StaticPool
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase
 from freqtrade.persistence.migrations import check_migrate
 from freqtrade.persistence.pairlock import PairLock
 from freqtrade.persistence.trade_model import Order, Trade
@@ -29,7 +30,7 @@ def init_db(db_url: str) -> None:
     :param db_url: Database to use
     :return: None
     """
-    kwargs = {}
+    kwargs: Dict[str, Any] = {}
 
     if db_url == 'sqlite:///':
         raise OperationalException(
@@ -54,10 +55,12 @@ def init_db(db_url: str) -> None:
     # Scoped sessions proxy requests to the appropriate thread-local session.
     # We should use the scoped_session object - not a seperately initialized version
     Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=False))
+    Order._session = Trade._session
+    PairLock._session = Trade._session
     Trade.query = Trade._session.query_property()
     Order.query = Trade._session.query_property()
     PairLock.query = Trade._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
-    _DECL_BASE.metadata.create_all(engine)
+    ModelBase.metadata.create_all(engine)
-    check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
+    check_migrate(engine, decl_base=ModelBase, previous_tables=previous_tables)

freqtrade/persistence/pairlock.py

@@ -1,33 +1,36 @@
 from datetime import datetime, timezone
-from typing import Any, Dict, Optional
+from typing import Any, ClassVar, Dict, Optional
 
-from sqlalchemy import Boolean, Column, DateTime, Integer, String, or_
+from sqlalchemy import String, or_
-from sqlalchemy.orm import Query
+from sqlalchemy.orm import Mapped, Query, mapped_column
+from sqlalchemy.orm.scoping import _QueryDescriptorType
 
 from freqtrade.constants import DATETIME_PRINT_FORMAT
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase, SessionType
 
 
-class PairLock(_DECL_BASE):
+class PairLock(ModelBase):
     """
     Pair Locks database model.
     """
     __tablename__ = 'pairlocks'
+    query: ClassVar[_QueryDescriptorType]
+    _session: ClassVar[SessionType]
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(primary_key=True)
 
-    pair = Column(String(25), nullable=False, index=True)
+    pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)
     # lock direction - long, short or * (for both)
-    side = Column(String(25), nullable=False, default="*")
+    side: Mapped[str] = mapped_column(String(25), nullable=False, default="*")
-    reason = Column(String(255), nullable=True)
+    reason: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     # Time the pair was locked (start time)
-    lock_time = Column(DateTime(), nullable=False)
+    lock_time: Mapped[datetime] = mapped_column(nullable=False)
     # Time until the pair is locked (end time)
-    lock_end_time = Column(DateTime(), nullable=False, index=True)
+    lock_end_time: Mapped[datetime] = mapped_column(nullable=False, index=True)
 
-    active = Column(Boolean, nullable=False, default=True, index=True)
+    active: Mapped[bool] = mapped_column(nullable=False, default=True, index=True)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
         lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
         return (

freqtrade/persistence/pairlock_middleware.py

@@ -133,8 +133,8 @@ class PairLocks():
             PairLock.query.session.commit()
         else:
             # used in backtesting mode; don't show log messages for speed
-            locks = PairLocks.get_pair_locks(None)
+            locksb = PairLocks.get_pair_locks(None)
-            for lock in locks:
+            for lock in locksb:
                 if lock.reason == reason:
                     lock.active = False
 

freqtrade/persistence/trade_model.py

@@ -5,11 +5,11 @@ import logging
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from math import isclose
-from typing import Any, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional, cast
 
-from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String,
+from sqlalchemy import Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func
-                        UniqueConstraint, desc, func)
+from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship
-from sqlalchemy.orm import Query, lazyload, relationship
+from sqlalchemy.orm.scoping import _QueryDescriptorType
 
 from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES,
                                  BuySell, LongShort)
@@ -17,14 +17,14 @@ from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.exchange import amount_to_contract_precision, price_to_precision
 from freqtrade.leverage import interest
-from freqtrade.persistence.base import _DECL_BASE
+from freqtrade.persistence.base import ModelBase, SessionType
 from freqtrade.util import FtPrecise
 
 
 logger = logging.getLogger(__name__)
 
 
-class Order(_DECL_BASE):
+class Order(ModelBase):
     """
     Order database model
     Keeps a record of all orders placed on the exchange
@@ -36,41 +36,44 @@ class Order(_DECL_BASE):
     Mirrors CCXT Order structure
     """
     __tablename__ = 'orders'
+    query: ClassVar[_QueryDescriptorType]
+    _session: ClassVar[SessionType]
+
     # Uniqueness should be ensured over pair, order_id
     # its likely that order_id is unique per Pair on some exchanges.
     __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
-    ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
+    ft_trade_id: Mapped[int] = mapped_column(Integer, ForeignKey('trades.id'), index=True)
 
-    trade = relationship("Trade", back_populates="orders")
+    trade: Mapped[List["Trade"]] = relationship("Trade", back_populates="orders")
 
     # order_side can only be 'buy', 'sell' or 'stoploss'
-    ft_order_side = Column(String(25), nullable=False)
+    ft_order_side: Mapped[str] = mapped_column(String(25), nullable=False)
-    ft_pair = Column(String(25), nullable=False)
+    ft_pair: Mapped[str] = mapped_column(String(25), nullable=False)
-    ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
+    ft_is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True)
-    ft_amount = Column(Float(), nullable=False)
+    ft_amount: Mapped[float] = mapped_column(Float(), nullable=False)
-    ft_price = Column(Float(), nullable=False)
+    ft_price: Mapped[float] = mapped_column(Float(), nullable=False)
 
-    order_id = Column(String(255), nullable=False, index=True)
+    order_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
-    status = Column(String(255), nullable=True)
+    status: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
-    symbol = Column(String(25), nullable=True)
+    symbol: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)
-    order_type = Column(String(50), nullable=True)
+    # TODO: type: order_type type is Optional[str]
-    side = Column(String(25), nullable=True)
+    order_type: Mapped[str] = mapped_column(String(50), nullable=True)
-    price = Column(Float(), nullable=True)
+    side: Mapped[str] = mapped_column(String(25), nullable=True)
-    average = Column(Float(), nullable=True)
+    price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    amount = Column(Float(), nullable=True)
+    average: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    filled = Column(Float(), nullable=True)
+    amount: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    remaining = Column(Float(), nullable=True)
+    filled: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    cost = Column(Float(), nullable=True)
+    remaining: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    stop_price = Column(Float(), nullable=True)
+    cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    order_date = Column(DateTime(), nullable=True, default=datetime.utcnow)
+    stop_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-    order_filled_date = Column(DateTime(), nullable=True)
+    order_date: Mapped[datetime] = mapped_column(nullable=True, default=datetime.utcnow)
-    order_update_date = Column(DateTime(), nullable=True)
+    order_filled_date: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    order_update_date: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
 
-    funding_fee = Column(Float(), nullable=True)
+    ft_fee_base: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)
-
-    ft_fee_base = Column(Float(), nullable=True)
 
     @property
     def order_date_utc(self) -> datetime:
@@ -96,6 +99,10 @@ class Order(_DECL_BASE):
     def safe_filled(self) -> float:
         return self.filled if self.filled is not None else self.amount or 0.0
 
+    @property
+    def safe_cost(self) -> float:
+        return self.cost or 0.0
+
     @property
     def safe_remaining(self) -> float:
         return (
@@ -151,7 +158,7 @@ class Order(_DECL_BASE):
         self.order_update_date = datetime.now(timezone.utc)
 
     def to_ccxt_object(self) -> Dict[str, Any]:
-        order = {
+        order: Dict[str, Any] = {
             'id': self.order_id,
             'symbol': self.ft_pair,
             'price': self.price,
@@ -213,7 +220,7 @@ class Order(_DECL_BASE):
         # Assumes backtesting will use date_last_filled_utc to calculate future funding fees.
         self.funding_fee = trade.funding_fees
 
-        if (self.ft_order_side == trade.entry_side):
+        if (self.ft_order_side == trade.entry_side and self.price):
             trade.open_rate = self.price
             trade.recalc_trade_from_orders()
             trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True)
@@ -293,15 +300,15 @@ class LocalTrade():
 
     exchange: str = ''
     pair: str = ''
-    base_currency: str = ''
+    base_currency: Optional[str] = ''
-    stake_currency: str = ''
+    stake_currency: Optional[str] = ''
     is_open: bool = True
     fee_open: float = 0.0
     fee_open_cost: Optional[float] = None
-    fee_open_currency: str = ''
+    fee_open_currency: Optional[str] = ''
-    fee_close: float = 0.0
+    fee_close: Optional[float] = 0.0
     fee_close_cost: Optional[float] = None
-    fee_close_currency: str = ''
+    fee_close_currency: Optional[str] = ''
     open_rate: float = 0.0
     open_rate_requested: Optional[float] = None
     # open_trade_value - calculated via _calc_open_trade_value
@@ -311,7 +318,7 @@ class LocalTrade():
     close_profit: Optional[float] = None
     close_profit_abs: Optional[float] = None
     stake_amount: float = 0.0
-    max_stake_amount: float = 0.0
+    max_stake_amount: Optional[float] = 0.0
     amount: float = 0.0
     amount_requested: Optional[float] = None
     open_date: datetime
@@ -320,9 +327,9 @@ class LocalTrade():
     # absolute value of the stop loss
     stop_loss: float = 0.0
     # percentage value of the stop loss
-    stop_loss_pct: float = 0.0
+    stop_loss_pct: Optional[float] = 0.0
     # absolute value of the initial stop loss
-    initial_stop_loss: float = 0.0
+    initial_stop_loss: Optional[float] = 0.0
     # percentage value of the initial stop loss
     initial_stop_loss_pct: Optional[float] = None
     # stoploss order id which is on exchange
@@ -330,12 +337,12 @@ class LocalTrade():
     # last update time of the stoploss order on exchange
     stoploss_last_update: Optional[datetime] = None
     # absolute value of the highest reached price
-    max_rate: float = 0.0
+    max_rate: Optional[float] = None
     # Lowest price reached
-    min_rate: float = 0.0
+    min_rate: Optional[float] = None
-    exit_reason: str = ''
+    exit_reason: Optional[str] = ''
-    exit_order_status: str = ''
+    exit_order_status: Optional[str] = ''
-    strategy: str = ''
+    strategy: Optional[str] = ''
     enter_tag: Optional[str] = None
     timeframe: Optional[int] = None
 
@@ -592,7 +599,7 @@ class LocalTrade():
 
         self.stop_loss_pct = -1 * abs(percent)
 
-    def adjust_stop_loss(self, current_price: float, stoploss: float,
+    def adjust_stop_loss(self, current_price: float, stoploss: Optional[float],
                          initial: bool = False, refresh: bool = False) -> None:
         """
         This adjusts the stop loss to it's most recently observed setting
@@ -601,7 +608,7 @@ class LocalTrade():
         :param initial: Called to initiate stop_loss.
             Skips everything if self.stop_loss is already set.
         """
-        if initial and not (self.stop_loss is None or self.stop_loss == 0):
+        if stoploss is None or (initial and not (self.stop_loss is None or self.stop_loss == 0)):
             # Don't modify if called with initial and nothing to do
             return
         refresh = True if refresh and self.nr_of_successful_entries == 1 else False
@@ -640,7 +647,7 @@ class LocalTrade():
             f"initial_stop_loss={self.initial_stop_loss:.8f}, "
             f"stop_loss={self.stop_loss:.8f}. "
             f"Trailing stoploss saved us: "
-            f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.")
+            f"{float(self.stop_loss) - float(self.initial_stop_loss or 0.0):.8f}.")
 
     def update_trade(self, order: Order) -> None:
         """
@@ -792,10 +799,10 @@ class LocalTrade():
 
         return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours)
 
-    def _calc_base_close(self, amount: FtPrecise, rate: float, fee: float) -> FtPrecise:
+    def _calc_base_close(self, amount: FtPrecise, rate: float, fee: Optional[float]) -> FtPrecise:
 
         close_trade = amount * FtPrecise(rate)
-        fees = close_trade * FtPrecise(fee)
+        fees = close_trade * FtPrecise(fee or 0.0)
 
         if self.is_short:
             return close_trade + fees
@@ -1059,10 +1066,14 @@ class LocalTrade():
         return len(self.select_filled_orders('sell'))
 
     @property
-    def sell_reason(self) -> str:
+    def sell_reason(self) -> Optional[str]:
         """ DEPRECATED! Please use exit_reason instead."""
         return self.exit_reason
 
+    @property
+    def safe_close_rate(self) -> float:
+        return self.close_rate or self.close_rate_requested or 0.0
+
     @staticmethod
     def get_trades_proxy(*, pair: Optional[str] = None, is_open: Optional[bool] = None,
                          open_date: Optional[datetime] = None,
@@ -1124,7 +1135,7 @@ class LocalTrade():
     @staticmethod
     def get_open_trades() -> List[Any]:
         """
-        Query trades from persistence layer
+        Retrieve open trades
         """
         return Trade.get_trades_proxy(is_open=True)
 
@@ -1159,7 +1170,7 @@ class LocalTrade():
                 logger.info(f"New stoploss: {trade.stop_loss}.")
 
 
-class Trade(_DECL_BASE, LocalTrade):
+class Trade(ModelBase, LocalTrade):
     """
     Trade database model.
     Also handles updating and querying trades
@@ -1167,79 +1178,98 @@ class Trade(_DECL_BASE, LocalTrade):
     Note: Fields must be aligned with LocalTrade class
     """
     __tablename__ = 'trades'
+    query: ClassVar[_QueryDescriptorType]
+    _session: ClassVar[SessionType]
 
     use_db: bool = True
 
-    id = Column(Integer, primary_key=True)
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)  # type: ignore
 
-    orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan",
+    orders: Mapped[List[Order]] = relationship(
-                          lazy="selectin", innerjoin=True)
+        "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin",
+        innerjoin=True)  # type: ignore
 
-    exchange = Column(String(25), nullable=False)
+    exchange: Mapped[str] = mapped_column(String(25), nullable=False)  # type: ignore
-    pair = Column(String(25), nullable=False, index=True)
+    pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True)  # type: ignore
-    base_currency = Column(String(25), nullable=True)
+    base_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)  # type: ignore
-    stake_currency = Column(String(25), nullable=True)
+    stake_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True)  # type: ignore
-    is_open = Column(Boolean, nullable=False, default=True, index=True)
+    is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True)  # type: ignore
-    fee_open = Column(Float(), nullable=False, default=0.0)
+    fee_open: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0)  # type: ignore
-    fee_open_cost = Column(Float(), nullable=True)
+    fee_open_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
-    fee_open_currency = Column(String(25), nullable=True)
+    fee_open_currency: Mapped[Optional[str]] = mapped_column(
-    fee_close = Column(Float(), nullable=False, default=0.0)
+        String(25), nullable=True)  # type: ignore
-    fee_close_cost = Column(Float(), nullable=True)
+    fee_close: Mapped[Optional[float]] = mapped_column(
-    fee_close_currency = Column(String(25), nullable=True)
+        Float(), nullable=False, default=0.0)  # type: ignore
-    open_rate: float = Column(Float())
+    fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
-    open_rate_requested = Column(Float())
+    fee_close_currency: Mapped[Optional[str]] = mapped_column(
+        String(25), nullable=True)  # type: ignore
+    open_rate: Mapped[float] = mapped_column(Float())  # type: ignore
+    open_rate_requested: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True)  # type: ignore
     # open_trade_value - calculated via _calc_open_trade_value
-    open_trade_value = Column(Float())
+    open_trade_value: Mapped[float] = mapped_column(Float(), nullable=True)  # type: ignore
-    close_rate: Optional[float] = Column(Float())
+    close_rate: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    close_rate_requested = Column(Float())
+    close_rate_requested: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    realized_profit = Column(Float(), default=0.0)
+    realized_profit: Mapped[float] = mapped_column(
-    close_profit = Column(Float())
+        Float(), default=0.0, nullable=True)  # type: ignore
-    close_profit_abs = Column(Float())
+    close_profit: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    stake_amount = Column(Float(), nullable=False)
+    close_profit_abs: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    max_stake_amount = Column(Float())
+    stake_amount: Mapped[float] = mapped_column(Float(), nullable=False)  # type: ignore
-    amount = Column(Float())
+    max_stake_amount: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    amount_requested = Column(Float())
+    amount: Mapped[float] = mapped_column(Float())  # type: ignore
-    open_date = Column(DateTime(), nullable=False, default=datetime.utcnow)
+    amount_requested: Mapped[Optional[float]] = mapped_column(Float())  # type: ignore
-    close_date = Column(DateTime())
+    open_date: Mapped[datetime] = mapped_column(
-    open_order_id = Column(String(255))
+        nullable=False, default=datetime.utcnow)  # type: ignore
+    close_date: Mapped[Optional[datetime]] = mapped_column()  # type: ignore
+    open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)  # type: ignore
     # absolute value of the stop loss
-    stop_loss = Column(Float(), nullable=True, default=0.0)
+    stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0)  # type: ignore
     # percentage value of the stop loss
-    stop_loss_pct = Column(Float(), nullable=True)
+    stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
     # absolute value of the initial stop loss
-    initial_stop_loss = Column(Float(), nullable=True, default=0.0)
+    initial_stop_loss: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True, default=0.0)  # type: ignore
     # percentage value of the initial stop loss
-    initial_stop_loss_pct = Column(Float(), nullable=True)
+    initial_stop_loss_pct: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True)  # type: ignore
     # stoploss order id which is on exchange
-    stoploss_order_id = Column(String(255), nullable=True, index=True)
+    stoploss_order_id: Mapped[Optional[str]] = mapped_column(
+        String(255), nullable=True, index=True)  # type: ignore
     # last update time of the stoploss order on exchange
-    stoploss_last_update = Column(DateTime(), nullable=True)
+    stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True)  # type: ignore
     # absolute value of the highest reached price
-    max_rate = Column(Float(), nullable=True, default=0.0)
+    max_rate: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True, default=0.0)  # type: ignore
     # Lowest price reached
-    min_rate = Column(Float(), nullable=True)
+    min_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
-    exit_reason = Column(String(100), nullable=True)
+    exit_reason: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)  # type: ignore
-    exit_order_status = Column(String(100), nullable=True)
+    exit_order_status: Mapped[Optional[str]] = mapped_column(
-    strategy = Column(String(100), nullable=True)
+        String(100), nullable=True)  # type: ignore
-    enter_tag = Column(String(100), nullable=True)
+    strategy: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)  # type: ignore
-    timeframe = Column(Integer, nullable=True)
+    enter_tag: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)  # type: ignore
+    timeframe: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)  # type: ignore
 
-    trading_mode = Column(Enum(TradingMode), nullable=True)
+    trading_mode: Mapped[TradingMode] = mapped_column(
-    amount_precision = Column(Float(), nullable=True)
+        Enum(TradingMode), nullable=True)  # type: ignore
-    price_precision = Column(Float(), nullable=True)
+    amount_precision: Mapped[Optional[float]] = mapped_column(
-    precision_mode = Column(Integer, nullable=True)
+        Float(), nullable=True)  # type: ignore
-    contract_size = Column(Float(), nullable=True)
+    price_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
+    precision_mode: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)  # type: ignore
+    contract_size: Mapped[Optional[float]] = mapped_column(Float(), nullable=True)  # type: ignore
 
     # Leverage trading properties
-    leverage = Column(Float(), nullable=True, default=1.0)
+    leverage: Mapped[float] = mapped_column(Float(), nullable=True, default=1.0)  # type: ignore
-    is_short = Column(Boolean, nullable=False, default=False)
+    is_short: Mapped[bool] = mapped_column(nullable=False, default=False)  # type: ignore
-    liquidation_price = Column(Float(), nullable=True)
+    liquidation_price: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True)  # type: ignore
 
     # Margin Trading Properties
-    interest_rate = Column(Float(), nullable=False, default=0.0)
+    interest_rate: Mapped[float] = mapped_column(
+        Float(), nullable=False, default=0.0)  # type: ignore
 
     # Futures properties
-    funding_fees = Column(Float(), nullable=True, default=None)
+    funding_fees: Mapped[Optional[float]] = mapped_column(
+        Float(), nullable=True, default=None)  # type: ignore
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -1285,7 +1315,7 @@ class Trade(_DECL_BASE, LocalTrade):
                 trade_filter.append(Trade.close_date > close_date)
             if is_open is not None:
                 trade_filter.append(Trade.is_open.is_(is_open))
-            return Trade.get_trades(trade_filter).all()
+            return cast(List[LocalTrade], Trade.get_trades(trade_filter).all())
         else:
             return LocalTrade.get_trades_proxy(
                 pair=pair, is_open=is_open,
@@ -1294,7 +1324,7 @@ class Trade(_DECL_BASE, LocalTrade):
             )
 
     @staticmethod
-    def get_trades(trade_filter=None, include_orders: bool = True) -> Query:
+    def get_trades(trade_filter=None, include_orders: bool = True) -> Query['Trade']:
         """
         Helper function to query Trades using filters.
         NOTE: Not supported in Backtesting.
@@ -1381,7 +1411,7 @@ class Trade(_DECL_BASE, LocalTrade):
         Returns List of dicts containing all Trades, including profit and trade count
         NOTE: Not supported in Backtesting.
         """
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if minutes:
             start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes)
             filters.append(Trade.close_date >= start_date)
@@ -1414,7 +1444,7 @@ class Trade(_DECL_BASE, LocalTrade):
         NOTE: Not supported in Backtesting.
         """
 
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if (pair is not None):
             filters.append(Trade.pair == pair)
 
@@ -1447,7 +1477,7 @@ class Trade(_DECL_BASE, LocalTrade):
         NOTE: Not supported in Backtesting.
         """
 
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if (pair is not None):
             filters.append(Trade.pair == pair)
 
@@ -1480,7 +1510,7 @@ class Trade(_DECL_BASE, LocalTrade):
         NOTE: Not supported in Backtesting.
         """
 
-        filters = [Trade.is_open.is_(False)]
+        filters: List = [Trade.is_open.is_(False)]
         if (pair is not None):
             filters.append(Trade.pair == pair)
 

freqtrade/rpc/rpc.py

@@ -189,8 +189,8 @@ class RPC:
                 else:
                     # Closed trade ...
                     current_rate = trade.close_rate
-                    current_profit = trade.close_profit
+                    current_profit = trade.close_profit or 0.0
-                    current_profit_abs = trade.close_profit_abs
+                    current_profit_abs = trade.close_profit_abs or 0.0
                 total_profit_abs = trade.realized_profit + current_profit_abs
 
                 # Calculate fiat profit
@@ -373,13 +373,13 @@ class RPC:
 
     def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict:
         """ Returns the X last trades """
-        order_by = Trade.id if order_by_id else Trade.close_date.desc()
+        order_by: Any = Trade.id if order_by_id else Trade.close_date.desc()
         if limit:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
                 order_by).limit(limit).offset(offset)
         else:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
-                Trade.close_date.desc()).all()
+                Trade.close_date.desc())
 
         output = [trade.to_json() for trade in trades]
 
@@ -401,7 +401,7 @@ class RPC:
                 return 'losses'
             else:
                 return 'draws'
-        trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False)
+        trades = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False)
         # Sell reason
         exit_reasons = {}
         for trade in trades:
@@ -410,7 +410,7 @@ class RPC:
             exit_reasons[trade.exit_reason][trade_win_loss(trade)] += 1
 
         # Duration
-        dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []}
+        dur: Dict[str, List[float]] = {'wins': [], 'draws': [], 'losses': []}
         for trade in trades:
             if trade.close_date is not None and trade.open_date is not None:
                 trade_dur = (trade.close_date - trade.open_date).total_seconds()
@@ -449,11 +449,11 @@ class RPC:
                 durations.append((trade.close_date - trade.open_date).total_seconds())
 
             if not trade.is_open:
-                profit_ratio = trade.close_profit
+                profit_ratio = trade.close_profit or 0.0
-                profit_abs = trade.close_profit_abs
+                profit_abs = trade.close_profit_abs or 0.0
                 profit_closed_coin.append(profit_abs)
                 profit_closed_ratio.append(profit_ratio)
-                if trade.close_profit >= 0:
+                if profit_ratio >= 0:
                     winning_trades += 1
                     winning_profit += profit_abs
                 else:
@@ -506,7 +506,7 @@ class RPC:
 
         trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
                                 'profit_abs': trade.close_profit_abs}
-                               for trade in trades if not trade.is_open])
+                               for trade in trades if not trade.is_open and trade.close_date])
         max_drawdown_abs = 0.0
         max_drawdown = 0.0
         if len(trades_df) > 0:
@@ -785,7 +785,8 @@ class RPC:
         # check if valid pair
 
         # check if pair already has an open pair
-        trade: Trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
+        trade: Optional[Trade] = Trade.get_trades(
+            [Trade.is_open.is_(True), Trade.pair == pair]).first()
         is_short = (order_side == SignalDirection.SHORT)
         if trade:
             is_short = trade.is_short

freqtrade/rpc/telegram.py

@@ -1055,10 +1055,14 @@ class Telegram(RPCHandler):
                     query.answer()
                     query.edit_message_text(text="Force exit canceled.")
                     return
-                trade: Trade = Trade.get_trades(trade_filter=Trade.id == trade_id).first()
+                trade: Optional[Trade] = Trade.get_trades(trade_filter=Trade.id == trade_id).first()
                 query.answer()
-                query.edit_message_text(text=f"Manually exiting Trade #{trade_id}, {trade.pair}")
+                if trade:
-                self._force_exit_action(trade_id)
+                    query.edit_message_text(
+                        text=f"Manually exiting Trade #{trade_id}, {trade.pair}")
+                    self._force_exit_action(trade_id)
+                else:
+                    query.edit_message_text(text=f"Trade {trade_id} not found.")
 
     def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection):
         if pair != 'cancel':

pyproject.toml

@@ -35,6 +35,9 @@ warn_unused_ignores = true
 exclude = [
     '^build_helpers\.py$'
 ]
+plugins = [
+  "sqlalchemy.ext.mypy.plugin"
+]
 
 [[tool.mypy.overrides]]
 module = "tests.*"

requirements.txt

@@ -5,7 +5,7 @@ pandas-ta==0.3.14b
 ccxt==2.8.54
 cryptography==39.0.1
 aiohttp==3.8.4
-SQLAlchemy==1.4.46
+SQLAlchemy==2.0.4
 python-telegram-bot==13.15
 arrow==1.2.3
 cachetools==4.2.2

tests/persistence/test_persistence.py

@@ -2440,6 +2440,7 @@ def test_select_filled_orders(fee):
 def test_order_to_ccxt(limit_buy_order_open):
 
     order = Order.parse_from_ccxt_object(limit_buy_order_open, 'mocked', 'buy')
+    order.ft_trade_id = 1
     order.query.session.add(order)
     Order.query.session.commit()
 

tests/strategy/strats/broken_strats/broken_futures_strategies.py

@@ -7,6 +7,7 @@ from datetime import datetime
 
 from pandas import DataFrame
 
+from freqtrade.persistence.trade_model import Order
 from freqtrade.strategy.interface import IStrategy
 
 
@@ -35,7 +36,7 @@ class TestStrategyImplementBuyTimeout(TestStrategyNoImplementSell):
     def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         return super().populate_exit_trend(dataframe, metadata)
 
-    def check_buy_timeout(self, pair: str, trade, order: dict,
+    def check_buy_timeout(self, pair: str, trade, order: Order,
                           current_time: datetime, **kwargs) -> bool:
         return False
 
@@ -44,6 +45,6 @@ class TestStrategyImplementSellTimeout(TestStrategyNoImplementSell):
     def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         return super().populate_exit_trend(dataframe, metadata)
 
-    def check_sell_timeout(self, pair: str, trade, order: dict,
+    def check_sell_timeout(self, pair: str, trade, order: Order,
                            current_time: datetime, **kwargs) -> bool:
         return False

tests/strategy/strats/strategy_test_v3.py

@@ -197,7 +197,7 @@ class StrategyTestV3(IStrategy):
 
         if current_profit < -0.0075:
             orders = trade.select_filled_orders(trade.entry_side)
-            return round(orders[0].cost, 0)
+            return round(orders[0].safe_cost, 0)
 
         return None
 

tests/test_freqtradebot.py

@@ -3036,6 +3036,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_
     # Order remained open for some reason (cancel failed)
     cancel_buy_order['status'] = 'open'
     cancel_order_mock = MagicMock(return_value=cancel_buy_order)
+    trade.open_order_id = 'some_open_order'
     mocker.patch(f'{EXMS}.cancel_order_with_result', cancel_order_mock)
     assert not freqtrade.handle_cancel_enter(trade, l_order, reason)
     assert log_has_re(r"Order .* for .* not cancelled.", caplog)
@@ -3231,6 +3232,7 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None:
     trade = MagicMock()
     reason = CANCEL_REASON['TIMEOUT']
     order = {'remaining': 1,
+             'id': '125',
              'amount': 1,
              'status': "open"}
     assert not freqtrade.handle_cancel_exit(trade, order, reason)