From a629d455fb534510d06551be661846daf75dd117 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Feb 2023 08:50:57 +0000 Subject: [PATCH 01/42] Bump sqlalchemy from 1.4.46 to 2.0.3 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.46 to 2.0.3. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ea0e8ecb4..3e6c66757 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ pandas-ta==0.3.14b ccxt==2.8.54 cryptography==39.0.1 aiohttp==3.8.4 -SQLAlchemy==1.4.46 +SQLAlchemy==2.0.3 python-telegram-bot==13.15 arrow==1.2.3 cachetools==4.2.2 From a553a9923ad57feda53b911fac918bae1e0516ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 08:51:00 +0000 Subject: [PATCH 02/42] Update types for pairlock --- freqtrade/persistence/pairlock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index 938cd14bc..b8af1421f 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -21,9 +21,9 @@ class PairLock(_DECL_BASE): side = Column(String(25), nullable=False, default="*") reason = Column(String(255), nullable=True) # Time the pair was locked (start time) - lock_time = Column(DateTime(), nullable=False) + lock_time: datetime = Column(DateTime(), nullable=False) # Time until the pair is locked (end time) - lock_end_time = Column(DateTime(), nullable=False, index=True) + lock_end_time: datetime = Column(DateTime(), nullable=False, index=True) active = Column(Boolean, nullable=False, default=True, index=True) From b62830031f0f70689d985d2150343fe6a6249695 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 09:56:42 +0000 Subject: [PATCH 03/42] Dummy-type query objects --- freqtrade/persistence/pairlock.py | 2 ++ freqtrade/persistence/trade_model.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index b8af1421f..2c4cfa3b4 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -13,6 +13,8 @@ class PairLock(_DECL_BASE): Pair Locks database model. """ __tablename__ = 'pairlocks' + # TODO: Properly type query. + query: Any id = Column(Integer, primary_key=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index c84fcec9e..e6f4f5b08 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -36,6 +36,8 @@ class Order(_DECL_BASE): Mirrors CCXT Order structure """ __tablename__ = 'orders' + # TODO: Properly type query. + query: Any # Uniqueness should be ensured over pair, order_id # its likely that order_id is unique per Pair on some exchanges. __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) @@ -1167,6 +1169,9 @@ class Trade(_DECL_BASE, LocalTrade): Note: Fields must be aligned with LocalTrade class """ __tablename__ = 'trades' + # TODO: Type query type throughout. + query: Any + _session: Any = None use_db: bool = True From 829e10ff87907bbcb0495c75e88a1e4e462a798f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 10:04:09 +0000 Subject: [PATCH 04/42] Improve Type for models.py --- freqtrade/persistence/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 7f851322e..38dbf212f 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -2,6 +2,7 @@ This module contains the class to persist trades into SQLite """ import logging +from typing import Any, Dict from sqlalchemy import create_engine, inspect from sqlalchemy.exc import NoSuchModuleError @@ -29,7 +30,7 @@ def init_db(db_url: str) -> None: :param db_url: Database to use :return: None """ - kwargs = {} + kwargs: Dict[str, Any] = {} if db_url == 'sqlite:///': raise OperationalException( From 9d455f58b1d643974add4b92177562a6312c77ad Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 10:04:36 +0000 Subject: [PATCH 05/42] Improve some trade model Types --- freqtrade/persistence/trade_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index e6f4f5b08..c0019eaf2 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1386,7 +1386,7 @@ class Trade(_DECL_BASE, LocalTrade): Returns List of dicts containing all Trades, including profit and trade count NOTE: Not supported in Backtesting. """ - filters = [Trade.is_open.is_(False)] + filters: List = [Trade.is_open.is_(False)] if minutes: start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes) filters.append(Trade.close_date >= start_date) @@ -1419,7 +1419,7 @@ class Trade(_DECL_BASE, LocalTrade): NOTE: Not supported in Backtesting. """ - filters = [Trade.is_open.is_(False)] + filters: List = [Trade.is_open.is_(False)] if (pair is not None): filters.append(Trade.pair == pair) @@ -1452,7 +1452,7 @@ class Trade(_DECL_BASE, LocalTrade): NOTE: Not supported in Backtesting. """ - filters = [Trade.is_open.is_(False)] + filters: List = [Trade.is_open.is_(False)] if (pair is not None): filters.append(Trade.pair == pair) @@ -1485,7 +1485,7 @@ class Trade(_DECL_BASE, LocalTrade): NOTE: Not supported in Backtesting. """ - filters = [Trade.is_open.is_(False)] + filters: List = [Trade.is_open.is_(False)] if (pair is not None): filters.append(Trade.pair == pair) From 3a9d83f86c79ccdd90a2ad250dc2af47e7e9868f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 10:05:04 +0000 Subject: [PATCH 06/42] Mypy: define sqlalchemy plugin --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6f9e5205c..71687961d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,9 @@ warn_unused_ignores = true exclude = [ '^build_helpers\.py$' ] +plugins = [ + "sqlalchemy.ext.mypy.plugin" +] [[tool.mypy.overrides]] module = "tests.*" From 41e27ba62198c228a2e06dafeb6822381ec54030 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 10:35:09 +0000 Subject: [PATCH 07/42] Enhance some type info --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 633e9dc71..ceb078669 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -633,7 +633,7 @@ class FreqtradeBot(LoggingMixin): return remaining = (trade.amount - amount) * current_exit_rate - if remaining < min_exit_stake: + if min_exit_stake and remaining < min_exit_stake: logger.info(f"Remaining amount of {remaining} would be smaller " f"than the minimum of {min_exit_stake}.") return @@ -1694,7 +1694,7 @@ class FreqtradeBot(LoggingMixin): raise DependencyException( f"Order_obj not found for {order_id}. This should not have happened.") - profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested + profit_rate: float = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) current_rate = self.exchange.get_rate( trade.pair, side='exit', is_short=trade.is_short, refresh=False) From 3c019e0e16c64a6d488baa056f3f117c51ee3e24 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Feb 2023 10:35:18 +0000 Subject: [PATCH 08/42] tentative augmented typing of Trade object --- freqtrade/freqtradebot.py | 3 +- freqtrade/persistence/trade_model.py | 72 ++++++++++++++-------------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ceb078669..adc630036 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1737,7 +1737,8 @@ class FreqtradeBot(LoggingMixin): # def update_trade_state( - self, trade: Trade, order_id: str, action_order: Optional[Dict[str, Any]] = None, + self, trade: Trade, order_id: Optional[str], + action_order: Optional[Dict[str, Any]] = None, stoploss_order: bool = False, send_msg: bool = True) -> bool: """ Checks trades with open orders and updates the amount if necessary diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index c0019eaf2..9355d2a84 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -45,34 +45,34 @@ class Order(_DECL_BASE): id = Column(Integer, primary_key=True) ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) - trade = relationship("Trade", back_populates="orders") + trade: List["Trade"] = relationship("Trade", back_populates="orders") # order_side can only be 'buy', 'sell' or 'stoploss' - ft_order_side = Column(String(25), nullable=False) - ft_pair = Column(String(25), nullable=False) - ft_is_open = Column(Boolean, nullable=False, default=True, index=True) - ft_amount = Column(Float(), nullable=False) - ft_price = Column(Float(), nullable=False) + ft_order_side: str = Column(String(25), nullable=False) + ft_pair: str = Column(String(25), nullable=False) + ft_is_open: bool = Column(Boolean, nullable=False, default=True, index=True) + ft_amount: float = Column(Float(), nullable=False) + ft_price: float = Column(Float(), nullable=False) order_id = Column(String(255), nullable=False, index=True) status = Column(String(255), nullable=True) symbol = Column(String(25), nullable=True) - order_type = Column(String(50), nullable=True) + # TODO: type: order_type type is Optional[str] + order_type: str = Column(String(50), nullable=True) side = Column(String(25), nullable=True) - price = Column(Float(), nullable=True) - average = Column(Float(), nullable=True) - amount = Column(Float(), nullable=True) - filled = Column(Float(), nullable=True) - remaining = Column(Float(), nullable=True) - cost = Column(Float(), nullable=True) - stop_price = Column(Float(), nullable=True) - order_date = Column(DateTime(), nullable=True, default=datetime.utcnow) + price: Optional[float] = Column(Float(), nullable=True) + average: Optional[float] = Column(Float(), nullable=True) + amount: Optional[float] = Column(Float(), nullable=True) + filled: Optional[float] = Column(Float(), nullable=True) + remaining: Optional[float] = Column(Float(), nullable=True) + cost: Optional[float] = Column(Float(), nullable=True) + stop_price: Optional[float] = Column(Float(), nullable=True) + order_date: datetime = Column(DateTime(), nullable=True, default=datetime.utcnow) order_filled_date = Column(DateTime(), nullable=True) order_update_date = Column(DateTime(), nullable=True) + funding_fee: Optional[float] = Column(Float(), nullable=True) - funding_fee = Column(Float(), nullable=True) - - ft_fee_base = Column(Float(), nullable=True) + ft_fee_base: Optional[float] = Column(Float(), nullable=True) @property def order_date_utc(self) -> datetime: @@ -1175,13 +1175,13 @@ class Trade(_DECL_BASE, LocalTrade): use_db: bool = True - id = Column(Integer, primary_key=True) + id: int = Column(Integer, primary_key=True) - orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", - lazy="selectin", innerjoin=True) + orders: List[Order] = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", + lazy="selectin", innerjoin=True) - exchange = Column(String(25), nullable=False) - pair = Column(String(25), nullable=False, index=True) + exchange: str = Column(String(25), nullable=False) + pair: str = Column(String(25), nullable=False, index=True) base_currency = Column(String(25), nullable=True) stake_currency = Column(String(25), nullable=True) is_open = Column(Boolean, nullable=False, default=True, index=True) @@ -1192,21 +1192,23 @@ class Trade(_DECL_BASE, LocalTrade): fee_close_cost = Column(Float(), nullable=True) fee_close_currency = Column(String(25), nullable=True) open_rate: float = Column(Float()) - open_rate_requested = Column(Float()) + open_rate_requested: float = Column(Float()) # open_trade_value - calculated via _calc_open_trade_value open_trade_value = Column(Float()) close_rate: Optional[float] = Column(Float()) - close_rate_requested = Column(Float()) - realized_profit = Column(Float(), default=0.0) + close_rate_requested: Optional[float] = Column(Float()) + # TODO: is the below type really correct? + realized_profit: float = Column(Float(), default=0.0) close_profit = Column(Float()) - close_profit_abs = Column(Float()) - stake_amount = Column(Float(), nullable=False) - max_stake_amount = Column(Float()) - amount = Column(Float()) - amount_requested = Column(Float()) + close_profit_abs: Optional[float] = Column(Float()) + stake_amount: float = Column(Float(), nullable=False) + max_stake_amount: Optional[float] = Column(Float()) + amount: float = Column(Float()) + amount_requested: Optional[float] = Column(Float()) open_date = Column(DateTime(), nullable=False, default=datetime.utcnow) close_date = Column(DateTime()) - open_order_id = Column(String(255)) + # TODO: open_order_id type should be Optional[str] + open_order_id: str = Column(String(255)) # absolute value of the stop loss stop_loss = Column(Float(), nullable=True, default=0.0) # percentage value of the stop loss @@ -1236,15 +1238,15 @@ class Trade(_DECL_BASE, LocalTrade): contract_size = Column(Float(), nullable=True) # Leverage trading properties - leverage = Column(Float(), nullable=True, default=1.0) - is_short = Column(Boolean, nullable=False, default=False) + leverage: float = Column(Float(), nullable=True, default=1.0) + is_short: bool = Column(Boolean, nullable=False, default=False) liquidation_price = Column(Float(), nullable=True) # Margin Trading Properties interest_rate = Column(Float(), nullable=False, default=0.0) # Futures properties - funding_fees = Column(Float(), nullable=True, default=None) + funding_fees: Optional[float] = Column(Float(), nullable=True, default=None) def __init__(self, **kwargs): super().__init__(**kwargs) From 39a658eac22a70726d4b7bfa8139ee423325731e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 07:04:45 +0000 Subject: [PATCH 09/42] Update DeclarativeBase --- freqtrade/persistence/base.py | 7 +++---- freqtrade/persistence/models.py | 6 +++--- freqtrade/persistence/pairlock.py | 4 ++-- freqtrade/persistence/trade_model.py | 6 +++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/freqtrade/persistence/base.py b/freqtrade/persistence/base.py index fb2d561e1..2fed715d7 100644 --- a/freqtrade/persistence/base.py +++ b/freqtrade/persistence/base.py @@ -1,7 +1,6 @@ -from typing import Any - -from sqlalchemy.orm import declarative_base +from sqlalchemy.orm import DeclarativeBase -_DECL_BASE: Any = declarative_base() +class ModelBase(DeclarativeBase): + pass diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 38dbf212f..9f90f8a74 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException -from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.base import ModelBase from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade @@ -60,5 +60,5 @@ def init_db(db_url: str) -> None: PairLock.query = Trade._session.query_property() previous_tables = inspect(engine).get_table_names() - _DECL_BASE.metadata.create_all(engine) - check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables) + ModelBase.metadata.create_all(engine) + check_migrate(engine, decl_base=ModelBase, previous_tables=previous_tables) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index 2c4cfa3b4..e1f659b0a 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -5,10 +5,10 @@ from sqlalchemy import Boolean, Column, DateTime, Integer, String, or_ from sqlalchemy.orm import Query from freqtrade.constants import DATETIME_PRINT_FORMAT -from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.base import ModelBase -class PairLock(_DECL_BASE): +class PairLock(ModelBase): """ Pair Locks database model. """ diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 9355d2a84..ebec36e81 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -17,14 +17,14 @@ from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import amount_to_contract_precision, price_to_precision from freqtrade.leverage import interest -from freqtrade.persistence.base import _DECL_BASE +from freqtrade.persistence.base import ModelBase from freqtrade.util import FtPrecise logger = logging.getLogger(__name__) -class Order(_DECL_BASE): +class Order(ModelBase): """ Order database model Keeps a record of all orders placed on the exchange @@ -1161,7 +1161,7 @@ class LocalTrade(): logger.info(f"New stoploss: {trade.stop_loss}.") -class Trade(_DECL_BASE, LocalTrade): +class Trade(ModelBase, LocalTrade): """ Trade database model. Also handles updating and querying trades From 0bd9b00132623db0499325545ef210ed0512e5c5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 07:05:57 +0000 Subject: [PATCH 10/42] Pairlock to mappedColumn --- freqtrade/persistence/pairlock.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index e1f659b0a..b721cceac 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -1,8 +1,8 @@ from datetime import datetime, timezone from typing import Any, Dict, Optional -from sqlalchemy import Boolean, Column, DateTime, Integer, String, or_ -from sqlalchemy.orm import Query +from sqlalchemy import Boolean, DateTime, Integer, String, or_ +from sqlalchemy.orm import Query, mapped_column from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.persistence.base import ModelBase @@ -16,18 +16,18 @@ class PairLock(ModelBase): # TODO: Properly type query. query: Any - id = Column(Integer, primary_key=True) + id = mapped_column(Integer, primary_key=True) - pair = Column(String(25), nullable=False, index=True) + pair = mapped_column(String(25), nullable=False, index=True) # lock direction - long, short or * (for both) - side = Column(String(25), nullable=False, default="*") - reason = Column(String(255), nullable=True) + side = mapped_column(String(25), nullable=False, default="*") + reason = mapped_column(String(255), nullable=True) # Time the pair was locked (start time) - lock_time: datetime = Column(DateTime(), nullable=False) + lock_time: datetime = mapped_column(DateTime(), nullable=False) # Time until the pair is locked (end time) - lock_end_time: datetime = Column(DateTime(), nullable=False, index=True) + lock_end_time: datetime = mapped_column(DateTime(), nullable=False, index=True) - active = Column(Boolean, nullable=False, default=True, index=True) + active = mapped_column(Boolean, nullable=False, default=True, index=True) def __repr__(self): lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) From 98791752a9c39827385bd0c20509f560bbff88d4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 07:07:08 +0000 Subject: [PATCH 11/42] Update TradeModels to mapped_column --- freqtrade/persistence/trade_model.py | 152 +++++++++++++-------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ebec36e81..e8f78661e 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -7,9 +7,9 @@ from datetime import datetime, timedelta, timezone from math import isclose from typing import Any, Dict, List, Optional -from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String, +from sqlalchemy import (Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func) -from sqlalchemy.orm import Query, lazyload, relationship +from sqlalchemy.orm import Query, lazyload, mapped_column, relationship from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) @@ -42,37 +42,37 @@ class Order(ModelBase): # its likely that order_id is unique per Pair on some exchanges. __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) - id = Column(Integer, primary_key=True) - ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) + id = mapped_column(Integer, primary_key=True) + ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True) trade: List["Trade"] = relationship("Trade", back_populates="orders") # order_side can only be 'buy', 'sell' or 'stoploss' - ft_order_side: str = Column(String(25), nullable=False) - ft_pair: str = Column(String(25), nullable=False) - ft_is_open: bool = Column(Boolean, nullable=False, default=True, index=True) - ft_amount: float = Column(Float(), nullable=False) - ft_price: float = Column(Float(), nullable=False) + ft_order_side: str = mapped_column(String(25), nullable=False) + ft_pair: str = mapped_column(String(25), nullable=False) + ft_is_open: bool = mapped_column(Boolean, nullable=False, default=True, index=True) + ft_amount: float = mapped_column(Float(), nullable=False) + ft_price: float = mapped_column(Float(), nullable=False) - order_id = Column(String(255), nullable=False, index=True) - status = Column(String(255), nullable=True) - symbol = Column(String(25), nullable=True) + order_id = mapped_column(String(255), nullable=False, index=True) + status = mapped_column(String(255), nullable=True) + symbol = mapped_column(String(25), nullable=True) # TODO: type: order_type type is Optional[str] - order_type: str = Column(String(50), nullable=True) - side = Column(String(25), nullable=True) - price: Optional[float] = Column(Float(), nullable=True) - average: Optional[float] = Column(Float(), nullable=True) - amount: Optional[float] = Column(Float(), nullable=True) - filled: Optional[float] = Column(Float(), nullable=True) - remaining: Optional[float] = Column(Float(), nullable=True) - cost: Optional[float] = Column(Float(), nullable=True) - stop_price: Optional[float] = Column(Float(), nullable=True) - order_date: datetime = Column(DateTime(), nullable=True, default=datetime.utcnow) - order_filled_date = Column(DateTime(), nullable=True) - order_update_date = Column(DateTime(), nullable=True) - funding_fee: Optional[float] = Column(Float(), nullable=True) + order_type: str = mapped_column(String(50), nullable=True) + side = mapped_column(String(25), nullable=True) + price: Optional[float] = mapped_column(Float(), nullable=True) + average: Optional[float] = mapped_column(Float(), nullable=True) + amount: Optional[float] = mapped_column(Float(), nullable=True) + filled: Optional[float] = mapped_column(Float(), nullable=True) + remaining: Optional[float] = mapped_column(Float(), nullable=True) + cost: Optional[float] = mapped_column(Float(), nullable=True) + stop_price: Optional[float] = mapped_column(Float(), nullable=True) + order_date: datetime = mapped_column(DateTime(), nullable=True, default=datetime.utcnow) + order_filled_date = mapped_column(DateTime(), nullable=True) + order_update_date = mapped_column(DateTime(), nullable=True) + funding_fee: Optional[float] = mapped_column(Float(), nullable=True) - ft_fee_base: Optional[float] = Column(Float(), nullable=True) + ft_fee_base: Optional[float] = mapped_column(Float(), nullable=True) @property def order_date_utc(self) -> datetime: @@ -1175,78 +1175,78 @@ class Trade(ModelBase, LocalTrade): use_db: bool = True - id: int = Column(Integer, primary_key=True) + id: int = mapped_column(Integer, primary_key=True) orders: List[Order] = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", innerjoin=True) - exchange: str = Column(String(25), nullable=False) - pair: str = Column(String(25), nullable=False, index=True) - base_currency = Column(String(25), nullable=True) - stake_currency = Column(String(25), nullable=True) - is_open = Column(Boolean, nullable=False, default=True, index=True) - fee_open = Column(Float(), nullable=False, default=0.0) - fee_open_cost = Column(Float(), nullable=True) - fee_open_currency = Column(String(25), nullable=True) - fee_close = Column(Float(), nullable=False, default=0.0) - fee_close_cost = Column(Float(), nullable=True) - fee_close_currency = Column(String(25), nullable=True) - open_rate: float = Column(Float()) - open_rate_requested: float = Column(Float()) + exchange: str = mapped_column(String(25), nullable=False) + pair: str = mapped_column(String(25), nullable=False, index=True) + base_currency = mapped_column(String(25), nullable=True) + stake_currency = mapped_column(String(25), nullable=True) + is_open = mapped_column(Boolean, nullable=False, default=True, index=True) + fee_open = mapped_column(Float(), nullable=False, default=0.0) + fee_open_cost = mapped_column(Float(), nullable=True) + fee_open_currency = mapped_column(String(25), nullable=True) + fee_close = mapped_column(Float(), nullable=False, default=0.0) + fee_close_cost = mapped_column(Float(), nullable=True) + fee_close_currency = mapped_column(String(25), nullable=True) + open_rate: float = mapped_column(Float()) + open_rate_requested: float = mapped_column(Float()) # open_trade_value - calculated via _calc_open_trade_value - open_trade_value = Column(Float()) - close_rate: Optional[float] = Column(Float()) - close_rate_requested: Optional[float] = Column(Float()) + open_trade_value = mapped_column(Float()) + close_rate: Optional[float] = mapped_column(Float()) + close_rate_requested: Optional[float] = mapped_column(Float()) # TODO: is the below type really correct? - realized_profit: float = Column(Float(), default=0.0) - close_profit = Column(Float()) - close_profit_abs: Optional[float] = Column(Float()) - stake_amount: float = Column(Float(), nullable=False) - max_stake_amount: Optional[float] = Column(Float()) - amount: float = Column(Float()) - amount_requested: Optional[float] = Column(Float()) - open_date = Column(DateTime(), nullable=False, default=datetime.utcnow) - close_date = Column(DateTime()) + realized_profit: float = mapped_column(Float(), default=0.0) + close_profit = mapped_column(Float()) + close_profit_abs: Optional[float] = mapped_column(Float()) + stake_amount: float = mapped_column(Float(), nullable=False) + max_stake_amount: Optional[float] = mapped_column(Float()) + amount: float = mapped_column(Float()) + amount_requested: Optional[float] = mapped_column(Float()) + open_date = mapped_column(DateTime(), nullable=False, default=datetime.utcnow) + close_date = mapped_column(DateTime()) # TODO: open_order_id type should be Optional[str] - open_order_id: str = Column(String(255)) + open_order_id: str = mapped_column(String(255)) # absolute value of the stop loss - stop_loss = Column(Float(), nullable=True, default=0.0) + stop_loss = mapped_column(Float(), nullable=True, default=0.0) # percentage value of the stop loss - stop_loss_pct = Column(Float(), nullable=True) + stop_loss_pct = mapped_column(Float(), nullable=True) # absolute value of the initial stop loss - initial_stop_loss = Column(Float(), nullable=True, default=0.0) + initial_stop_loss = mapped_column(Float(), nullable=True, default=0.0) # percentage value of the initial stop loss - initial_stop_loss_pct = Column(Float(), nullable=True) + initial_stop_loss_pct = mapped_column(Float(), nullable=True) # stoploss order id which is on exchange - stoploss_order_id = Column(String(255), nullable=True, index=True) + stoploss_order_id = mapped_column(String(255), nullable=True, index=True) # last update time of the stoploss order on exchange - stoploss_last_update = Column(DateTime(), nullable=True) + stoploss_last_update = mapped_column(DateTime(), nullable=True) # absolute value of the highest reached price - max_rate = Column(Float(), nullable=True, default=0.0) + max_rate = mapped_column(Float(), nullable=True, default=0.0) # Lowest price reached - min_rate = Column(Float(), nullable=True) - exit_reason = Column(String(100), nullable=True) - exit_order_status = Column(String(100), nullable=True) - strategy = Column(String(100), nullable=True) - enter_tag = Column(String(100), nullable=True) - timeframe = Column(Integer, nullable=True) + min_rate = mapped_column(Float(), nullable=True) + exit_reason = mapped_column(String(100), nullable=True) + exit_order_status = mapped_column(String(100), nullable=True) + strategy = mapped_column(String(100), nullable=True) + enter_tag = mapped_column(String(100), nullable=True) + timeframe = mapped_column(Integer, nullable=True) - trading_mode = Column(Enum(TradingMode), nullable=True) - amount_precision = Column(Float(), nullable=True) - price_precision = Column(Float(), nullable=True) - precision_mode = Column(Integer, nullable=True) - contract_size = Column(Float(), nullable=True) + trading_mode = mapped_column(Enum(TradingMode), nullable=True) + amount_precision = mapped_column(Float(), nullable=True) + price_precision = mapped_column(Float(), nullable=True) + precision_mode = mapped_column(Integer, nullable=True) + contract_size = mapped_column(Float(), nullable=True) # Leverage trading properties - leverage: float = Column(Float(), nullable=True, default=1.0) - is_short: bool = Column(Boolean, nullable=False, default=False) - liquidation_price = Column(Float(), nullable=True) + leverage: float = mapped_column(Float(), nullable=True, default=1.0) + is_short: bool = mapped_column(Boolean, nullable=False, default=False) + liquidation_price = mapped_column(Float(), nullable=True) # Margin Trading Properties - interest_rate = Column(Float(), nullable=False, default=0.0) + interest_rate = mapped_column(Float(), nullable=False, default=0.0) # Futures properties - funding_fees: Optional[float] = Column(Float(), nullable=True, default=None) + funding_fees: Optional[float] = mapped_column(Float(), nullable=True, default=None) def __init__(self, **kwargs): super().__init__(**kwargs) From 13b1a3e737689effd4dc14dcc1a60abda309375b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 07:14:07 +0000 Subject: [PATCH 12/42] Properly pairlock columns using mapped --- freqtrade/persistence/pairlock.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index b721cceac..2c81beb85 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -1,8 +1,8 @@ from datetime import datetime, timezone from typing import Any, Dict, Optional -from sqlalchemy import Boolean, DateTime, Integer, String, or_ -from sqlalchemy.orm import Query, mapped_column +from sqlalchemy import String, or_ +from sqlalchemy.orm import Mapped, Query, mapped_column from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.persistence.base import ModelBase @@ -16,20 +16,20 @@ class PairLock(ModelBase): # TODO: Properly type query. query: Any - id = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(primary_key=True) - pair = mapped_column(String(25), nullable=False, index=True) + pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) # lock direction - long, short or * (for both) - side = mapped_column(String(25), nullable=False, default="*") - reason = mapped_column(String(255), nullable=True) + side: Mapped[str] = mapped_column(String(25), nullable=False, default="*") + reason: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # Time the pair was locked (start time) - lock_time: datetime = mapped_column(DateTime(), nullable=False) + lock_time: Mapped[datetime] = mapped_column(nullable=False) # Time until the pair is locked (end time) - lock_end_time: datetime = mapped_column(DateTime(), nullable=False, index=True) + lock_end_time: Mapped[datetime] = mapped_column(nullable=False, index=True) - active = mapped_column(Boolean, nullable=False, default=True, index=True) + active: Mapped[bool] = mapped_column(nullable=False, default=True, index=True) - def __repr__(self): + def __repr__(self) -> str: lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT) return ( From bb116456a957defcb5fe422b91094cd7dd640a51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 07:17:56 +0000 Subject: [PATCH 13/42] Update Types for Order object --- freqtrade/persistence/trade_model.py | 52 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index e8f78661e..c4bfc7b1f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func) -from sqlalchemy.orm import Query, lazyload, mapped_column, relationship +from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) @@ -42,37 +42,37 @@ class Order(ModelBase): # its likely that order_id is unique per Pair on some exchanges. __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) - id = mapped_column(Integer, primary_key=True) - ft_trade_id = mapped_column(Integer, ForeignKey('trades.id'), index=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + ft_trade_id: Mapped[int] = mapped_column(Integer, ForeignKey('trades.id'), index=True) - trade: List["Trade"] = relationship("Trade", back_populates="orders") + trade: Mapped[List["Trade"]] = relationship("Trade", back_populates="orders") # order_side can only be 'buy', 'sell' or 'stoploss' - ft_order_side: str = mapped_column(String(25), nullable=False) - ft_pair: str = mapped_column(String(25), nullable=False) - ft_is_open: bool = mapped_column(Boolean, nullable=False, default=True, index=True) - ft_amount: float = mapped_column(Float(), nullable=False) - ft_price: float = mapped_column(Float(), nullable=False) + ft_order_side: Mapped[str] = mapped_column(String(25), nullable=False) + ft_pair: Mapped[str] = mapped_column(String(25), nullable=False) + ft_is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True) + ft_amount: Mapped[float] = mapped_column(Float(), nullable=False) + ft_price: Mapped[float] = mapped_column(Float(), nullable=False) - order_id = mapped_column(String(255), nullable=False, index=True) - status = mapped_column(String(255), nullable=True) - symbol = mapped_column(String(25), nullable=True) + order_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + status: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + symbol: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) # TODO: type: order_type type is Optional[str] - order_type: str = mapped_column(String(50), nullable=True) - side = mapped_column(String(25), nullable=True) - price: Optional[float] = mapped_column(Float(), nullable=True) - average: Optional[float] = mapped_column(Float(), nullable=True) - amount: Optional[float] = mapped_column(Float(), nullable=True) - filled: Optional[float] = mapped_column(Float(), nullable=True) - remaining: Optional[float] = mapped_column(Float(), nullable=True) - cost: Optional[float] = mapped_column(Float(), nullable=True) - stop_price: Optional[float] = mapped_column(Float(), nullable=True) - order_date: datetime = mapped_column(DateTime(), nullable=True, default=datetime.utcnow) - order_filled_date = mapped_column(DateTime(), nullable=True) - order_update_date = mapped_column(DateTime(), nullable=True) - funding_fee: Optional[float] = mapped_column(Float(), nullable=True) + order_type: Mapped[str] = mapped_column(String(50), nullable=True) + side: Mapped[str] = mapped_column(String(25), nullable=True) + price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + average: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + amount: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + filled: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + remaining: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + stop_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + order_date: Mapped[datetime] = mapped_column(nullable=True, default=datetime.utcnow) + order_filled_date: Mapped[Optional[datetime]] = mapped_column(nullable=True) + order_update_date: Mapped[Optional[datetime]] = mapped_column(nullable=True) + funding_fee: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - ft_fee_base: Optional[float] = mapped_column(Float(), nullable=True) + ft_fee_base: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) @property def order_date_utc(self) -> datetime: From 491f49388cc1387e1aa6716f1181c150a396670b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 07:27:57 +0000 Subject: [PATCH 14/42] "Mapped" for trade_model --- freqtrade/persistence/trade_model.py | 95 ++++++++++++++-------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index c4bfc7b1f..9db79fd8b 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -215,7 +215,7 @@ class Order(ModelBase): # Assumes backtesting will use date_last_filled_utc to calculate future funding fees. self.funding_fee = trade.funding_fees - if (self.ft_order_side == trade.entry_side): + if (self.ft_order_side == trade.entry_side and self.price): trade.open_rate = self.price trade.recalc_trade_from_orders() trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True) @@ -1175,78 +1175,77 @@ class Trade(ModelBase, LocalTrade): use_db: bool = True - id: int = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) - orders: List[Order] = relationship("Order", order_by="Order.id", cascade="all, delete-orphan", - lazy="selectin", innerjoin=True) + orders: Mapped[List[Order]] = relationship( + "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", + innerjoin=True) - exchange: str = mapped_column(String(25), nullable=False) - pair: str = mapped_column(String(25), nullable=False, index=True) + exchange: Mapped[str] = mapped_column(String(25), nullable=False) + pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) base_currency = mapped_column(String(25), nullable=True) stake_currency = mapped_column(String(25), nullable=True) - is_open = mapped_column(Boolean, nullable=False, default=True, index=True) - fee_open = mapped_column(Float(), nullable=False, default=0.0) - fee_open_cost = mapped_column(Float(), nullable=True) - fee_open_currency = mapped_column(String(25), nullable=True) - fee_close = mapped_column(Float(), nullable=False, default=0.0) - fee_close_cost = mapped_column(Float(), nullable=True) - fee_close_currency = mapped_column(String(25), nullable=True) - open_rate: float = mapped_column(Float()) + is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True) + fee_open: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0) + fee_open_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + fee_open_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) + fee_close: Mapped[Optional[float]] = mapped_column(Float(), nullable=False, default=0.0) + fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + fee_close_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) + open_rate: Mapped[float] = mapped_column(Float()) open_rate_requested: float = mapped_column(Float()) # open_trade_value - calculated via _calc_open_trade_value open_trade_value = mapped_column(Float()) close_rate: Optional[float] = mapped_column(Float()) close_rate_requested: Optional[float] = mapped_column(Float()) - # TODO: is the below type really correct? - realized_profit: float = mapped_column(Float(), default=0.0) - close_profit = mapped_column(Float()) - close_profit_abs: Optional[float] = mapped_column(Float()) - stake_amount: float = mapped_column(Float(), nullable=False) - max_stake_amount: Optional[float] = mapped_column(Float()) - amount: float = mapped_column(Float()) - amount_requested: Optional[float] = mapped_column(Float()) - open_date = mapped_column(DateTime(), nullable=False, default=datetime.utcnow) - close_date = mapped_column(DateTime()) - # TODO: open_order_id type should be Optional[str] - open_order_id: str = mapped_column(String(255)) + realized_profit: Mapped[float] = mapped_column(Float(), default=0.0) + close_profit: Mapped[Optional[float]] = mapped_column(Float()) + close_profit_abs: Mapped[Optional[float]] = mapped_column(Float()) + stake_amount: Mapped[float] = mapped_column(Float(), nullable=False) + max_stake_amount: Mapped[Optional[float]] = mapped_column(Float()) + amount: Mapped[float] = mapped_column(Float()) + amount_requested: Mapped[Optional[float]] = mapped_column(Float()) + open_date: Mapped[datetime] = mapped_column(nullable=False, default=datetime.utcnow) + close_date: Mapped[Optional[datetime]] = mapped_column() + open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # absolute value of the stop loss - stop_loss = mapped_column(Float(), nullable=True, default=0.0) + stop_loss: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=0.0) # percentage value of the stop loss - stop_loss_pct = mapped_column(Float(), nullable=True) + stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # absolute value of the initial stop loss - initial_stop_loss = mapped_column(Float(), nullable=True, default=0.0) + initial_stop_loss: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=0.0) # percentage value of the initial stop loss - initial_stop_loss_pct = mapped_column(Float(), nullable=True) + initial_stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # stoploss order id which is on exchange - stoploss_order_id = mapped_column(String(255), nullable=True, index=True) + stoploss_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True) # last update time of the stoploss order on exchange - stoploss_last_update = mapped_column(DateTime(), nullable=True) + stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True) # absolute value of the highest reached price - max_rate = mapped_column(Float(), nullable=True, default=0.0) + max_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=0.0) # Lowest price reached - min_rate = mapped_column(Float(), nullable=True) - exit_reason = mapped_column(String(100), nullable=True) - exit_order_status = mapped_column(String(100), nullable=True) - strategy = mapped_column(String(100), nullable=True) - enter_tag = mapped_column(String(100), nullable=True) - timeframe = mapped_column(Integer, nullable=True) + min_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + exit_reason: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + exit_order_status: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + strategy: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + enter_tag: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + timeframe: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) trading_mode = mapped_column(Enum(TradingMode), nullable=True) - amount_precision = mapped_column(Float(), nullable=True) - price_precision = mapped_column(Float(), nullable=True) - precision_mode = mapped_column(Integer, nullable=True) - contract_size = mapped_column(Float(), nullable=True) + amount_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + price_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + precision_mode: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + contract_size: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # Leverage trading properties - leverage: float = mapped_column(Float(), nullable=True, default=1.0) - is_short: bool = mapped_column(Boolean, nullable=False, default=False) - liquidation_price = mapped_column(Float(), nullable=True) + leverage: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=1.0) + is_short: Mapped[bool] = mapped_column(nullable=False, default=False) + liquidation_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # Margin Trading Properties - interest_rate = mapped_column(Float(), nullable=False, default=0.0) + interest_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=False, default=0.0) # Futures properties - funding_fees: Optional[float] = mapped_column(Float(), nullable=True, default=None) + funding_fees: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=None) def __init__(self, **kwargs): super().__init__(**kwargs) From 47b66f322076ac22b986bac0930016adcf3d66bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 08:40:34 +0000 Subject: [PATCH 15/42] More fun with types --- freqtrade/persistence/trade_model.py | 10 +++++----- freqtrade/rpc/rpc.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 9db79fd8b..be296c52c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1183,8 +1183,8 @@ class Trade(ModelBase, LocalTrade): exchange: Mapped[str] = mapped_column(String(25), nullable=False) pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) - base_currency = mapped_column(String(25), nullable=True) - stake_currency = mapped_column(String(25), nullable=True) + base_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) + stake_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True) fee_open: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0) fee_open_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) @@ -1209,7 +1209,7 @@ class Trade(ModelBase, LocalTrade): close_date: Mapped[Optional[datetime]] = mapped_column() open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # absolute value of the stop loss - stop_loss: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=0.0) + stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0) # percentage value of the stop loss stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # absolute value of the initial stop loss @@ -1230,14 +1230,14 @@ class Trade(ModelBase, LocalTrade): enter_tag: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) timeframe: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) - trading_mode = mapped_column(Enum(TradingMode), nullable=True) + trading_mode: Mapped[TradingMode] = mapped_column(Enum(TradingMode), nullable=True) amount_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) price_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) precision_mode: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) contract_size: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # Leverage trading properties - leverage: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=1.0) + leverage: Mapped[float] = mapped_column(Float(), nullable=True, default=1.0) is_short: Mapped[bool] = mapped_column(nullable=False, default=False) liquidation_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 82f892101..0c1a8fbbd 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -506,7 +506,7 @@ class RPC: trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT), 'profit_abs': trade.close_profit_abs} - for trade in trades if not trade.is_open]) + for trade in trades if not trade.is_open and trade.close_date]) max_drawdown_abs = 0.0 max_drawdown = 0.0 if len(trades_df) > 0: From e59eaf33e09f2b613c44f3ac67a326140c8b6176 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 08:51:32 +0000 Subject: [PATCH 16/42] Update _session to session --- freqtrade/commands/db_commands.py | 2 +- freqtrade/persistence/models.py | 10 ++++++---- tests/persistence/test_migrations.py | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index c424016b1..b4997582d 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -20,7 +20,7 @@ def start_convert_db(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) init_db(config['db_url']) - session_target = Trade._session + session_target = Trade.session init_db(config['db_url_from']) logger.info("Starting db migration.") diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 9f90f8a74..f4058b4eb 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -54,10 +54,12 @@ def init_db(db_url: str) -> None: # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope # Scoped sessions proxy requests to the appropriate thread-local session. # We should use the scoped_session object - not a seperately initialized version - Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=False)) - Trade.query = Trade._session.query_property() - Order.query = Trade._session.query_property() - PairLock.query = Trade._session.query_property() + Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=False)) + Order.session = Trade.session + PairLock.session = Trade.session + Trade.query = Trade.session.query_property() + Order.query = Trade.session.query_property() + PairLock.query = Trade.session.query_property() previous_tables = inspect(engine).get_table_names() ModelBase.metadata.create_all(engine) diff --git a/tests/persistence/test_migrations.py b/tests/persistence/test_migrations.py index 2a6959d58..d49b7d207 100644 --- a/tests/persistence/test_migrations.py +++ b/tests/persistence/test_migrations.py @@ -22,7 +22,7 @@ def test_init_create_session(default_conf): # Check if init create a session init_db(default_conf['db_url']) assert hasattr(Trade, '_session') - assert 'scoped_session' in type(Trade._session).__name__ + assert 'scoped_session' in type(Trade.session).__name__ def test_init_custom_db_url(default_conf, tmpdir): @@ -34,7 +34,7 @@ def test_init_custom_db_url(default_conf, tmpdir): init_db(default_conf['db_url']) assert Path(filename).is_file() - r = Trade._session.execute(text("PRAGMA journal_mode")) + r = Trade.session.execute(text("PRAGMA journal_mode")) assert r.first() == ('wal',) From 608a7c2d384e588233d92e37c795e24c7689c0f8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 09:02:57 +0000 Subject: [PATCH 17/42] Add safe_close_rate --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/persistence/trade_model.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index adc630036..0bed27cb9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1639,7 +1639,7 @@ class FreqtradeBot(LoggingMixin): profit = trade.calc_profit(rate=order_rate, amount=amount, open_rate=trade.open_rate) profit_ratio = trade.calc_profit_ratio(order_rate, amount, trade.open_rate) else: - order_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested + order_rate = trade.safe_close_rate profit = trade.calc_profit(rate=order_rate) + (0.0 if fill else trade.realized_profit) profit_ratio = trade.calc_profit_ratio(order_rate) amount = trade.amount @@ -1694,7 +1694,7 @@ class FreqtradeBot(LoggingMixin): raise DependencyException( f"Order_obj not found for {order_id}. This should not have happened.") - profit_rate: float = trade.close_rate if trade.close_rate else trade.close_rate_requested + profit_rate: float = trade.safe_close_rate profit_trade = trade.calc_profit(rate=profit_rate) current_rate = self.exchange.get_rate( trade.pair, side='exit', is_short=trade.is_short, refresh=False) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index be296c52c..091152196 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1065,6 +1065,10 @@ class LocalTrade(): """ DEPRECATED! Please use exit_reason instead.""" return self.exit_reason + @property + def safe_close_rate(self) -> float: + return self.close_rate or self.close_rate_requested or 0.0 + @staticmethod def get_trades_proxy(*, pair: Optional[str] = None, is_open: Optional[bool] = None, open_date: Optional[datetime] = None, From 65a5cf64df91f51e5c82bf0fecc8821fa59cd717 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Feb 2023 09:03:32 +0000 Subject: [PATCH 18/42] Re-type session --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 091152196..b3ccf7949 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1175,7 +1175,7 @@ class Trade(ModelBase, LocalTrade): __tablename__ = 'trades' # TODO: Type query type throughout. query: Any - _session: Any = None + session: Any = None use_db: bool = True From 101d9ab87f6208375fc1e990dff38ed275e4cbce Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Feb 2023 19:51:25 +0100 Subject: [PATCH 19/42] Improvements - tests runnable again --- freqtrade/persistence/pairlock.py | 4 ++-- freqtrade/persistence/trade_model.py | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index 2c81beb85..f8c0586a4 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -from typing import Any, Dict, Optional +from typing import Any, ClassVar, Dict, Optional from sqlalchemy import String, or_ from sqlalchemy.orm import Mapped, Query, mapped_column @@ -14,7 +14,7 @@ class PairLock(ModelBase): """ __tablename__ = 'pairlocks' # TODO: Properly type query. - query: Any + query: ClassVar[Any] id: Mapped[int] = mapped_column(primary_key=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index b3ccf7949..4b7a09cbe 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -5,10 +5,9 @@ import logging from collections import defaultdict from datetime import datetime, timedelta, timezone from math import isclose -from typing import Any, Dict, List, Optional +from typing import Any, ClassVar, Dict, List, Optional -from sqlalchemy import (Boolean, DateTime, Enum, Float, ForeignKey, Integer, String, - UniqueConstraint, desc, func) +from sqlalchemy import Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, @@ -37,7 +36,7 @@ class Order(ModelBase): """ __tablename__ = 'orders' # TODO: Properly type query. - query: Any + query: ClassVar[Any] # Uniqueness should be ensured over pair, order_id # its likely that order_id is unique per Pair on some exchanges. __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) @@ -1174,8 +1173,8 @@ class Trade(ModelBase, LocalTrade): """ __tablename__ = 'trades' # TODO: Type query type throughout. - query: Any - session: Any = None + query: ClassVar[Any] + session: ClassVar[Any] = None use_db: bool = True @@ -1197,11 +1196,11 @@ class Trade(ModelBase, LocalTrade): fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) fee_close_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) open_rate: Mapped[float] = mapped_column(Float()) - open_rate_requested: float = mapped_column(Float()) + open_rate_requested: Mapped[float] = mapped_column(Float()) # open_trade_value - calculated via _calc_open_trade_value open_trade_value = mapped_column(Float()) - close_rate: Optional[float] = mapped_column(Float()) - close_rate_requested: Optional[float] = mapped_column(Float()) + close_rate: Mapped[Optional[float]] = mapped_column(Float()) + close_rate_requested: Mapped[Optional[float]] = mapped_column(Float()) realized_profit: Mapped[float] = mapped_column(Float(), default=0.0) close_profit: Mapped[Optional[float]] = mapped_column(Float()) close_profit_abs: Mapped[Optional[float]] = mapped_column(Float()) From 0691bbaad9aeda6fdcc1ec9a9c75edf358974931 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Feb 2023 20:18:46 +0100 Subject: [PATCH 20/42] Update some db types --- freqtrade/persistence/trade_model.py | 2 +- tests/persistence/test_migrations.py | 2 +- tests/persistence/test_persistence.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 4b7a09cbe..abb989940 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1196,7 +1196,7 @@ class Trade(ModelBase, LocalTrade): fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) fee_close_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) open_rate: Mapped[float] = mapped_column(Float()) - open_rate_requested: Mapped[float] = mapped_column(Float()) + open_rate_requested: Mapped[float] = mapped_column(Float(), nullable=True) # open_trade_value - calculated via _calc_open_trade_value open_trade_value = mapped_column(Float()) close_rate: Mapped[Optional[float]] = mapped_column(Float()) diff --git a/tests/persistence/test_migrations.py b/tests/persistence/test_migrations.py index d49b7d207..5254164c1 100644 --- a/tests/persistence/test_migrations.py +++ b/tests/persistence/test_migrations.py @@ -21,7 +21,7 @@ spot, margin, futures = TradingMode.SPOT, TradingMode.MARGIN, TradingMode.FUTURE def test_init_create_session(default_conf): # Check if init create a session init_db(default_conf['db_url']) - assert hasattr(Trade, '_session') + assert hasattr(Trade, 'session') assert 'scoped_session' in type(Trade.session).__name__ diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index d06f05179..6d907ccf0 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2440,6 +2440,7 @@ def test_select_filled_orders(fee): def test_order_to_ccxt(limit_buy_order_open): order = Order.parse_from_ccxt_object(limit_buy_order_open, 'mocked', 'buy') + order.ft_trade_id = 1 order.query.session.add(order) Order.query.session.commit() From f6b3998bbda8c8ac22e7c13f40b2dfe40217212e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Feb 2023 20:23:04 +0100 Subject: [PATCH 21/42] Fix backtesting type incompatibilities --- freqtrade/freqtradebot.py | 3 ++- freqtrade/optimize/backtesting.py | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0bed27cb9..ad0628c59 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1330,7 +1330,8 @@ class FreqtradeBot(LoggingMixin): # place new order only if new price is supplied self.execute_entry( pair=trade.pair, - stake_amount=(order_obj.remaining * order_obj.price / trade.leverage), + stake_amount=( + order_obj.safe_remaining * order_obj.safe_price / trade.leverage), price=adjusted_entry_price, trade=trade, is_short=trade.is_short, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 065a88f40..7f3036037 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -563,7 +563,7 @@ class Backtesting: pos_trade = self._get_exit_for_signal(trade, row, exit_, amount) if pos_trade is not None: order = pos_trade.orders[-1] - if self._get_order_filled(order.price, row): + if self._get_order_filled(order.ft_price, row): order.close_bt_order(current_date, trade) trade.recalc_trade_from_orders() self.wallets.update() @@ -664,6 +664,7 @@ class Backtesting: side=trade.exit_side, order_type=order_type, status="open", + ft_price=close_rate, price=close_rate, average=close_rate, amount=amount, @@ -887,6 +888,7 @@ class Backtesting: order_date=current_time, order_filled_date=current_time, order_update_date=current_time, + ft_price=propose_rate, price=propose_rate, average=propose_rate, amount=amount, @@ -895,7 +897,7 @@ class Backtesting: cost=stake_amount + trade.fee_open, ) trade.orders.append(order) - if pos_adjust and self._get_order_filled(order.price, row): + if pos_adjust and self._get_order_filled(order.ft_price, row): order.close_bt_order(current_time, trade) else: trade.open_order_id = str(self.order_id_counter) @@ -1008,15 +1010,15 @@ class Backtesting: # only check on new candles for open entry orders if order.side == trade.entry_side and current_time > order.order_date_utc: requested_rate = strategy_safe_wrapper(self.strategy.adjust_entry_price, - default_retval=order.price)( + default_retval=order.ft_price)( trade=trade, # type: ignore[arg-type] order=order, pair=trade.pair, current_time=current_time, - proposed_rate=row[OPEN_IDX], current_order_rate=order.price, + proposed_rate=row[OPEN_IDX], current_order_rate=order.ft_price, entry_tag=trade.enter_tag, side=trade.trade_direction ) # default value is current order price # cancel existing order whenever a new rate is requested (or None) - if requested_rate == order.price: + if requested_rate == order.ft_price: # assumption: there can't be multiple open entry orders at any given time return False else: @@ -1028,7 +1030,8 @@ class Backtesting: if requested_rate: self._enter_trade(pair=trade.pair, row=row, trade=trade, requested_rate=requested_rate, - requested_stake=(order.remaining * order.price / trade.leverage), + requested_stake=( + order.safe_remaining * order.ft_price / trade.leverage), direction='short' if trade.is_short else 'long') self.replaced_entry_orders += 1 else: @@ -1095,7 +1098,7 @@ class Backtesting: for trade in list(LocalTrade.bt_trades_open_pp[pair]): # 3. Process entry orders. order = trade.select_order(trade.entry_side, is_open=True) - if order and self._get_order_filled(order.price, row): + if order and self._get_order_filled(order.ft_price, row): order.close_bt_order(current_time, trade) trade.open_order_id = None self.wallets.update() @@ -1106,7 +1109,7 @@ class Backtesting: # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) - if order and self._get_order_filled(order.price, row): + if order and self._get_order_filled(order.ft_price, row): order.close_bt_order(current_time, trade) trade.open_order_id = None sub_trade = order.safe_amount_after_fee != trade.amount @@ -1115,7 +1118,7 @@ class Backtesting: trade.recalc_trade_from_orders() else: trade.close_date = current_time - trade.close(order.price, show_msg=False) + trade.close(order.ft_price, show_msg=False) # logger.debug(f"{pair} - Backtesting exit {trade}") LocalTrade.close_bt_trade(trade) From 8765e3a4d63a30e1711d6701553966d74d867694 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Feb 2023 07:12:09 +0100 Subject: [PATCH 22/42] Fix some Type issues --- freqtrade/rpc/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0c1a8fbbd..f8537b0f8 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -189,8 +189,8 @@ class RPC: else: # Closed trade ... current_rate = trade.close_rate - current_profit = trade.close_profit - current_profit_abs = trade.close_profit_abs + current_profit = trade.close_profit or 0.0 + current_profit_abs = trade.close_profit_abs or 0.0 total_profit_abs = trade.realized_profit + current_profit_abs # Calculate fiat profit @@ -449,11 +449,11 @@ class RPC: durations.append((trade.close_date - trade.open_date).total_seconds()) if not trade.is_open: - profit_ratio = trade.close_profit - profit_abs = trade.close_profit_abs + profit_ratio = trade.close_profit or 0.0 + profit_abs = trade.close_profit_abs or 0.0 profit_closed_coin.append(profit_abs) profit_closed_ratio.append(profit_ratio) - if trade.close_profit >= 0: + if profit_ratio >= 0: winning_trades += 1 winning_profit += profit_abs else: From c2c039151cbcdc176ecc506bf6ee6b8fc2d311f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Feb 2023 18:26:32 +0100 Subject: [PATCH 23/42] Improve typesafety around trade object --- freqtrade/freqtradebot.py | 9 +++++++-- freqtrade/rpc/rpc.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ad0628c59..cec7176f6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1314,7 +1314,7 @@ class FreqtradeBot(LoggingMixin): default_retval=order_obj.price)( trade=trade, order=order_obj, pair=trade.pair, current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, - current_order_rate=order_obj.price, entry_tag=trade.enter_tag, + current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag, side=trade.entry_side) replacing = True @@ -1345,6 +1345,8 @@ class FreqtradeBot(LoggingMixin): """ for trade in Trade.get_open_order_trades(): + if not trade.open_order_id: + continue try: order = self.exchange.fetch_order(trade.open_order_id, trade.pair) except (ExchangeError): @@ -1369,6 +1371,9 @@ class FreqtradeBot(LoggingMixin): """ was_trade_fully_canceled = False side = trade.entry_side.capitalize() + if not trade.open_order_id: + logger.warning(f"No open order for {trade}.") + return False # Cancelled orders may have the status of 'canceled' or 'closed' if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES: @@ -1455,7 +1460,7 @@ class FreqtradeBot(LoggingMixin): return False try: - co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, + co = self.exchange.cancel_order_with_result(order['id'], trade.pair, trade.amount) except InvalidOrderException: logger.exception( diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f8537b0f8..38f478f4c 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -410,7 +410,7 @@ class RPC: exit_reasons[trade.exit_reason][trade_win_loss(trade)] += 1 # Duration - dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []} + dur: Dict[str, List[float]] = {'wins': [], 'draws': [], 'losses': []} for trade in trades: if trade.close_date is not None and trade.open_date is not None: trade_dur = (trade.close_date - trade.open_date).total_seconds() From db4f4498dc6583e6a1eb9e3183f6b11c6232de96 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Feb 2023 20:00:01 +0100 Subject: [PATCH 24/42] Experimentally type query property ... --- freqtrade/persistence/models.py | 3 ++- freqtrade/persistence/pairlock.py | 6 ++++-- freqtrade/persistence/trade_model.py | 12 +++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f4058b4eb..b94be950a 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -6,7 +6,7 @@ from typing import Any, Dict from sqlalchemy import create_engine, inspect from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.orm import Session, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) _SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' +SessionType = scoped_session[Session] def init_db(db_url: str) -> None: diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index f8c0586a4..b10d74693 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -3,9 +3,11 @@ from typing import Any, ClassVar, Dict, Optional from sqlalchemy import String, or_ from sqlalchemy.orm import Mapped, Query, mapped_column +from sqlalchemy.orm.scoping import _QueryDescriptorType from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.persistence.base import ModelBase +from freqtrade.persistence.models import SessionType class PairLock(ModelBase): @@ -13,8 +15,8 @@ class PairLock(ModelBase): Pair Locks database model. """ __tablename__ = 'pairlocks' - # TODO: Properly type query. - query: ClassVar[Any] + query: ClassVar[_QueryDescriptorType] + session: ClassVar[SessionType] id: Mapped[int] = mapped_column(primary_key=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index abb989940..10643db17 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -9,6 +9,7 @@ from typing import Any, ClassVar, Dict, List, Optional from sqlalchemy import Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship +from sqlalchemy.orm.scoping import _QueryDescriptorType from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) @@ -17,6 +18,7 @@ from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import amount_to_contract_precision, price_to_precision from freqtrade.leverage import interest from freqtrade.persistence.base import ModelBase +from freqtrade.persistence.models import SessionType from freqtrade.util import FtPrecise @@ -35,8 +37,9 @@ class Order(ModelBase): Mirrors CCXT Order structure """ __tablename__ = 'orders' - # TODO: Properly type query. - query: ClassVar[Any] + query: ClassVar[_QueryDescriptorType] + session: ClassVar[SessionType] + # Uniqueness should be ensured over pair, order_id # its likely that order_id is unique per Pair on some exchanges. __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) @@ -1172,9 +1175,8 @@ class Trade(ModelBase, LocalTrade): Note: Fields must be aligned with LocalTrade class """ __tablename__ = 'trades' - # TODO: Type query type throughout. - query: ClassVar[Any] - session: ClassVar[Any] = None + query: ClassVar[_QueryDescriptorType] + session: ClassVar[SessionType] = None use_db: bool = True From b65cff0adcd6498a0dfe288a97c93488c3bb7fab Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Feb 2023 20:22:41 +0100 Subject: [PATCH 25/42] Update "Query" type --- freqtrade/persistence/trade_model.py | 2 +- freqtrade/rpc/rpc.py | 9 +++++---- freqtrade/rpc/telegram.py | 10 +++++++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 10643db17..0f85528e0 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1305,7 +1305,7 @@ class Trade(ModelBase, LocalTrade): ) @staticmethod - def get_trades(trade_filter=None, include_orders: bool = True) -> Query: + def get_trades(trade_filter=None, include_orders: bool = True) -> Query['Trade']: """ Helper function to query Trades using filters. NOTE: Not supported in Backtesting. diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 38f478f4c..8692c477f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -373,13 +373,13 @@ class RPC: def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict: """ Returns the X last trades """ - order_by = Trade.id if order_by_id else Trade.close_date.desc() + order_by: Any = Trade.id if order_by_id else Trade.close_date.desc() if limit: trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by( order_by).limit(limit).offset(offset) else: trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by( - Trade.close_date.desc()).all() + Trade.close_date.desc()) output = [trade.to_json() for trade in trades] @@ -401,7 +401,7 @@ class RPC: return 'losses' else: return 'draws' - trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False) + trades = Trade.get_trades([Trade.is_open.is_(False)], include_orders=False) # Sell reason exit_reasons = {} for trade in trades: @@ -785,7 +785,8 @@ class RPC: # check if valid pair # check if pair already has an open pair - trade: Trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() + trade: Optional[Trade] = Trade.get_trades( + [Trade.is_open.is_(True), Trade.pair == pair]).first() is_short = (order_side == SignalDirection.SHORT) if trade: is_short = trade.is_short diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7bbeea2a2..dc92478ab 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1055,10 +1055,14 @@ class Telegram(RPCHandler): query.answer() query.edit_message_text(text="Force exit canceled.") return - trade: Trade = Trade.get_trades(trade_filter=Trade.id == trade_id).first() + trade: Optional[Trade] = Trade.get_trades(trade_filter=Trade.id == trade_id).first() query.answer() - query.edit_message_text(text=f"Manually exiting Trade #{trade_id}, {trade.pair}") - self._force_exit_action(trade_id) + if trade: + query.edit_message_text( + text=f"Manually exiting Trade #{trade_id}, {trade.pair}") + self._force_exit_action(trade_id) + else: + query.edit_message_text(text=f"Trade {trade_id} not found.") def _force_enter_action(self, pair, price: Optional[float], order_side: SignalDirection): if pair != 'cancel': From 764001a4c290ba9bca30a224171f4f88ef20001f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 21 Feb 2023 06:49:15 +0100 Subject: [PATCH 26/42] Don't reuse variable --- freqtrade/data/btanalysis.py | 2 +- freqtrade/persistence/pairlock_middleware.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index c682436c7..9772506a7 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -346,7 +346,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str, return df_final[df_final['open_trades'] > max_open_trades] -def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: +def trade_list_to_dataframe(trades: Union[List[Trade], List[LocalTrade]]) -> pd.DataFrame: """ Convert list of Trade objects to pandas Dataframe :param trades: List of trade objects diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 4485bb88e..5ed131a9b 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -133,8 +133,8 @@ class PairLocks(): PairLock.query.session.commit() else: # used in backtesting mode; don't show log messages for speed - locks = PairLocks.get_pair_locks(None) - for lock in locks: + locksb = PairLocks.get_pair_locks(None) + for lock in locksb: if lock.reason == reason: lock.active = False From f2f4158974b294339ba3c22352b44c4dd9f31c5d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 21 Feb 2023 06:58:21 +0100 Subject: [PATCH 27/42] Bump sqlalchemy to 2.0.4 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3e6c66757..6b1c888b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ pandas-ta==0.3.14b ccxt==2.8.54 cryptography==39.0.1 aiohttp==3.8.4 -SQLAlchemy==2.0.3 +SQLAlchemy==2.0.4 python-telegram-bot==13.15 arrow==1.2.3 cachetools==4.2.2 From d175ab495b39a9a249274b97ea52292840592283 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Feb 2023 07:03:22 +0100 Subject: [PATCH 28/42] Move SessionType to base module --- freqtrade/persistence/base.py | 4 +++- freqtrade/persistence/models.py | 5 ++--- freqtrade/persistence/pairlock.py | 3 +-- freqtrade/persistence/trade_model.py | 3 +-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/freqtrade/persistence/base.py b/freqtrade/persistence/base.py index 2fed715d7..98e483c90 100644 --- a/freqtrade/persistence/base.py +++ b/freqtrade/persistence/base.py @@ -1,6 +1,8 @@ -from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import DeclarativeBase, Session, scoped_session +SessionType = scoped_session[Session] + class ModelBase(DeclarativeBase): pass diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index b94be950a..98d1d7a8a 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -6,11 +6,11 @@ from typing import Any, Dict from sqlalchemy import create_engine, inspect from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.orm import Session, scoped_session, sessionmaker +from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException -from freqtrade.persistence.base import ModelBase +from freqtrade.persistence.base import ModelBase, SessionType from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade @@ -20,7 +20,6 @@ logger = logging.getLogger(__name__) _SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' -SessionType = scoped_session[Session] def init_db(db_url: str) -> None: diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index b10d74693..d5a8d7ae1 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -6,8 +6,7 @@ from sqlalchemy.orm import Mapped, Query, mapped_column from sqlalchemy.orm.scoping import _QueryDescriptorType from freqtrade.constants import DATETIME_PRINT_FORMAT -from freqtrade.persistence.base import ModelBase -from freqtrade.persistence.models import SessionType +from freqtrade.persistence.base import ModelBase, SessionType class PairLock(ModelBase): diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 0f85528e0..ce7d56745 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -17,8 +17,7 @@ from freqtrade.enums import ExitType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import amount_to_contract_precision, price_to_precision from freqtrade.leverage import interest -from freqtrade.persistence.base import ModelBase -from freqtrade.persistence.models import SessionType +from freqtrade.persistence.base import ModelBase, SessionType from freqtrade.util import FtPrecise From 0f914cf2bd54902ceb96c29499eca888b4422b03 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Feb 2023 07:24:20 +0100 Subject: [PATCH 29/42] Use Mapped for LocalTrade this won't initialize sqlalchemy, as the base class is not inheriting from sqlalchemy. --- freqtrade/persistence/trade_model.py | 108 +++++++++++++-------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ce7d56745..93701b1ae 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -288,76 +288,76 @@ class LocalTrade(): bt_trades_open_pp: Dict[str, List['LocalTrade']] = defaultdict(list) bt_open_open_trade_count: int = 0 total_profit: float = 0 - realized_profit: float = 0 + realized_profit: Mapped[float] = 0 # type: ignore - id: int = 0 + id: Mapped[int] = 0 # type: ignore - orders: List[Order] = [] + orders: Mapped[List[Order]] = [] # type: ignore - exchange: str = '' - pair: str = '' - base_currency: str = '' - stake_currency: str = '' - is_open: bool = True - fee_open: float = 0.0 - fee_open_cost: Optional[float] = None - fee_open_currency: str = '' - fee_close: float = 0.0 - fee_close_cost: Optional[float] = None - fee_close_currency: str = '' - open_rate: float = 0.0 - open_rate_requested: Optional[float] = None + exchange: Mapped[str] = '' # type: ignore + pair: Mapped[str] = '' # type: ignore + base_currency: Mapped[Optional[str]] = '' # type: ignore + stake_currency: Mapped[Optional[str]] = '' # type: ignore + is_open: Mapped[bool] = True # type: ignore + fee_open: Mapped[float] = 0.0 # type: ignore + fee_open_cost: Mapped[Optional[float]] = None # type: ignore + fee_open_currency: Mapped[Optional[str]] = '' # type: ignore + fee_close: Mapped[Optional[float]] = 0.0 # type: ignore + fee_close_cost: Mapped[Optional[float]] = None # type: ignore + fee_close_currency: Mapped[Optional[str]] = '' # type: ignore + open_rate: Mapped[float] = 0.0 # type: ignore + open_rate_requested: Mapped[Optional[float]] = None # type: ignore # open_trade_value - calculated via _calc_open_trade_value - open_trade_value: float = 0.0 - close_rate: Optional[float] = None - close_rate_requested: Optional[float] = None - close_profit: Optional[float] = None - close_profit_abs: Optional[float] = None - stake_amount: float = 0.0 - max_stake_amount: float = 0.0 - amount: float = 0.0 - amount_requested: Optional[float] = None - open_date: datetime - close_date: Optional[datetime] = None - open_order_id: Optional[str] = None + open_trade_value: Mapped[float] = 0.0 # type: ignore + close_rate: Mapped[Optional[float]] = None # type: ignore + close_rate_requested: Mapped[Optional[float]] = None # type: ignore + close_profit: Mapped[Optional[float]] = None # type: ignore + close_profit_abs: Mapped[Optional[float]] = None # type: ignore + stake_amount: Mapped[float] = 0.0 # type: ignore + max_stake_amount: Mapped[Optional[float]] = 0.0 # type: ignore + amount: Mapped[float] = 0.0 # type: ignore + amount_requested: Mapped[Optional[float]] = None # type: ignore + open_date: Mapped[datetime] + close_date: Mapped[Optional[datetime]] = None # type: ignore + open_order_id: Mapped[Optional[str]] = None # type: ignore # absolute value of the stop loss - stop_loss: float = 0.0 + stop_loss: Mapped[float] = 0.0 # type: ignore # percentage value of the stop loss - stop_loss_pct: float = 0.0 + stop_loss_pct: Mapped[Optional[float]] = 0.0 # type: ignore # absolute value of the initial stop loss - initial_stop_loss: float = 0.0 + initial_stop_loss: Mapped[Optional[float]] = 0.0 # type: ignore # percentage value of the initial stop loss - initial_stop_loss_pct: Optional[float] = None + initial_stop_loss_pct: Mapped[Optional[float]] = None # type: ignore # stoploss order id which is on exchange - stoploss_order_id: Optional[str] = None + stoploss_order_id: Mapped[Optional[str]] = None # type: ignore # last update time of the stoploss order on exchange - stoploss_last_update: Optional[datetime] = None + stoploss_last_update: Mapped[Optional[datetime]] = None # type: ignore # absolute value of the highest reached price - max_rate: float = 0.0 + max_rate: Mapped[Optional[float]] = None # type: ignore # Lowest price reached - min_rate: float = 0.0 - exit_reason: str = '' - exit_order_status: str = '' - strategy: str = '' - enter_tag: Optional[str] = None - timeframe: Optional[int] = None + min_rate: Mapped[Optional[float]] = None # type: ignore + exit_reason: Mapped[Optional[str]] = '' # type: ignore + exit_order_status: Mapped[Optional[str]] = '' # type: ignore + strategy: Mapped[Optional[str]] = '' # type: ignore + enter_tag: Mapped[Optional[str]] = None # type: ignore + timeframe: Mapped[Optional[int]] = None # type: ignore - trading_mode: TradingMode = TradingMode.SPOT - amount_precision: Optional[float] = None - price_precision: Optional[float] = None - precision_mode: Optional[int] = None - contract_size: Optional[float] = None + trading_mode: Mapped[TradingMode] = TradingMode.SPOT # type: ignore + amount_precision: Mapped[Optional[float]] = None # type: ignore + price_precision: Mapped[Optional[float]] = None # type: ignore + precision_mode: Mapped[Optional[int]] = None # type: ignore + contract_size: Mapped[Optional[float]] = None # type: ignore # Leverage trading properties - liquidation_price: Optional[float] = None - is_short: bool = False - leverage: float = 1.0 + liquidation_price: Mapped[Optional[float]] = None # type: ignore + is_short: Mapped[bool] = False # type: ignore + leverage: Mapped[float] = 1.0 # type: ignore # Margin trading properties - interest_rate: float = 0.0 + interest_rate: Mapped[float] = 0.0 # type: ignore # Futures properties - funding_fees: Optional[float] = None + funding_fees: Mapped[Optional[float]] = None # type: ignore @property def stoploss_or_liquidation(self) -> float: @@ -1175,7 +1175,7 @@ class Trade(ModelBase, LocalTrade): """ __tablename__ = 'trades' query: ClassVar[_QueryDescriptorType] - session: ClassVar[SessionType] = None + session: ClassVar[SessionType] use_db: bool = True @@ -1197,7 +1197,7 @@ class Trade(ModelBase, LocalTrade): fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) fee_close_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) open_rate: Mapped[float] = mapped_column(Float()) - open_rate_requested: Mapped[float] = mapped_column(Float(), nullable=True) + open_rate_requested: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # open_trade_value - calculated via _calc_open_trade_value open_trade_value = mapped_column(Float()) close_rate: Mapped[Optional[float]] = mapped_column(Float()) @@ -1246,7 +1246,7 @@ class Trade(ModelBase, LocalTrade): liquidation_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # Margin Trading Properties - interest_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=False, default=0.0) + interest_rate: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0) # Futures properties funding_fees: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=None) From 7c09c0178815a725a3ac0df5509e7a0d5100a475 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Feb 2023 07:27:01 +0100 Subject: [PATCH 30/42] Add some more typehints --- freqtrade/persistence/trade_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 93701b1ae..887784be3 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -154,7 +154,7 @@ class Order(ModelBase): self.order_update_date = datetime.now(timezone.utc) def to_ccxt_object(self) -> Dict[str, Any]: - order = { + order: Dict[str, Any] = { 'id': self.order_id, 'symbol': self.ft_pair, 'price': self.price, @@ -1062,7 +1062,7 @@ class LocalTrade(): return len(self.select_filled_orders('sell')) @property - def sell_reason(self) -> str: + def sell_reason(self) -> Optional[str]: """ DEPRECATED! Please use exit_reason instead.""" return self.exit_reason @@ -1276,7 +1276,7 @@ class Trade(ModelBase, LocalTrade): def get_trades_proxy(*, pair: Optional[str] = None, is_open: Optional[bool] = None, open_date: Optional[datetime] = None, close_date: Optional[datetime] = None, - ) -> List['LocalTrade']: + ) -> List['LocalTrade', 'Trade']: """ Helper function to query Trades.j Returns a List of trades, filtered on the parameters given. From b5f55c9b145129567d5d642efb7229fbf36c4ff9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 06:40:07 +0100 Subject: [PATCH 31/42] Improve type safety in backtesting --- freqtrade/optimize/backtesting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7f3036037..1f868f7bf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -440,7 +440,8 @@ class Backtesting: side_1 * abs(self.strategy.trailing_stop_positive / leverage))) else: # Worst case: price ticks tiny bit above open and dives down. - stop_rate = row[OPEN_IDX] * (1 - side_1 * abs(trade.stop_loss_pct / leverage)) + stop_rate = row[OPEN_IDX] * (1 - side_1 * abs( + (trade.stop_loss_pct or 0.0) / leverage)) if is_short: assert stop_rate > row[LOW_IDX] else: @@ -472,7 +473,7 @@ class Backtesting: # - (Expected abs profit - open_rate - open_fee) / (fee_close -1) roi_rate = trade.open_rate * roi / leverage open_fee_rate = side_1 * trade.open_rate * (1 + side_1 * trade.fee_open) - close_rate = -(roi_rate + open_fee_rate) / (trade.fee_close - side_1 * 1) + close_rate = -(roi_rate + open_fee_rate) / ((trade.fee_close or 0.0) - side_1 * 1) if is_short: is_new_roi = row[OPEN_IDX] < close_rate else: From e5c9cde36f713c50a7085c7ad4c9377a97bd34b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 07:23:26 +0100 Subject: [PATCH 32/42] Update trades_proxy typing --- freqtrade/persistence/trade_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 887784be3..531e86282 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -5,7 +5,7 @@ import logging from collections import defaultdict from datetime import datetime, timedelta, timezone from math import isclose -from typing import Any, ClassVar, Dict, List, Optional +from typing import Any, ClassVar, Dict, List, Optional, cast from sqlalchemy import Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship @@ -1131,7 +1131,7 @@ class LocalTrade(): @staticmethod def get_open_trades() -> List[Any]: """ - Query trades from persistence layer + Retrieve open trades """ return Trade.get_trades_proxy(is_open=True) @@ -1276,7 +1276,7 @@ class Trade(ModelBase, LocalTrade): def get_trades_proxy(*, pair: Optional[str] = None, is_open: Optional[bool] = None, open_date: Optional[datetime] = None, close_date: Optional[datetime] = None, - ) -> List['LocalTrade', 'Trade']: + ) -> List['LocalTrade']: """ Helper function to query Trades.j Returns a List of trades, filtered on the parameters given. @@ -1295,7 +1295,7 @@ class Trade(ModelBase, LocalTrade): trade_filter.append(Trade.close_date > close_date) if is_open is not None: trade_filter.append(Trade.is_open.is_(is_open)) - return Trade.get_trades(trade_filter).all() + return cast(List[LocalTrade], Trade.get_trades(trade_filter).all()) else: return LocalTrade.get_trades_proxy( pair=pair, is_open=is_open, From a1166b1077ca96db704d97e5d3775d960ee67ba9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 07:25:33 +0100 Subject: [PATCH 33/42] allow null fee on calc_base_close --- freqtrade/persistence/trade_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 531e86282..7eabfab0f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -795,10 +795,10 @@ class LocalTrade(): return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours) - def _calc_base_close(self, amount: FtPrecise, rate: float, fee: float) -> FtPrecise: + def _calc_base_close(self, amount: FtPrecise, rate: float, fee: Optional[float]) -> FtPrecise: close_trade = amount * FtPrecise(rate) - fees = close_trade * FtPrecise(fee) + fees = close_trade * FtPrecise(fee or 0.0) if self.is_short: return close_trade + fees From 4a35d32b6ae84b2c0e372a7e00df283b7df76078 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 07:26:35 +0100 Subject: [PATCH 34/42] Improve trade stop types --- freqtrade/persistence/trade_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 7eabfab0f..d747971fb 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -595,7 +595,7 @@ class LocalTrade(): self.stop_loss_pct = -1 * abs(percent) - def adjust_stop_loss(self, current_price: float, stoploss: float, + def adjust_stop_loss(self, current_price: float, stoploss: Optional[float], initial: bool = False, refresh: bool = False) -> None: """ This adjusts the stop loss to it's most recently observed setting @@ -604,7 +604,7 @@ class LocalTrade(): :param initial: Called to initiate stop_loss. Skips everything if self.stop_loss is already set. """ - if initial and not (self.stop_loss is None or self.stop_loss == 0): + if stoploss is None or (initial and not (self.stop_loss is None or self.stop_loss == 0)): # Don't modify if called with initial and nothing to do return refresh = True if refresh and self.nr_of_successful_entries == 1 else False From 874413ccc59ebe0ab7346de74ea3ff06c82b513c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 11:16:56 +0000 Subject: [PATCH 35/42] Fix some style violations --- freqtrade/persistence/base.py | 1 + freqtrade/persistence/models.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/base.py b/freqtrade/persistence/base.py index 98e483c90..fc2dac75e 100644 --- a/freqtrade/persistence/base.py +++ b/freqtrade/persistence/base.py @@ -4,5 +4,6 @@ from sqlalchemy.orm import DeclarativeBase, Session, scoped_session SessionType = scoped_session[Session] + class ModelBase(DeclarativeBase): pass diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 98d1d7a8a..f4058b4eb 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException -from freqtrade.persistence.base import ModelBase, SessionType +from freqtrade.persistence.base import ModelBase from freqtrade.persistence.migrations import check_migrate from freqtrade.persistence.pairlock import PairLock from freqtrade.persistence.trade_model import Order, Trade From 388dfec50bbf3ff4884dae25b4edf213a203edf8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 18:17:37 +0100 Subject: [PATCH 36/42] Remove last type error --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index d747971fb..ab7d56766 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -643,7 +643,7 @@ class LocalTrade(): f"initial_stop_loss={self.initial_stop_loss:.8f}, " f"stop_loss={self.stop_loss:.8f}. " f"Trailing stoploss saved us: " - f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.") + f"{float(self.stop_loss) - float(self.initial_stop_loss or 0.0):.8f}.") def update_trade(self, order: Order) -> None: """ From f0f72fdd33db951f6bbc24a8e93a6f0acdadafb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 19:48:15 +0100 Subject: [PATCH 37/42] Don't define "mapped" on LocalTrade class --- freqtrade/persistence/trade_model.py | 220 ++++++++++++++------------- 1 file changed, 118 insertions(+), 102 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ab7d56766..62e2e2a7f 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -288,76 +288,76 @@ class LocalTrade(): bt_trades_open_pp: Dict[str, List['LocalTrade']] = defaultdict(list) bt_open_open_trade_count: int = 0 total_profit: float = 0 - realized_profit: Mapped[float] = 0 # type: ignore + realized_profit: float = 0 - id: Mapped[int] = 0 # type: ignore + id: int = 0 - orders: Mapped[List[Order]] = [] # type: ignore + orders: List[Order] = [] - exchange: Mapped[str] = '' # type: ignore - pair: Mapped[str] = '' # type: ignore - base_currency: Mapped[Optional[str]] = '' # type: ignore - stake_currency: Mapped[Optional[str]] = '' # type: ignore - is_open: Mapped[bool] = True # type: ignore - fee_open: Mapped[float] = 0.0 # type: ignore - fee_open_cost: Mapped[Optional[float]] = None # type: ignore - fee_open_currency: Mapped[Optional[str]] = '' # type: ignore - fee_close: Mapped[Optional[float]] = 0.0 # type: ignore - fee_close_cost: Mapped[Optional[float]] = None # type: ignore - fee_close_currency: Mapped[Optional[str]] = '' # type: ignore - open_rate: Mapped[float] = 0.0 # type: ignore - open_rate_requested: Mapped[Optional[float]] = None # type: ignore + exchange: str = '' + pair: str = '' + base_currency: Optional[str] = '' + stake_currency: Optional[str] = '' + is_open: bool = True + fee_open: float = 0.0 + fee_open_cost: Optional[float] = None + fee_open_currency: Optional[str] = '' + fee_close: Optional[float] = 0.0 + fee_close_cost: Optional[float] = None + fee_close_currency: Optional[str] = '' + open_rate: float = 0.0 + open_rate_requested: Optional[float] = None # open_trade_value - calculated via _calc_open_trade_value - open_trade_value: Mapped[float] = 0.0 # type: ignore - close_rate: Mapped[Optional[float]] = None # type: ignore - close_rate_requested: Mapped[Optional[float]] = None # type: ignore - close_profit: Mapped[Optional[float]] = None # type: ignore - close_profit_abs: Mapped[Optional[float]] = None # type: ignore - stake_amount: Mapped[float] = 0.0 # type: ignore - max_stake_amount: Mapped[Optional[float]] = 0.0 # type: ignore - amount: Mapped[float] = 0.0 # type: ignore - amount_requested: Mapped[Optional[float]] = None # type: ignore - open_date: Mapped[datetime] - close_date: Mapped[Optional[datetime]] = None # type: ignore - open_order_id: Mapped[Optional[str]] = None # type: ignore + open_trade_value: float = 0.0 + close_rate: Optional[float] = None + close_rate_requested: Optional[float] = None + close_profit: Optional[float] = None + close_profit_abs: Optional[float] = None + stake_amount: float = 0.0 + max_stake_amount: Optional[float] = 0.0 + amount: float = 0.0 + amount_requested: Optional[float] = None + open_date: datetime + close_date: Optional[datetime] = None + open_order_id: Optional[str] = None # absolute value of the stop loss - stop_loss: Mapped[float] = 0.0 # type: ignore + stop_loss: float = 0.0 # percentage value of the stop loss - stop_loss_pct: Mapped[Optional[float]] = 0.0 # type: ignore + stop_loss_pct: Optional[float] = 0.0 # absolute value of the initial stop loss - initial_stop_loss: Mapped[Optional[float]] = 0.0 # type: ignore + initial_stop_loss: Optional[float] = 0.0 # percentage value of the initial stop loss - initial_stop_loss_pct: Mapped[Optional[float]] = None # type: ignore + initial_stop_loss_pct: Optional[float] = None # stoploss order id which is on exchange - stoploss_order_id: Mapped[Optional[str]] = None # type: ignore + stoploss_order_id: Optional[str] = None # last update time of the stoploss order on exchange - stoploss_last_update: Mapped[Optional[datetime]] = None # type: ignore + stoploss_last_update: Optional[datetime] = None # absolute value of the highest reached price - max_rate: Mapped[Optional[float]] = None # type: ignore + max_rate: Optional[float] = None # Lowest price reached - min_rate: Mapped[Optional[float]] = None # type: ignore - exit_reason: Mapped[Optional[str]] = '' # type: ignore - exit_order_status: Mapped[Optional[str]] = '' # type: ignore - strategy: Mapped[Optional[str]] = '' # type: ignore - enter_tag: Mapped[Optional[str]] = None # type: ignore - timeframe: Mapped[Optional[int]] = None # type: ignore + min_rate: Optional[float] = None + exit_reason: Optional[str] = '' + exit_order_status: Optional[str] = '' + strategy: Optional[str] = '' + enter_tag: Optional[str] = None + timeframe: Optional[int] = None - trading_mode: Mapped[TradingMode] = TradingMode.SPOT # type: ignore - amount_precision: Mapped[Optional[float]] = None # type: ignore - price_precision: Mapped[Optional[float]] = None # type: ignore - precision_mode: Mapped[Optional[int]] = None # type: ignore - contract_size: Mapped[Optional[float]] = None # type: ignore + trading_mode: TradingMode = TradingMode.SPOT + amount_precision: Optional[float] = None + price_precision: Optional[float] = None + precision_mode: Optional[int] = None + contract_size: Optional[float] = None # Leverage trading properties - liquidation_price: Mapped[Optional[float]] = None # type: ignore - is_short: Mapped[bool] = False # type: ignore - leverage: Mapped[float] = 1.0 # type: ignore + liquidation_price: Optional[float] = None + is_short: bool = False + leverage: float = 1.0 # Margin trading properties - interest_rate: Mapped[float] = 0.0 # type: ignore + interest_rate: float = 0.0 # Futures properties - funding_fees: Mapped[Optional[float]] = None # type: ignore + funding_fees: Optional[float] = None @property def stoploss_or_liquidation(self) -> float: @@ -1179,77 +1179,93 @@ class Trade(ModelBase, LocalTrade): use_db: bool = True - id: Mapped[int] = mapped_column(Integer, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) # type: ignore orders: Mapped[List[Order]] = relationship( "Order", order_by="Order.id", cascade="all, delete-orphan", lazy="selectin", - innerjoin=True) + innerjoin=True) # type: ignore - exchange: Mapped[str] = mapped_column(String(25), nullable=False) - pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) - base_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) - stake_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) - is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True) - fee_open: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0) - fee_open_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - fee_open_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) - fee_close: Mapped[Optional[float]] = mapped_column(Float(), nullable=False, default=0.0) - fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - fee_close_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) - open_rate: Mapped[float] = mapped_column(Float()) - open_rate_requested: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + exchange: Mapped[str] = mapped_column(String(25), nullable=False) # type: ignore + pair: Mapped[str] = mapped_column(String(25), nullable=False, index=True) # type: ignore + base_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) # type: ignore + stake_currency: Mapped[Optional[str]] = mapped_column(String(25), nullable=True) # type: ignore + is_open: Mapped[bool] = mapped_column(nullable=False, default=True, index=True) # type: ignore + fee_open: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0) # type: ignore + fee_open_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # type: ignore + fee_open_currency: Mapped[Optional[str]] = mapped_column( + String(25), nullable=True) # type: ignore + fee_close: Mapped[Optional[float]] = mapped_column( + Float(), nullable=False, default=0.0) # type: ignore + fee_close_cost: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # type: ignore + fee_close_currency: Mapped[Optional[str]] = mapped_column( + String(25), nullable=True) # type: ignore + open_rate: Mapped[float] = mapped_column(Float()) # type: ignore + open_rate_requested: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True) # type: ignore # open_trade_value - calculated via _calc_open_trade_value - open_trade_value = mapped_column(Float()) - close_rate: Mapped[Optional[float]] = mapped_column(Float()) - close_rate_requested: Mapped[Optional[float]] = mapped_column(Float()) - realized_profit: Mapped[float] = mapped_column(Float(), default=0.0) - close_profit: Mapped[Optional[float]] = mapped_column(Float()) - close_profit_abs: Mapped[Optional[float]] = mapped_column(Float()) - stake_amount: Mapped[float] = mapped_column(Float(), nullable=False) - max_stake_amount: Mapped[Optional[float]] = mapped_column(Float()) - amount: Mapped[float] = mapped_column(Float()) - amount_requested: Mapped[Optional[float]] = mapped_column(Float()) - open_date: Mapped[datetime] = mapped_column(nullable=False, default=datetime.utcnow) - close_date: Mapped[Optional[datetime]] = mapped_column() - open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + open_trade_value: Mapped[float] = mapped_column(Float(), nullable=True) # type: ignore + close_rate: Mapped[Optional[float]] = mapped_column(Float()) # type: ignore + close_rate_requested: Mapped[Optional[float]] = mapped_column(Float()) # type: ignore + realized_profit: Mapped[float] = mapped_column( + Float(), default=0.0, nullable=True) # type: ignore + close_profit: Mapped[Optional[float]] = mapped_column(Float()) # type: ignore + close_profit_abs: Mapped[Optional[float]] = mapped_column(Float()) # type: ignore + stake_amount: Mapped[float] = mapped_column(Float(), nullable=False) # type: ignore + max_stake_amount: Mapped[Optional[float]] = mapped_column(Float()) # type: ignore + amount: Mapped[float] = mapped_column(Float()) # type: ignore + amount_requested: Mapped[Optional[float]] = mapped_column(Float()) # type: ignore + open_date: Mapped[datetime] = mapped_column( + nullable=False, default=datetime.utcnow) # type: ignore + close_date: Mapped[Optional[datetime]] = mapped_column() # type: ignore + open_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) # type: ignore # absolute value of the stop loss - stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0) + stop_loss: Mapped[float] = mapped_column(Float(), nullable=True, default=0.0) # type: ignore # percentage value of the stop loss - stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # type: ignore # absolute value of the initial stop loss - initial_stop_loss: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=0.0) + initial_stop_loss: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True, default=0.0) # type: ignore # percentage value of the initial stop loss - initial_stop_loss_pct: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + initial_stop_loss_pct: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True) # type: ignore # stoploss order id which is on exchange - stoploss_order_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True) + stoploss_order_id: Mapped[Optional[str]] = mapped_column( + String(255), nullable=True, index=True) # type: ignore # last update time of the stoploss order on exchange - stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True) + stoploss_last_update: Mapped[Optional[datetime]] = mapped_column(nullable=True) # type: ignore # absolute value of the highest reached price - max_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=0.0) + max_rate: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True, default=0.0) # type: ignore # Lowest price reached - min_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - exit_reason: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) - exit_order_status: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) - strategy: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) - enter_tag: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) - timeframe: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + min_rate: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # type: ignore + exit_reason: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) # type: ignore + exit_order_status: Mapped[Optional[str]] = mapped_column( + String(100), nullable=True) # type: ignore + strategy: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) # type: ignore + enter_tag: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) # type: ignore + timeframe: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) # type: ignore - trading_mode: Mapped[TradingMode] = mapped_column(Enum(TradingMode), nullable=True) - amount_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - price_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) - precision_mode: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) - contract_size: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + trading_mode: Mapped[TradingMode] = mapped_column( + Enum(TradingMode), nullable=True) # type: ignore + amount_precision: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True) # type: ignore + price_precision: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # type: ignore + precision_mode: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) # type: ignore + contract_size: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) # type: ignore # Leverage trading properties - leverage: Mapped[float] = mapped_column(Float(), nullable=True, default=1.0) - is_short: Mapped[bool] = mapped_column(nullable=False, default=False) - liquidation_price: Mapped[Optional[float]] = mapped_column(Float(), nullable=True) + leverage: Mapped[float] = mapped_column(Float(), nullable=True, default=1.0) # type: ignore + is_short: Mapped[bool] = mapped_column(nullable=False, default=False) # type: ignore + liquidation_price: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True) # type: ignore # Margin Trading Properties - interest_rate: Mapped[float] = mapped_column(Float(), nullable=False, default=0.0) + interest_rate: Mapped[float] = mapped_column( + Float(), nullable=False, default=0.0) # type: ignore # Futures properties - funding_fees: Mapped[Optional[float]] = mapped_column(Float(), nullable=True, default=None) + funding_fees: Mapped[Optional[float]] = mapped_column( + Float(), nullable=True, default=None) # type: ignore def __init__(self, **kwargs): super().__init__(**kwargs) From 59d57d34667524e3eeed404fbfb0e3c62b4faa3b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 20:01:43 +0100 Subject: [PATCH 38/42] Improve test resiliance --- tests/test_freqtradebot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 1ea3ebfc6..800e76342 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3036,6 +3036,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf_usdt, limit_order, is_ # Order remained open for some reason (cancel failed) cancel_buy_order['status'] = 'open' cancel_order_mock = MagicMock(return_value=cancel_buy_order) + trade.open_order_id = 'some_open_order' mocker.patch(f'{EXMS}.cancel_order_with_result', cancel_order_mock) assert not freqtrade.handle_cancel_enter(trade, l_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) @@ -3231,6 +3232,7 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: trade = MagicMock() reason = CANCEL_REASON['TIMEOUT'] order = {'remaining': 1, + 'id': '125', 'amount': 1, 'status': "open"} assert not freqtrade.handle_cancel_exit(trade, order, reason) From b4b8dde4fb9a0646297c17af8175f5f1cc3d70af Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 1 Mar 2023 20:41:49 +0100 Subject: [PATCH 39/42] Add sqlalchemy to pre-commit dependencies --- .pre-commit-config.yaml | 1 + build_helpers/pre_commit_update.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 05f4df92b..acc0c8a17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ repos: - types-requests==2.28.11.15 - types-tabulate==0.9.0.1 - types-python-dateutil==2.8.19.9 + - SQLAlchemy==2.0.4 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/build_helpers/pre_commit_update.py b/build_helpers/pre_commit_update.py index 8724d8ade..e6b47d100 100644 --- a/build_helpers/pre_commit_update.py +++ b/build_helpers/pre_commit_update.py @@ -8,12 +8,17 @@ import yaml pre_commit_file = Path('.pre-commit-config.yaml') require_dev = Path('requirements-dev.txt') +require = Path('requirements.txt') with require_dev.open('r') as rfile: requirements = rfile.readlines() +with require.open('r') as rfile: + requirements.extend(rfile.readlines()) + # Extract types only -type_reqs = [r.strip('\n') for r in requirements if r.startswith('types-')] +type_reqs = [r.strip('\n') for r in requirements if r.startswith( + 'types-') or r.startswith('SQLAlchemy')] with pre_commit_file.open('r') as file: f = yaml.load(file, Loader=yaml.FullLoader) From b980f45b2b3bbb5ef1b381af9708694f9d1685d3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Mar 2023 06:23:01 +0100 Subject: [PATCH 40/42] Fix test mypy errors --- freqtrade/persistence/trade_model.py | 4 ++++ .../strats/broken_strats/broken_futures_strategies.py | 5 +++-- tests/strategy/strats/strategy_test_v3.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 62e2e2a7f..a05eb7409 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -99,6 +99,10 @@ class Order(ModelBase): def safe_filled(self) -> float: return self.filled if self.filled is not None else self.amount or 0.0 + @property + def safe_cost(self) -> float: + return self.cost or 0.0 + @property def safe_remaining(self) -> float: return ( diff --git a/tests/strategy/strats/broken_strats/broken_futures_strategies.py b/tests/strategy/strats/broken_strats/broken_futures_strategies.py index 7e6955d37..bb7ce2b32 100644 --- a/tests/strategy/strats/broken_strats/broken_futures_strategies.py +++ b/tests/strategy/strats/broken_strats/broken_futures_strategies.py @@ -7,6 +7,7 @@ from datetime import datetime from pandas import DataFrame +from freqtrade.persistence.trade_model import Order from freqtrade.strategy.interface import IStrategy @@ -35,7 +36,7 @@ class TestStrategyImplementBuyTimeout(TestStrategyNoImplementSell): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return super().populate_exit_trend(dataframe, metadata) - def check_buy_timeout(self, pair: str, trade, order: dict, + def check_buy_timeout(self, pair: str, trade, order: Order, current_time: datetime, **kwargs) -> bool: return False @@ -44,6 +45,6 @@ class TestStrategyImplementSellTimeout(TestStrategyNoImplementSell): def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: return super().populate_exit_trend(dataframe, metadata) - def check_sell_timeout(self, pair: str, trade, order: dict, + def check_sell_timeout(self, pair: str, trade, order: Order, current_time: datetime, **kwargs) -> bool: return False diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 6f5ff573b..2d5121403 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -197,7 +197,7 @@ class StrategyTestV3(IStrategy): if current_profit < -0.0075: orders = trade.select_filled_orders(trade.entry_side) - return round(orders[0].cost, 0) + return round(orders[0].safe_cost, 0) return None From 8103656ae17e4e851584f2f9146491faf7529bf9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Mar 2023 06:36:03 +0100 Subject: [PATCH 41/42] Bump mypy in pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index acc0c8a17..565eb96f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: # stages: [push] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.991" + rev: "v1.0.1" hooks: - id: mypy exclude: build_helpers From 103bd9e2f217bbf277153f9f0fb63f25b30246d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Mar 2023 06:55:33 +0100 Subject: [PATCH 42/42] keep Trade.session private --- freqtrade/commands/db_commands.py | 2 +- freqtrade/persistence/models.py | 12 ++++++------ freqtrade/persistence/pairlock.py | 2 +- freqtrade/persistence/trade_model.py | 4 ++-- tests/persistence/test_migrations.py | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py index b4997582d..c424016b1 100644 --- a/freqtrade/commands/db_commands.py +++ b/freqtrade/commands/db_commands.py @@ -20,7 +20,7 @@ def start_convert_db(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) init_db(config['db_url']) - session_target = Trade.session + session_target = Trade._session init_db(config['db_url_from']) logger.info("Starting db migration.") diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f4058b4eb..d718af2f4 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -54,12 +54,12 @@ def init_db(db_url: str) -> None: # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope # Scoped sessions proxy requests to the appropriate thread-local session. # We should use the scoped_session object - not a seperately initialized version - Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=False)) - Order.session = Trade.session - PairLock.session = Trade.session - Trade.query = Trade.session.query_property() - Order.query = Trade.session.query_property() - PairLock.query = Trade.session.query_property() + Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=False)) + Order._session = Trade._session + PairLock._session = Trade._session + Trade.query = Trade._session.query_property() + Order.query = Trade._session.query_property() + PairLock.query = Trade._session.query_property() previous_tables = inspect(engine).get_table_names() ModelBase.metadata.create_all(engine) diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index d5a8d7ae1..a6d1eeaf0 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -15,7 +15,7 @@ class PairLock(ModelBase): """ __tablename__ = 'pairlocks' query: ClassVar[_QueryDescriptorType] - session: ClassVar[SessionType] + _session: ClassVar[SessionType] id: Mapped[int] = mapped_column(primary_key=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index a05eb7409..0ae5fba25 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -37,7 +37,7 @@ class Order(ModelBase): """ __tablename__ = 'orders' query: ClassVar[_QueryDescriptorType] - session: ClassVar[SessionType] + _session: ClassVar[SessionType] # Uniqueness should be ensured over pair, order_id # its likely that order_id is unique per Pair on some exchanges. @@ -1179,7 +1179,7 @@ class Trade(ModelBase, LocalTrade): """ __tablename__ = 'trades' query: ClassVar[_QueryDescriptorType] - session: ClassVar[SessionType] + _session: ClassVar[SessionType] use_db: bool = True diff --git a/tests/persistence/test_migrations.py b/tests/persistence/test_migrations.py index 5254164c1..2a6959d58 100644 --- a/tests/persistence/test_migrations.py +++ b/tests/persistence/test_migrations.py @@ -21,8 +21,8 @@ spot, margin, futures = TradingMode.SPOT, TradingMode.MARGIN, TradingMode.FUTURE def test_init_create_session(default_conf): # Check if init create a session init_db(default_conf['db_url']) - assert hasattr(Trade, 'session') - assert 'scoped_session' in type(Trade.session).__name__ + assert hasattr(Trade, '_session') + assert 'scoped_session' in type(Trade._session).__name__ def test_init_custom_db_url(default_conf, tmpdir): @@ -34,7 +34,7 @@ def test_init_custom_db_url(default_conf, tmpdir): init_db(default_conf['db_url']) assert Path(filename).is_file() - r = Trade.session.execute(text("PRAGMA journal_mode")) + r = Trade._session.execute(text("PRAGMA journal_mode")) assert r.first() == ('wal',)