merge upstream

freqtrade/persistence/migrations.py

@@ -3,11 +3,13 @@ from typing import List
 
 from sqlalchemy import inspect, text
 
+from freqtrade.exceptions import OperationalException
+
 
 logger = logging.getLogger(__name__)
 
 
-def get_table_names_for_table(inspector, tabletype):
+def get_table_names_for_table(inspector, tabletype) -> List[str]:
     return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
 
 
@@ -19,7 +21,7 @@ def get_column_def(columns: List, column: str, default: str) -> str:
     return default if not has_column(columns, column) else column
 
 
-def get_backup_name(tabs, backup_prefix: str):
+def get_backup_name(tabs: List[str], backup_prefix: str):
     table_back_name = backup_prefix
     for i, table_back_name in enumerate(tabs):
         table_back_name = f'{backup_prefix}{i}'
@@ -54,10 +56,22 @@ def set_sequence_ids(engine, order_id, trade_id):
                 connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
 
 
+def drop_index_on_table(engine, inspector, table_bak_name):
+    with engine.begin() as connection:
+        # drop indexes on backup table in new session
+        for index in inspector.get_indexes(table_bak_name):
+            if engine.name == 'mysql':
+                connection.execute(text(f"drop index {index['name']} on {table_bak_name}"))
+            else:
+                connection.execute(text(f"drop index {index['name']}"))
+
+
 def migrate_trades_and_orders_table(
         decl_base, inspector, engine,
         trade_back_name: str, cols: List,
         order_back_name: str, cols_order: List):
+    base_currency = get_column_def(cols, 'base_currency', 'null')
+    stake_currency = get_column_def(cols, 'stake_currency', 'null')
     fee_open = get_column_def(cols, 'fee_open', 'fee')
     fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
     fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
@@ -112,13 +126,7 @@ def migrate_trades_and_orders_table(
     with engine.begin() as connection:
         connection.execute(text(f"alter table trades rename to {trade_back_name}"))
 
-    with engine.begin() as connection:
-        # drop indexes on backup table in new session
-        for index in inspector.get_indexes(trade_back_name):
-            if engine.name == 'mysql':
-                connection.execute(text(f"drop index {index['name']} on {trade_back_name}"))
-            else:
-                connection.execute(text(f"drop index {index['name']}"))
+    drop_index_on_table(engine, inspector, trade_back_name)
 
     order_id, trade_id = get_last_sequence_ids(engine, trade_back_name, order_back_name)
 
@@ -130,7 +138,7 @@ def migrate_trades_and_orders_table(
     # Copy data back - following the correct schema
     with engine.begin() as connection:
         connection.execute(text(f"""insert into trades
-            (id, exchange, pair, is_open,
+            (id, exchange, pair, base_currency, stake_currency, is_open,
             fee_open, fee_open_cost, fee_open_currency,
             fee_close, fee_close_cost, fee_close_currency, open_rate,
             open_rate_requested, close_rate, close_rate_requested, close_profit,
@@ -142,7 +150,8 @@ def migrate_trades_and_orders_table(
             trading_mode, leverage, liquidation_price, is_short,
             interest_rate, funding_fees
             )
-        select id, lower(exchange), pair,
+        select id, lower(exchange), pair, {base_currency} base_currency,
+            {stake_currency} stake_currency,
             is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
             {fee_open_currency} fee_open_currency, {fee_close} fee_close,
             {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -154,10 +163,10 @@ def migrate_trades_and_orders_table(
             {initial_stop_loss_pct} initial_stop_loss_pct,
             {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
             {max_rate} max_rate, {min_rate} min_rate,
-            case when {exit_reason} == 'sell_signal' then 'exit_signal'
-                 when {exit_reason} == 'custom_sell' then 'custom_exit'
-                 when {exit_reason} == 'force_sell' then 'force_exit'
-                 when {exit_reason} == 'emergency_sell' then 'emergency_exit'
+            case when {exit_reason} = 'sell_signal' then 'exit_signal'
+                 when {exit_reason} = 'custom_sell' then 'custom_exit'
+                 when {exit_reason} = 'force_sell' then 'force_exit'
+                 when {exit_reason} = 'emergency_sell' then 'emergency_exit'
                  else {exit_reason}
             end exit_reason,
             {exit_order_status} exit_order_status,
@@ -173,23 +182,6 @@ def migrate_trades_and_orders_table(
     set_sequence_ids(engine, order_id, trade_id)
 
 
-def migrate_open_orders_to_trades(engine):
-    with engine.begin() as connection:
-        connection.execute(text("""
-        insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
-        select id ft_trade_id, pair ft_pair, open_order_id,
-            case when close_rate_requested is null then 'buy'
-            else 'sell' end ft_order_side, 1 ft_is_open
-        from trades
-        where open_order_id is not null
-        union all
-        select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
-            'stoploss' ft_order_side, 1 ft_is_open
-        from trades
-        where stoploss_order_id is not null
-        """))
-
-
 def drop_orders_table(engine, table_back_name: str):
     # Drop and recreate orders table as backup
     # This drops foreign keys, too.
@@ -207,7 +199,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
     # sqlite does not support literals for booleans
     with engine.begin() as connection:
         connection.execute(text(f"""
-            insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
+            insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
             status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
             order_date, order_filled_date, order_update_date, ft_fee_base)
             select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
@@ -217,6 +209,31 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
             """))
 
 
+def migrate_pairlocks_table(
+        decl_base, inspector, engine,
+        pairlock_back_name: str, cols: List):
+
+    # Schema migration necessary
+    with engine.begin() as connection:
+        connection.execute(text(f"alter table pairlocks rename to {pairlock_back_name}"))
+
+    drop_index_on_table(engine, inspector, pairlock_back_name)
+
+    side = get_column_def(cols, 'side', "'*'")
+
+    # let SQLAlchemy create the schema as required
+    decl_base.metadata.create_all(engine)
+    # Copy data back - following the correct schema
+    with engine.begin() as connection:
+        connection.execute(text(f"""insert into pairlocks
+        (id, pair, side, reason, lock_time,
+         lock_end_time, active)
+        select id, pair, {side} side, reason, lock_time,
+         lock_end_time, active
+        from {pairlock_back_name}
+        """))
+
+
 def set_sqlite_to_wal(engine):
     if engine.name == 'sqlite' and str(engine.url) != 'sqlite://':
         # Set Mode to
@@ -230,24 +247,38 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     """
     inspector = inspect(engine)
 
-    cols = inspector.get_columns('trades')
+    cols_trades = inspector.get_columns('trades')
     cols_orders = inspector.get_columns('orders')
+    cols_pairlocks = inspector.get_columns('pairlocks')
     tabs = get_table_names_for_table(inspector, 'trades')
     table_back_name = get_backup_name(tabs, 'trades_bak')
     order_tabs = get_table_names_for_table(inspector, 'orders')
     order_table_bak_name = get_backup_name(order_tabs, 'orders_bak')
+    pairlock_tabs = get_table_names_for_table(inspector, 'pairlocks')
+    pairlock_table_bak_name = get_backup_name(pairlock_tabs, 'pairlocks_bak')
 
     # Check if migration necessary
     # Migrates both trades and orders table!
     # if ('orders' not in previous_tables
     # or not has_column(cols_orders, 'leverage')):
-    if not has_column(cols, 'exit_order_status'):
+    if not has_column(cols_trades, 'base_currency'):
         logger.info(f"Running database migration for trades - "
                     f"backup: {table_back_name}, {order_table_bak_name}")
         migrate_trades_and_orders_table(
-            decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_orders)
+            decl_base, inspector, engine, table_back_name, cols_trades,
+            order_table_bak_name, cols_orders)
 
+    if not has_column(cols_pairlocks, 'side'):
+        logger.info(f"Running database migration for pairlocks - "
+                    f"backup: {pairlock_table_bak_name}")
+
+        migrate_pairlocks_table(
+            decl_base, inspector, engine, pairlock_table_bak_name, cols_pairlocks
+        )
     if 'orders' not in previous_tables and 'trades' in previous_tables:
-        logger.info('Moving open orders to Orders table.')
-        migrate_open_orders_to_trades(engine)
+        raise OperationalException(
+            "Your database seems to be very old. "
+            "Please update to freqtrade 2022.3 to migrate this database or "
+            "start with a fresh database.")
+
     set_sqlite_to_wal(engine)

freqtrade/persistence/models.py

@@ -8,13 +8,14 @@ from math import isclose
 from typing import Any, Dict, List, Optional
 
 from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String,
-                        create_engine, desc, func, inspect)
+                        create_engine, desc, func, inspect, or_)
 from sqlalchemy.exc import NoSuchModuleError
 from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker
 from sqlalchemy.pool import StaticPool
 from sqlalchemy.sql.schema import UniqueConstraint
 
-from freqtrade.constants import DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES
+from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES,
+                                 LongShort)
 from freqtrade.enums import ExitType, TradingMode
 from freqtrade.exceptions import DependencyException, OperationalException
 from freqtrade.leverage import interest
@@ -281,6 +282,8 @@ class LocalTrade():
 
     exchange: str = ''
     pair: str = ''
+    base_currency: str = ''
+    stake_currency: str = ''
     is_open: bool = True
     fee_open: float = 0.0
     fee_open_cost: Optional[float] = None
@@ -393,12 +396,32 @@ class LocalTrade():
             return "sell"
 
     @property
-    def trade_direction(self) -> str:
+    def trade_direction(self) -> LongShort:
         if self.is_short:
             return "short"
         else:
             return "long"
 
+    @property
+    def safe_base_currency(self) -> str:
+        """
+        Compatibility layer for asset - which can be empty for old trades.
+        """
+        try:
+            return self.base_currency or self.pair.split('/')[0]
+        except IndexError:
+            return ''
+
+    @property
+    def safe_quote_currency(self) -> str:
+        """
+        Compatibility layer for asset - which can be empty for old trades.
+        """
+        try:
+            return self.stake_currency or self.pair.split('/')[1].split(':')[0]
+        except IndexError:
+            return ''
+
     def __init__(self, **kwargs):
         for key in kwargs:
             setattr(self, key, kwargs[key])
@@ -409,12 +432,10 @@ class LocalTrade():
 
     def __repr__(self):
         open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
-        leverage = self.leverage or 1.0
-        is_short = self.is_short or False
 
         return (
             f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
-            f'is_short={is_short}, leverage={leverage}, '
+            f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, '
             f'open_rate={self.open_rate:.8f}, open_since={open_since})'
         )
 
@@ -425,6 +446,8 @@ class LocalTrade():
         return {
             'trade_id': self.id,
             'pair': self.pair,
+            'base_currency': self.safe_base_currency,
+            'quote_currency': self.safe_quote_currency,
             'is_open': self.is_open,
             'exchange': self.exchange,
             'amount': round(self.amount, 8),
@@ -1092,6 +1115,8 @@ class Trade(_DECL_BASE, LocalTrade):
 
     exchange = Column(String(25), nullable=False)
     pair = Column(String(25), nullable=False, index=True)
+    base_currency = Column(String(25), nullable=True)
+    stake_currency = Column(String(25), nullable=True)
     is_open = Column(Boolean, nullable=False, default=True, index=True)
     fee_open = Column(Float, nullable=False, default=0.0)
     fee_open_cost = Column(Float, nullable=True)
@@ -1445,6 +1470,8 @@ class PairLock(_DECL_BASE):
     id = Column(Integer, primary_key=True)
 
     pair = Column(String(25), nullable=False, index=True)
+    # lock direction - long, short or * (for both)
+    side = Column(String(25), nullable=False, default="*")
     reason = Column(String(255), nullable=True)
     # Time the pair was locked (start time)
     lock_time = Column(DateTime, nullable=False)
@@ -1456,11 +1483,12 @@ class PairLock(_DECL_BASE):
     def __repr__(self):
         lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
         lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
-        return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, '
-                f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
+        return (
+            f'PairLock(id={self.id}, pair={self.pair}, side={self.side}, lock_time={lock_time}, '
+            f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
 
     @staticmethod
-    def query_pair_locks(pair: Optional[str], now: datetime) -> Query:
+    def query_pair_locks(pair: Optional[str], now: datetime, side: str = '*') -> Query:
         """
         Get all currently active locks for this pair
         :param pair: Pair to check for. Returns all current locks if pair is empty
@@ -1471,6 +1499,11 @@ class PairLock(_DECL_BASE):
                    PairLock.active.is_(True), ]
         if pair:
             filters.append(PairLock.pair == pair)
+        if side != '*':
+            filters.append(or_(PairLock.side == side, PairLock.side == '*'))
+        else:
+            filters.append(PairLock.side == '*')
+
         return PairLock.query.filter(
             *filters
         )
@@ -1485,5 +1518,6 @@ class PairLock(_DECL_BASE):
             'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc
                                                                  ).timestamp() * 1000),
             'reason': self.reason,
+            'side': self.side,
             'active': self.active,
         }

freqtrade/persistence/pairlock_middleware.py

@@ -31,7 +31,7 @@ class PairLocks():
 
     @staticmethod
     def lock_pair(pair: str, until: datetime, reason: str = None, *,
-                  now: datetime = None) -> PairLock:
+                  now: datetime = None, side: str = '*') -> PairLock:
         """
         Create PairLock from now to "until".
         Uses database by default, unless PairLocks.use_db is set to False,
@@ -40,12 +40,14 @@ class PairLocks():
         :param until: End time of the lock. Will be rounded up to the next candle.
         :param reason: Reason string that will be shown as reason for the lock
         :param now: Current timestamp. Used to determine lock start time.
+        :param side: Side to lock pair, can be 'long', 'short' or '*'
         """
         lock = PairLock(
             pair=pair,
             lock_time=now or datetime.now(timezone.utc),
             lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until),
             reason=reason,
+            side=side,
             active=True
         )
         if PairLocks.use_db:
@@ -56,7 +58,8 @@ class PairLocks():
         return lock
 
     @staticmethod
-    def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:
+    def get_pair_locks(
+            pair: Optional[str], now: Optional[datetime] = None, side: str = '*') -> List[PairLock]:
         """
         Get all currently active locks for this pair
         :param pair: Pair to check for. Returns all current locks if pair is empty
@@ -67,26 +70,28 @@ class PairLocks():
             now = datetime.now(timezone.utc)
 
         if PairLocks.use_db:
-            return PairLock.query_pair_locks(pair, now).all()
+            return PairLock.query_pair_locks(pair, now, side).all()
         else:
             locks = [lock for lock in PairLocks.locks if (
                 lock.lock_end_time >= now
                 and lock.active is True
                 and (pair is None or lock.pair == pair)
+                and (lock.side == '*' or lock.side == side)
             )]
             return locks
 
     @staticmethod
-    def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]:
+    def get_pair_longest_lock(
+            pair: str, now: Optional[datetime] = None, side: str = '*') -> Optional[PairLock]:
         """
         Get the lock that expires the latest for the pair given.
         """
-        locks = PairLocks.get_pair_locks(pair, now)
+        locks = PairLocks.get_pair_locks(pair, now, side=side)
         locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True)
         return locks[0] if locks else None
 
     @staticmethod
-    def unlock_pair(pair: str, now: Optional[datetime] = None) -> None:
+    def unlock_pair(pair: str, now: Optional[datetime] = None, side: str = '*') -> None:
         """
         Release all locks for this pair.
         :param pair: Pair to unlock
@@ -97,7 +102,7 @@ class PairLocks():
             now = datetime.now(timezone.utc)
 
         logger.info(f"Releasing all locks for {pair}.")
-        locks = PairLocks.get_pair_locks(pair, now)
+        locks = PairLocks.get_pair_locks(pair, now, side=side)
         for lock in locks:
             lock.active = False
         if PairLocks.use_db:
@@ -134,7 +139,7 @@ class PairLocks():
                     lock.active = False
 
     @staticmethod
-    def is_global_lock(now: Optional[datetime] = None) -> bool:
+    def is_global_lock(now: Optional[datetime] = None, side: str = '*') -> bool:
         """
         :param now: Datetime object (generated via datetime.now(timezone.utc)).
             defaults to datetime.now(timezone.utc)
@@ -142,10 +147,10 @@ class PairLocks():
         if not now:
             now = datetime.now(timezone.utc)
 
-        return len(PairLocks.get_pair_locks('*', now)) > 0
+        return len(PairLocks.get_pair_locks('*', now, side)) > 0
 
     @staticmethod
-    def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool:
+    def is_pair_locked(pair: str, now: Optional[datetime] = None, side: str = '*') -> bool:
         """
         :param pair: Pair to check for
         :param now: Datetime object (generated via datetime.now(timezone.utc)).
@@ -154,7 +159,10 @@ class PairLocks():
         if not now:
             now = datetime.now(timezone.utc)
 
-        return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now)
+        return (
+            len(PairLocks.get_pair_locks(pair, now, side)) > 0
+            or PairLocks.is_global_lock(now, side)
+        )
 
     @staticmethod
     def get_all_locks() -> List[PairLock]: