Merge branch 'develop' into feat/short

freqtrade/persistence/migrations.py

@@ -28,7 +28,36 @@ def get_backup_name(tabs, backup_prefix: str):
     return table_back_name
 
 
-def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
+def get_last_sequence_ids(engine, trade_back_name, order_back_name):
+    order_id: int = None
+    trade_id: int = None
+
+    if engine.name == 'postgresql':
+        with engine.begin() as connection:
+            trade_id = connection.execute(text("select nextval('trades_id_seq')")).fetchone()[0]
+            order_id = connection.execute(text("select nextval('orders_id_seq')")).fetchone()[0]
+        with engine.begin() as connection:
+            connection.execute(text(
+                f"ALTER SEQUENCE orders_id_seq rename to {order_back_name}_id_seq_bak"))
+            connection.execute(text(
+                f"ALTER SEQUENCE trades_id_seq rename to {trade_back_name}_id_seq_bak"))
+    return order_id, trade_id
+
+
+def set_sequence_ids(engine, order_id, trade_id):
+
+    if engine.name == 'postgresql':
+        with engine.begin() as connection:
+            if order_id:
+                connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}"))
+            if trade_id:
+                connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
+
+
+def migrate_trades_and_orders_table(
+        decl_base, inspector, engine,
+        trade_back_name: str, cols: List,
+        order_back_name: str, cols_order: List):
     fee_open = get_column_def(cols, 'fee_open', 'fee')
     fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
     fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
@@ -79,11 +108,20 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
 
     # Schema migration necessary
     with engine.begin() as connection:
-        connection.execute(text(f"alter table trades rename to {table_back_name}"))
+        connection.execute(text(f"alter table trades rename to {trade_back_name}"))
+
     with engine.begin() as connection:
         # drop indexes on backup table in new session
-        for index in inspector.get_indexes(table_back_name):
-            connection.execute(text(f"drop index {index['name']}"))
+        for index in inspector.get_indexes(trade_back_name):
+            if engine.name == 'mysql':
+                connection.execute(text(f"drop index {index['name']} on {trade_back_name}"))
+            else:
+                connection.execute(text(f"drop index {index['name']}"))
+
+    order_id, trade_id = get_last_sequence_ids(engine, trade_back_name, order_back_name)
+
+    drop_orders_table(engine, order_back_name)
+
     # let SQLAlchemy create the schema as required
     decl_base.metadata.create_all(engine)
 
@@ -120,9 +158,12 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
             {trading_mode} trading_mode, {leverage} leverage, {isolated_liq} isolated_liq,
             {is_short} is_short, {interest_rate} interest_rate,
             {funding_fees} funding_fees
-            from {table_back_name}
+            from {trade_back_name}
             """))
 
+    migrate_orders_table(engine, order_back_name, cols_order)
+    set_sequence_ids(engine, order_id, trade_id)
+
 
 def migrate_open_orders_to_trades(engine):
     with engine.begin() as connection:
@@ -141,19 +182,18 @@ def migrate_open_orders_to_trades(engine):
         """))
 
 
-def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List):
-    # Schema migration necessary
+def drop_orders_table(engine, table_back_name: str):
+    # Drop and recreate orders table as backup
+    # This drops foreign keys, too.
 
     with engine.begin() as connection:
-        connection.execute(text(f"alter table orders rename to {table_back_name}"))
+        connection.execute(text(f"create table {table_back_name} as select * from orders"))
+        connection.execute(text("drop table orders"))
 
-    with engine.begin() as connection:
-        # drop indexes on backup table in new session
-        for index in inspector.get_indexes(table_back_name):
-            connection.execute(text(f"drop index {index['name']}"))
+
+def migrate_orders_table(engine, table_back_name: str, cols: List):
 
     # let SQLAlchemy create the schema as required
-    decl_base.metadata.create_all(engine)
     leverage = get_column_def(cols, 'leverage', '1.0')
     # sqlite does not support literals for booleans
     with engine.begin() as connection:
@@ -176,12 +216,18 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
 
     cols = inspector.get_columns('trades')
     tabs = get_table_names_for_table(inspector, 'trades')
+    cols_order = inspector.get_columns('orders')
     table_back_name = get_backup_name(tabs, 'trades_bak')
+    order_tabs = get_table_names_for_table(inspector, 'orders')
+    order_table_bak_name = get_backup_name(order_tabs, 'orders_bak')
 
-    # Check for latest column
+    # Check if migration necessary
+    # Migrates both trades and orders table!
     if not has_column(cols, 'enter_tag'):
-        logger.info(f'Running database migration for trades - backup: {table_back_name}')
-        migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
+        logger.info(f"Running database migration for trades - "
+                    f"backup: {table_back_name}, {order_table_bak_name}")
+        migrate_trades_and_orders_table(
+            decl_base, inspector, engine, table_back_name, cols, order_table_bak_name, cols_order)
         # Reread columns - the above recreated the table!
         inspector = inspect(engine)
         cols = inspector.get_columns('trades')
@@ -189,14 +235,3 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
     if 'orders' not in previous_tables and 'trades' in previous_tables:
         logger.info('Moving open orders to Orders table.')
         migrate_open_orders_to_trades(engine)
-    else:
-        cols_order = inspector.get_columns('orders')
-
-        # Last added column of order table
-        # To determine if migrations need to run
-        if not has_column(cols_order, 'leverage'):
-            tabs = get_table_names_for_table(inspector, 'orders')
-            # Empty for now - as there is only one iteration of the orders table so far.
-            table_back_name = get_backup_name(tabs, 'orders_bak')
-
-            migrate_orders_table(decl_base, inspector, engine, table_back_name, cols_order)

freqtrade/persistence/models.py

@@ -132,9 +132,12 @@ class Order(_DECL_BASE):
     order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
     order_filled_date = Column(DateTime, nullable=True)
     order_update_date = Column(DateTime, nullable=True)
-
     leverage = Column(Float, nullable=True, default=1.0)
 
+    @property
+    def order_date_utc(self):
+        return self.order_date.replace(tzinfo=timezone.utc)
+
     def __repr__(self):
 
         return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
@@ -170,6 +173,35 @@ class Order(_DECL_BASE):
                 self.order_filled_date = datetime.now(timezone.utc)
         self.order_update_date = datetime.now(timezone.utc)
 
+    def to_json(self) -> Dict[str, Any]:
+        return {
+            'amount': self.amount,
+            'average': round(self.average, 8) if self.average else 0,
+            'cost': self.cost if self.cost else 0,
+            'filled': self.filled,
+            'ft_order_side': self.ft_order_side,
+            'is_open': self.ft_is_open,
+            'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT)
+            if self.order_date else None,
+            'order_timestamp': int(self.order_date.replace(
+                tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None,
+            'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT)
+            if self.order_filled_date else None,
+            'order_filled_timestamp': int(self.order_filled_date.replace(
+                tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None,
+            'order_type': self.order_type,
+            'pair': self.ft_pair,
+            'price': self.price,
+            'remaining': self.remaining,
+            'status': self.status,
+        }
+
+    def close_bt_order(self, close_date: datetime):
+        self.order_filled_date = close_date
+        self.filled = self.amount
+        self.status = 'closed'
+        self.ft_is_open = False
+
     @staticmethod
     def update_orders(orders: List['Order'], order: Dict[str, Any]):
         """
@@ -390,6 +422,16 @@ class LocalTrade():
         )
 
     def to_json(self) -> Dict[str, Any]:
+        filled_orders = self.select_filled_orders()
+        filled_entries = []
+        filled_exits = []
+        if len(filled_orders) > 0:
+            for order in filled_orders:
+                if order.ft_order_side == 'buy':
+                    filled_entries.append(order.to_json())
+                if order.ft_order_side == 'sell':
+                    filled_exits.append(order.to_json())
+
         return {
             'trade_id': self.id,
             'pair': self.pair,
@@ -460,6 +502,8 @@ class LocalTrade():
             'trading_mode': self.trading_mode,
             'funding_fees': self.funding_fees,
             'open_order_id': self.open_order_id,
+            'filled_entry_orders': filled_entries,
+            'filled_exit_orders': filled_exits,
         }
 
     @staticmethod
@@ -794,8 +838,8 @@ class LocalTrade():
         return float(f"{profit_ratio:.8f}")
 
     def recalc_trade_from_orders(self):
-        # We need at least 2 orders for averaging amounts and rates.
-        if len(self.orders) < 2:
+        # We need at least 2 entry orders for averaging amounts and rates.
+        if len(self.select_filled_orders('buy')) < 2:
             # Just in case, still recalc open trade value
             self.recalc_open_trade_value()
             return
@@ -825,14 +869,27 @@ class LocalTrade():
             if self.stop_loss_pct is not None and self.open_rate is not None:
                 self.adjust_stop_loss(self.open_rate, self.stop_loss_pct)
 
-    def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]:
+    def select_order_by_order_id(self, order_id: str) -> Optional[Order]:
+        """
+        Finds order object by Order id.
+        :param order_id: Exchange order id
+        """
+        for o in self.orders:
+            if o.order_id == order_id:
+                return o
+        return None
+
+    def select_order(
+            self, order_side: str = None, is_open: Optional[bool] = None) -> Optional[Order]:
         """
         Finds latest order for this orderside and status
         :param order_side: Side of the order (either 'buy' or 'sell')
         :param is_open: Only search for open orders?
         :return: latest Order object if it exists, else None
         """
-        orders = [o for o in self.orders if o.side == order_side]
+        orders = self.orders
+        if order_side:
+            orders = [o for o in self.orders if o.side == order_side]
         if is_open is not None:
             orders = [o for o in orders if o.ft_is_open == is_open]
         if len(orders) > 0:
@@ -840,14 +897,14 @@ class LocalTrade():
         else:
             return None
 
-    def select_filled_orders(self, order_side: str) -> List['Order']:
+    def select_filled_orders(self, order_side: Optional[str] = None) -> List['Order']:
         """
         Finds filled orders for this orderside.
-        :param order_side: Side of the order (either 'buy' or 'sell')
+        :param order_side: Side of the order (either 'buy', 'sell', or None)
         :return: array of Order objects
         """
-        return [o for o in self.orders if o.ft_order_side == order_side and
-                o.ft_is_open is False and
+        return [o for o in self.orders if ((o.ft_order_side == order_side) or (order_side is None))
+                and o.ft_is_open is False and
                 (o.filled or 0) > 0 and
                 o.status in NON_OPEN_EXCHANGE_STATES]