Update migrations to populate Orders table for open orders

freqtrade/persistence/migrations.py

@@ -5,6 +5,7 @@ from sqlalchemy import inspect
 
 logger = logging.getLogger(__name__)
 
+
 def get_table_names_for_table(inspector, tabletype):
     return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
 
@@ -17,23 +18,16 @@ def get_column_def(columns: List, column: str, default: str) -> str:
     return default if not has_column(columns, column) else column
 
 
-def check_migrate(engine, decl_base) -> None:
+def get_backup_name(tabs, backup_prefix: str):
-    """
+    table_back_name = backup_prefix
-    Checks if migration is necessary and migrates if necessary
-    """
-    inspector = inspect(engine)
-
-    cols = inspector.get_columns('trades')
-    tabs = get_table_names_for_table(inspector, 'trades')
-    table_back_name = 'trades_bak'
     for i, table_back_name in enumerate(tabs):
-        table_back_name = f'trades_bak{i}'
+        table_back_name = f'{backup_prefix}{i}'
         logger.debug(f'trying {table_back_name}')
 
-    # Check for latest column
+    return table_back_name
-    if not has_column(cols, 'amount_requested'):
-        logger.info(f'Running database migration - backup available as {table_back_name}')
 
+
+def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
     fee_open = get_column_def(cols, 'fee_open', 'fee')
     fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
     fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
@@ -111,6 +105,45 @@ def check_migrate(engine, decl_base) -> None:
             from {table_back_name}
             """)
 
+
+def migrate_open_orders_to_trades(engine):
+    engine.execute("""
+    insert into orders (trade_id, order_id, ft_order_side)
+    select id, open_order_id,
+        case when close_rate_requested is null then 'buy'
+        else 'sell' end ft_order_side
+    from trades
+    where open_order_id is not null
+    union all
+    select id, stoploss_order_id, 'stoploss'
+    from trades
+    where stoploss_order_id is not null
+    """)
+
+
+def check_migrate(engine, decl_base, previous_tables) -> None:
+    """
+    Checks if migration is necessary and migrates if necessary
+    """
+    inspector = inspect(engine)
+
+    cols = inspector.get_columns('trades')
+    tabs = get_table_names_for_table(inspector, 'trades')
+    table_back_name = get_backup_name(tabs, 'trades_bak')
+
+    # Check for latest column
+    if not has_column(cols, 'amount_requested'):
+        logger.info(f'Running database migration for trades - backup: {table_back_name}')
+        migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
         # Reread columns - the above recreated the table!
         inspector = inspect(engine)
         cols = inspector.get_columns('trades')
+
+    if 'orders' not in previous_tables:
+        logger.info('Moving open orders to Orders table.')
+        migrate_open_orders_to_trades(engine)
+    else:
+        logger.info(f'Running database migration for orders - backup: {table_back_name}')
+        pass
+        # Empty for now - as there is only one iteration of the orders table so far.
+        # table_back_name = get_backup_name(tabs, 'orders_bak')

freqtrade/persistence/models.py

@@ -7,8 +7,8 @@ from decimal import Decimal
 from typing import Any, Dict, List, Optional
 
 import arrow
-from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, ForeignKey,
+from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
-                        create_engine, desc, func)
+                        String, create_engine, desc, func, inspect)
 from sqlalchemy.exc import NoSuchModuleError
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import Query, relationship
@@ -61,9 +61,9 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
     # Copy session attributes to order object too
     Order.session = Trade.session
     Order.query = Order.session.query_property()
-
+    previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)
-    check_migrate(engine, decl_base=_DECL_BASE)
+    check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
 
     # Clean dry_run DB if the db is not in-memory
     if clean_open_orders and db_url != 'sqlite://':
@@ -117,7 +117,7 @@ class Order(_DECL_BASE):
     filled = Column(Float, nullable=True)
     remaining = Column(Float, nullable=True)
     cost = Column(Float, nullable=True)
-    order_date = Column(DateTime, nullable=False, default=datetime.utcnow)
+    order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
     order_filled_date = Column(DateTime, nullable=True)
     order_update_date = Column(DateTime, nullable=True)
 

tests/test_persistence.py

@@ -8,7 +8,7 @@ from sqlalchemy import create_engine
 
 from freqtrade import constants
 from freqtrade.exceptions import OperationalException
-from freqtrade.persistence import Trade, clean_dry_run_db, init
+from freqtrade.persistence import Trade, Order, clean_dry_run_db, init
 from tests.conftest import log_has, create_mock_trades
 
 
@@ -421,9 +421,9 @@ def test_migrate_old(mocker, default_conf, fee):
                                 PRIMARY KEY (id),
                                 CHECK (is_open IN (0, 1))
                                 );"""
-    insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
+    insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee,
                           open_rate, stake_amount, amount, open_date)
-                          VALUES ('BITTREX', 'BTC_ETC', 1, {fee},
+                          VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee},
                           0.00258580, {stake}, {amount},
                           '2017-11-28 12:44:24.000000')
                           """.format(fee=fee.return_value,
@@ -481,6 +481,12 @@ def test_migrate_old(mocker, default_conf, fee):
     assert pytest.approx(trade.close_profit_abs) == trade.calc_profit()
     assert trade.sell_order_status is None
 
+    # Should've created one order
+    assert len(Order.query.all()) == 1
+    order = Order.query.first()
+    assert order.order_id == '123123'
+    assert order.ft_order_side == 'buy'
+
 
 def test_migrate_new(mocker, default_conf, fee, caplog):
     """
@@ -509,16 +515,19 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
                                 sell_reason VARCHAR,
                                 strategy VARCHAR,
                                 ticker_interval INTEGER,
+                                stoploss_order_id VARCHAR,
                                 PRIMARY KEY (id),
                                 CHECK (is_open IN (0, 1))
                                 );"""
     insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
                           open_rate, stake_amount, amount, open_date,
-                          stop_loss, initial_stop_loss, max_rate, ticker_interval)
+                          stop_loss, initial_stop_loss, max_rate, ticker_interval,
+                          open_order_id, stoploss_order_id)
                           VALUES ('binance', 'ETC/BTC', 1, {fee},
                           0.00258580, {stake}, {amount},
                           '2019-11-28 12:44:24.000000',
-                          0.0, 0.0, 0.0, '5m')
+                          0.0, 0.0, 0.0, '5m',
+                          'buy_order', 'stop_order_id222')
                           """.format(fee=fee.return_value,
                                      stake=default_conf.get("stake_amount"),
                                      amount=amount
@@ -558,14 +567,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.sell_reason is None
     assert trade.strategy is None
     assert trade.timeframe == '5m'
-    assert trade.stoploss_order_id is None
+    assert trade.stoploss_order_id == 'stop_order_id222'
     assert trade.stoploss_last_update is None
     assert log_has("trying trades_bak1", caplog)
     assert log_has("trying trades_bak2", caplog)
-    assert log_has("Running database migration - backup available as trades_bak2", caplog)
+    assert log_has("Running database migration for trades - backup: trades_bak2", caplog)
     assert trade.open_trade_price == trade._calc_open_trade_price()
     assert trade.close_profit_abs is None
 
+    assert log_has("Moving open orders to Orders table.", caplog)
+    orders = Order.query.all()
+    assert len(orders) == 2
+    assert orders[0].order_id == 'buy_order'
+    assert orders[0].ft_order_side == 'buy'
+
+    assert orders[1].order_id == 'stop_order_id222'
+    assert orders[1].ft_order_side == 'stoploss'
+
 
 def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     """
@@ -626,7 +644,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     assert trade.initial_stop_loss == 0.0
     assert trade.open_trade_price == trade._calc_open_trade_price()
     assert log_has("trying trades_bak0", caplog)
-    assert log_has("Running database migration - backup available as trades_bak0", caplog)
+    assert log_has("Running database migration for trades - backup: trades_bak0", caplog)
 
 
 def test_adjust_stop_loss(fee):