From 850760bc00b68848cf929520a752d2767de20cfc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Apr 2022 17:37:04 +0200 Subject: [PATCH] Remove migration from very old database (database without Orders table) --- freqtrade/persistence/migrations.py | 28 +++----- tests/test_persistence.py | 100 +++++++++++++++++++++------- 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 996af7341..a28683e04 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -3,6 +3,8 @@ from typing import List from sqlalchemy import inspect, text +from freqtrade.exceptions import OperationalException + logger = logging.getLogger(__name__) @@ -176,23 +178,6 @@ def migrate_trades_and_orders_table( set_sequence_ids(engine, order_id, trade_id) -def migrate_open_orders_to_trades(engine): - with engine.begin() as connection: - connection.execute(text(""" - insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) - select id ft_trade_id, pair ft_pair, open_order_id, - case when close_rate_requested is null then 'buy' - else 'sell' end ft_order_side, 1 ft_is_open - from trades - where open_order_id is not null - union all - select id ft_trade_id, pair ft_pair, stoploss_order_id order_id, - 'stoploss' ft_order_side, 1 ft_is_open - from trades - where stoploss_order_id is not null - """)) - - def drop_orders_table(engine, table_back_name: str): # Drop and recreate orders table as backup # This drops foreign keys, too. @@ -210,7 +195,7 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List): # sqlite does not support literals for booleans with engine.begin() as connection: connection.execute(text(f""" - insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date, order_filled_date, order_update_date, ft_fee_base) select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, @@ -252,6 +237,9 @@ def check_migrate(engine, decl_base, previous_tables) -> None: order_table_bak_name, cols_orders) if 'orders' not in previous_tables and 'trades' in previous_tables: - logger.info('Moving open orders to Orders table.') - migrate_open_orders_to_trades(engine) + raise OperationalException( + "Your database seems to be very old. " + "Please update to freqtrade 2022.3 to migrate this database or " + "start with a fresh database.") + set_sqlite_to_wal(engine) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index d30d33d3b..ecac561f8 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1209,6 +1209,27 @@ def test_migrate_new(mocker, default_conf, fee, caplog): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" + create_table_order = """CREATE TABLE orders ( + id INTEGER NOT NULL, + ft_trade_id INTEGER, + ft_order_side VARCHAR(25) NOT NULL, + ft_pair VARCHAR(25) NOT NULL, + ft_is_open BOOLEAN NOT NULL, + order_id VARCHAR(255) NOT NULL, + status VARCHAR(255), + symbol VARCHAR(25), + order_type VARCHAR(50), + side VARCHAR(25), + price FLOAT, + amount FLOAT, + filled FLOAT, + remaining FLOAT, + cost FLOAT, + order_date DATETIME, + order_filled_date DATETIME, + order_update_date DATETIME, + PRIMARY KEY (id) + );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date, stop_loss, initial_stop_loss, max_rate, ticker_interval, @@ -1222,15 +1243,66 @@ def test_migrate_new(mocker, default_conf, fee, caplog): stake=default_conf.get("stake_amount"), amount=amount ) + insert_orders = f""" + insert into orders ( + ft_trade_id, + ft_order_side, + ft_pair, + ft_is_open, + order_id, + status, + symbol, + order_type, + side, + price, + amount, + filled, + remaining, + cost) + values ( + 1, + 'buy', + 'ETC/BTC', + 0, + 'buy_order', + 'closed', + 'ETC/BTC', + 'limit', + 'buy', + 0.00258580, + {amount}, + {amount}, + 0, + {amount * 0.00258580} + ), + ( + 1, + 'stoploss', + 'ETC/BTC', + 0, + 'stop_order_id222', + 'closed', + 'ETC/BTC', + 'limit', + 'sell', + 0.00258580, + {amount}, + {amount}, + 0, + {amount * 0.00258580} + ) + """ engine = create_engine('sqlite://') mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format with engine.begin() as connection: connection.execute(text(create_table_old)) + connection.execute(text(create_table_order)) connection.execute(text("create index ix_trades_is_open on trades(is_open)")) connection.execute(text("create index ix_trades_pair on trades(pair)")) connection.execute(text(insert_table_old)) + connection.execute(text(insert_orders)) # fake previous backup connection.execute(text("create table trades_bak as select * from trades")) @@ -1267,8 +1339,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.open_trade_value == trade._calc_open_trade_value() assert trade.close_profit_abs is None - assert log_has("Moving open orders to Orders table.", caplog) - orders = Order.query.all() + orders = trade.orders assert len(orders) == 2 assert orders[0].order_id == 'buy_order' assert orders[0].ft_order_side == 'buy' @@ -1277,7 +1348,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert orders[1].ft_order_side == 'stoploss' -def test_migrate_mid_state(mocker, default_conf, fee, caplog): +def test_migrate_too_old(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat) """ @@ -1301,6 +1372,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) VALUES ('binance', 'ETC/BTC', 1, {fee}, {fee}, @@ -1319,26 +1391,8 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): connection.execute(text(insert_table_old)) # Run init to test migration - init_db(default_conf['db_url'], default_conf['dry_run']) - - assert len(Trade.query.filter(Trade.id == 1).all()) == 1 - trade = Trade.query.filter(Trade.id == 1).first() - assert trade.fee_open == fee.return_value - assert trade.fee_close == fee.return_value - assert trade.open_rate_requested is None - assert trade.close_rate_requested is None - assert trade.is_open == 1 - assert trade.amount == amount - assert trade.stake_amount == default_conf.get("stake_amount") - assert trade.pair == "ETC/BTC" - assert trade.exchange == "binance" - assert trade.max_rate == 0.0 - assert trade.stop_loss == 0.0 - assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_value == trade._calc_open_trade_value() - assert log_has("trying trades_bak0", caplog) - assert log_has("Running database migration for trades - backup: trades_bak0, orders_bak0", - caplog) + with pytest.raises(OperationalException, match=r'Your database seems to be very old'): + init_db(default_conf['db_url'], default_conf['dry_run']) def test_migrate_get_last_sequence_ids():