From c5e3348b89d92087d3ad41cfb50486fbc8fae7ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 31 Jul 2021 16:36:28 +0200 Subject: [PATCH] Migrations for indexes should run in a seperate session closes #5349 --- docs/sql_cheatsheet.md | 2 +- freqtrade/persistence/migrations.py | 18 +++--- tests/test_persistence.py | 90 ----------------------------- 3 files changed, 8 insertions(+), 102 deletions(-) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 477396931..caa3f53a6 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -110,7 +110,7 @@ DELETE FROM trades WHERE id = 31; Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems. Installation: -`pip install psycopg2` +`pip install psycopg2-binary` Usage: `... --db-url postgresql+psycopg2://:@localhost:5432/` diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index a2d88cb31..1839c4130 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -65,7 +65,8 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col # Schema migration necessary with engine.begin() as connection: connection.execute(text(f"alter table trades rename to {table_back_name}")) - # drop indexes on backup table + with engine.begin() as connection: + # drop indexes on backup table in new session for index in inspector.get_indexes(table_back_name): connection.execute(text(f"drop index {index['name']}")) # let SQLAlchemy create the schema as required @@ -76,7 +77,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col connection.execute(text(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_open_cost, fee_open_currency, - fee_close, fee_close_cost, fee_open_currency, open_rate, + fee_close, fee_close_cost, fee_close_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, amount_requested, open_date, close_date, open_order_id, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, @@ -84,14 +85,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag, timeframe, open_trade_value, close_profit_abs ) - select id, lower(exchange), - case - when instr(pair, '_') != 0 then - substr(pair, instr(pair, '_') + 1) || '/' || - substr(pair, 1, instr(pair, '_') - 1) - else pair - end - pair, + select id, lower(exchange), pair, is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, {fee_open_currency} fee_open_currency, {fee_close} fee_close, {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, @@ -132,7 +126,9 @@ def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, col with engine.begin() as connection: connection.execute(text(f"alter table orders rename to {table_back_name}")) - # drop indexes on backup table + + with engine.begin() as connection: + # drop indexes on backup table in new session for index in inspector.get_indexes(table_back_name): connection.execute(text(f"drop index {index['name']}")) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 8b927be8b..84e7702e0 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -442,96 +442,6 @@ def test_clean_dry_run_db(default_conf, fee): assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1 -def test_migrate_old(mocker, default_conf, fee): - """ - Test Database migration(starting with old pairformat) - """ - amount = 103.223 - create_table_old = """CREATE TABLE IF NOT EXISTS "trades" ( - id INTEGER NOT NULL, - exchange VARCHAR NOT NULL, - pair VARCHAR NOT NULL, - is_open BOOLEAN NOT NULL, - fee FLOAT NOT NULL, - open_rate FLOAT, - close_rate FLOAT, - close_profit FLOAT, - stake_amount FLOAT NOT NULL, - amount FLOAT, - open_date DATETIME NOT NULL, - close_date DATETIME, - open_order_id VARCHAR, - PRIMARY KEY (id), - CHECK (is_open IN (0, 1)) - );""" - insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee, - open_rate, stake_amount, amount, open_date) - VALUES ('binance', 'BTC_ETC', 1, '123123', {fee}, - 0.00258580, {stake}, {amount}, - '2017-11-28 12:44:24.000000') - """.format(fee=fee.return_value, - stake=default_conf.get("stake_amount"), - amount=amount - ) - insert_table_old2 = """INSERT INTO trades (exchange, pair, is_open, fee, - open_rate, close_rate, stake_amount, amount, open_date) - VALUES ('binance', 'BTC_ETC', 0, {fee}, - 0.00258580, 0.00268580, {stake}, {amount}, - '2017-11-28 12:44:24.000000') - """.format(fee=fee.return_value, - stake=default_conf.get("stake_amount"), - amount=amount - ) - engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) - - # Create table using the old format - with engine.begin() as connection: - connection.execute(text(create_table_old)) - connection.execute(text(insert_table_old)) - connection.execute(text(insert_table_old2)) - # Run init to test migration - init_db(default_conf['db_url'], default_conf['dry_run']) - - assert len(Trade.query.filter(Trade.id == 1).all()) == 1 - trade = Trade.query.filter(Trade.id == 1).first() - assert trade.fee_open == fee.return_value - assert trade.fee_close == fee.return_value - assert trade.open_rate_requested is None - assert trade.close_rate_requested is None - assert trade.is_open == 1 - assert trade.amount == amount - assert trade.amount_requested == amount - assert trade.stake_amount == default_conf.get("stake_amount") - assert trade.pair == "ETC/BTC" - assert trade.exchange == "binance" - assert trade.max_rate == 0.0 - assert trade.stop_loss == 0.0 - assert trade.initial_stop_loss == 0.0 - assert trade.open_trade_value == trade._calc_open_trade_value() - assert trade.close_profit_abs is None - assert trade.fee_open_cost is None - assert trade.fee_open_currency is None - assert trade.fee_close_cost is None - assert trade.fee_close_currency is None - assert trade.timeframe is None - - trade = Trade.query.filter(Trade.id == 2).first() - assert trade.close_rate is not None - assert trade.is_open == 0 - assert trade.open_rate_requested is None - assert trade.close_rate_requested is None - assert trade.close_rate is not None - assert pytest.approx(trade.close_profit_abs) == trade.calc_profit() - assert trade.sell_order_status is None - - # Should've created one order - assert len(Order.query.all()) == 1 - order = Order.query.first() - assert order.order_id == '123123' - assert order.ft_order_side == 'buy' - - def test_migrate_new(mocker, default_conf, fee, caplog): """ Test Database migration (starting with new pairformat)