From 7d03a067ee107a7f2cd3dc48653a61b82e1988da Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Aug 2020 08:33:46 +0200 Subject: [PATCH] Extract migrations ot seperate module --- freqtrade/persistence/migrations.py | 113 ++++++++++++++++++++++++++++ freqtrade/persistence/models.py | 112 +-------------------------- 2 files changed, 116 insertions(+), 109 deletions(-) create mode 100644 freqtrade/persistence/migrations.py diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py new file mode 100644 index 000000000..bd7dd103f --- /dev/null +++ b/freqtrade/persistence/migrations.py @@ -0,0 +1,113 @@ +import logging +from typing import List + +from sqlalchemy import inspect + +logger = logging.getLogger(__name__) + + +def has_column(columns: List, searchname: str) -> bool: + return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 + + +def get_column_def(columns: List, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + +def check_migrate(engine, decl_base) -> None: + """ + Checks if migration is necessary and migrates if necessary + """ + inspector = inspect(engine) + + cols = inspector.get_columns('trades') + tabs = inspector.get_table_names() + table_back_name = 'trades_bak' + for i, table_back_name in enumerate(tabs): + table_back_name = f'trades_bak{i}' + logger.debug(f'trying {table_back_name}') + + # Check for latest column + if not has_column(cols, 'amount_requested'): + logger.info(f'Running database migration - backup available as {table_back_name}') + + fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') + fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') + stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') + stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') + max_rate = get_column_def(cols, 'max_rate', '0.0') + min_rate = get_column_def(cols, 'min_rate', 'null') + sell_reason = get_column_def(cols, 'sell_reason', 'null') + strategy = get_column_def(cols, 'strategy', 'null') + # If ticker-interval existed use that, else null. + if has_column(cols, 'ticker_interval'): + timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') + else: + timeframe = get_column_def(cols, 'timeframe', 'null') + + open_trade_price = get_column_def(cols, 'open_trade_price', + f'amount * open_rate * (1 + {fee_open})') + close_profit_abs = get_column_def( + cols, 'close_profit_abs', + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') + amount_requested = get_column_def(cols, 'amount_requested', 'amount') + + # Schema migration necessary + engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute(f"""insert into trades + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, + stoploss_order_id, stoploss_last_update, + max_rate, min_rate, sell_reason, sell_order_status, strategy, + timeframe, open_trade_price, close_profit_abs + ) + select id, lower(exchange), + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, + {initial_stop_loss} initial_stop_loss, + {initial_stop_loss_pct} initial_stop_loss_pct, + {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, + {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, + {sell_order_status} sell_order_status, + {strategy} strategy, {timeframe} timeframe, + {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs + from {table_back_name} + """) + + # Reread columns - the above recreated the table! + inspector = inspect(engine) + cols = inspector.get_columns('trades') diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 28753ed48..f56ff47fe 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, desc, func, inspect) + create_engine, desc, func) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Query @@ -18,6 +18,7 @@ from sqlalchemy.pool import StaticPool from freqtrade.exceptions import OperationalException from freqtrade.misc import safe_value_fallback +from freqtrade.persistence.migrations import check_migrate logger = logging.getLogger(__name__) @@ -58,120 +59,13 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.query = Trade.session.query_property() _DECL_BASE.metadata.create_all(engine) - check_migrate(engine) + check_migrate(engine, decl_base=_DECL_BASE) # Clean dry_run DB if the db is not in-memory if clean_open_orders and db_url != 'sqlite://': clean_dry_run_db() -def has_column(columns: List, searchname: str) -> bool: - return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 - - -def get_column_def(columns: List, column: str, default: str) -> str: - return default if not has_column(columns, column) else column - - -def check_migrate(engine) -> None: - """ - Checks if migration is necessary and migrates if necessary - """ - inspector = inspect(engine) - - cols = inspector.get_columns('trades') - tabs = inspector.get_table_names() - table_back_name = 'trades_bak' - for i, table_back_name in enumerate(tabs): - table_back_name = f'trades_bak{i}' - logger.debug(f'trying {table_back_name}') - - # Check for latest column - if not has_column(cols, 'amount_requested'): - logger.info(f'Running database migration - backup available as {table_back_name}') - - fee_open = get_column_def(cols, 'fee_open', 'fee') - fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') - fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') - fee_close = get_column_def(cols, 'fee_close', 'fee') - fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') - fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') - open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') - close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') - stop_loss = get_column_def(cols, 'stop_loss', '0.0') - stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') - initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') - initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') - stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') - stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') - max_rate = get_column_def(cols, 'max_rate', '0.0') - min_rate = get_column_def(cols, 'min_rate', 'null') - sell_reason = get_column_def(cols, 'sell_reason', 'null') - strategy = get_column_def(cols, 'strategy', 'null') - # If ticker-interval existed use that, else null. - if has_column(cols, 'ticker_interval'): - timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') - else: - timeframe = get_column_def(cols, 'timeframe', 'null') - - open_trade_price = get_column_def(cols, 'open_trade_price', - f'amount * open_rate * (1 + {fee_open})') - close_profit_abs = get_column_def( - cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") - sell_order_status = get_column_def(cols, 'sell_order_status', 'null') - amount_requested = get_column_def(cols, 'amount_requested', 'amount') - - # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") - # let SQLAlchemy create the schema as required - _DECL_BASE.metadata.create_all(engine) - - # Copy data back - following the correct schema - engine.execute(f"""insert into trades - (id, exchange, pair, is_open, - fee_open, fee_open_cost, fee_open_currency, - fee_close, fee_close_cost, fee_open_currency, open_rate, - open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, amount_requested, open_date, close_date, open_order_id, - stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, - stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, sell_order_status, strategy, - timeframe, open_trade_price, close_profit_abs - ) - select id, lower(exchange), - case - when instr(pair, '_') != 0 then - substr(pair, instr(pair, '_') + 1) || '/' || - substr(pair, 1, instr(pair, '_') - 1) - else pair - end - pair, - is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, - {fee_open_currency} fee_open_currency, {fee_close} fee_close, - {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, - open_rate, {open_rate_requested} open_rate_requested, close_rate, - {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, - {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, - {initial_stop_loss} initial_stop_loss, - {initial_stop_loss_pct} initial_stop_loss_pct, - {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, - {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {sell_order_status} sell_order_status, - {strategy} strategy, {timeframe} timeframe, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs - from {table_back_name} - """) - - # Reread columns - the above recreated the table! - inspector = inspect(engine) - cols = inspector.get_columns('trades') - - def cleanup() -> None: """ Flushes all pending operations to disk.