Merge pull request #6812 from freqtrade/db_migrate

Db migrate

freqtrade/commands/__init__.py

@@ -10,6 +10,7 @@ from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
 from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
                                               start_download_data, start_list_data)
+from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show

freqtrade/commands/arguments.py

@@ -82,7 +82,9 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "timeframe", "plot_auto_open", ]
 
-ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version']
+ARGS_CONVERT_DB = ["db_url", "db_url_from"]
+
+ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"]
 
 ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
 
@@ -181,7 +183,7 @@ class Arguments:
         self._build_args(optionlist=['version'], parser=self.parser)
 
         from freqtrade.commands import (start_backtesting, start_backtesting_show,
-                                        start_convert_data, start_convert_trades,
+                                        start_convert_data, start_convert_db, start_convert_trades,
                                         start_create_userdir, start_download_data, start_edge,
                                         start_hyperopt, start_hyperopt_list, start_hyperopt_show,
                                         start_install_ui, start_list_data, start_list_exchanges,
@@ -374,6 +376,14 @@ class Arguments:
         test_pairlist_cmd.set_defaults(func=start_test_pairlist)
         self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd)
 
+        # Add db-convert subcommand
+        convert_db = subparsers.add_parser(
+            "convert-db",
+            help="Migrate database to different system",
+        )
+        convert_db.set_defaults(func=start_convert_db)
+        self._build_args(optionlist=ARGS_CONVERT_DB, parser=convert_db)
+
         # Add install-ui subcommand
         install_ui_cmd = subparsers.add_parser(
             'install-ui',

freqtrade/commands/cli_options.py

@@ -106,6 +106,11 @@ AVAILABLE_CLI_OPTIONS = {
         f'`{constants.DEFAULT_DB_DRYRUN_URL}` for Dry Run).',
         metavar='PATH',
     ),
+    "db_url_from": Arg(
+        '--db-url-from',
+        help='Source db url to use when migrating a database.',
+        metavar='PATH',
+    ),
     "sd_notify": Arg(
         '--sd-notify',
         help='Notify systemd service manager.',

freqtrade/commands/db_commands.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Any, Dict
+
+from sqlalchemy import func
+
+from freqtrade.configuration.config_setup import setup_utils_configuration
+from freqtrade.enums.runmode import RunMode
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_convert_db(args: Dict[str, Any]) -> None:
+    from sqlalchemy.orm import make_transient
+
+    from freqtrade.persistence import Order, Trade, init_db
+    from freqtrade.persistence.migrations import set_sequence_ids
+    from freqtrade.persistence.pairlock import PairLock
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    init_db(config['db_url'], False)
+    session_target = Trade._session
+    init_db(config['db_url_from'], False)
+    logger.info("Starting db migration.")
+
+    trade_count = 0
+    pairlock_count = 0
+    for trade in Trade.get_trades():
+        trade_count += 1
+        make_transient(trade)
+        for o in trade.orders:
+            make_transient(o)
+
+        session_target.add(trade)
+
+    session_target.commit()
+
+    for pairlock in PairLock.query:
+        pairlock_count += 1
+        make_transient(pairlock)
+        session_target.add(pairlock)
+    session_target.commit()
+
+    # Update sequences
+    max_trade_id = session_target.query(func.max(Trade.id)).scalar()
+    max_order_id = session_target.query(func.max(Order.id)).scalar()
+    max_pairlock_id = session_target.query(func.max(PairLock.id)).scalar()
+
+    set_sequence_ids(session_target.get_bind(),
+                     trade_id=max_trade_id,
+                     order_id=max_order_id,
+                     pairlock_id=max_pairlock_id)
+
+    logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.")

freqtrade/configuration/configuration.py

@@ -147,6 +147,9 @@ class Configuration:
             config.update({'db_url': self.args['db_url']})
             logger.info('Parameter --db-url detected ...')
 
+        self._args_to_config(config, argname='db_url_from',
+                             logstring='Parameter --db-url-from detected ...')
+
         if config.get('force_entry_enable', False):
             logger.warning('`force_entry_enable` RPC message enabled.')
 

freqtrade/persistence/migrations.py

@@ -46,7 +46,7 @@ def get_last_sequence_ids(engine, trade_back_name, order_back_name):
     return order_id, trade_id
 
 
-def set_sequence_ids(engine, order_id, trade_id):
+def set_sequence_ids(engine, order_id, trade_id, pairlock_id=None):
 
     if engine.name == 'postgresql':
         with engine.begin() as connection:
@@ -54,6 +54,9 @@ def set_sequence_ids(engine, order_id, trade_id):
                 connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}"))
             if trade_id:
                 connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
+            if pairlock_id:
+                connection.execute(
+                    text(f"ALTER SEQUENCE pairlocks_id_seq RESTART WITH {pairlock_id}"))
 
 
 def drop_index_on_table(engine, inspector, table_bak_name):