Merge pull request #6812 from freqtrade/db_migrate

Db migrate

docs/utils.md

@@ -554,6 +554,27 @@ Show whitelist when using a [dynamic pairlist](plugins.md#pairlists).
 freqtrade test-pairlist --config config.json --quote USDT BTC
 ```
 
+## Convert database
+
+`freqtrade convert-db` can be used to convert your database from one system to another (sqlite -> postgres, postgres -> other postgres), migrating all trades, orders and Pairlocks.
+
+Please refer to the [SQL cheatsheet](sql_cheatsheet.md#use-a-different-database-system) to learn about requirements for different database systems.
+
+```
+usage: freqtrade convert-db [-h] [--db-url PATH] [--db-url-from PATH]
+
+optional arguments:
+  -h, --help          show this help message and exit
+  --db-url PATH       Override trades database URL, this is useful in custom
+                      deployments (default: `sqlite:///tradesv3.sqlite` for
+                      Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for
+                      Dry Run).
+  --db-url-from PATH  Source db url to use when migrating a database.
+```
+
+!!! Warning
+    Please ensure to only use this on an empty target database. Freqtrade will perform a regular migration, but may fail if entries already existed.
+
 ## Webserver mode
 
 !!! Warning "Experimental"

freqtrade/commands/__init__.py

@@ -10,6 +10,7 @@ from freqtrade.commands.arguments import Arguments
 from freqtrade.commands.build_config_commands import start_new_config
 from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
                                               start_download_data, start_list_data)
+from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
                                                 start_new_strategy)
 from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show

freqtrade/commands/arguments.py

@@ -82,7 +82,9 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "timeframe", "plot_auto_open", ]
 
-ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version']
+ARGS_CONVERT_DB = ["db_url", "db_url_from"]
+
+ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"]
 
 ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
 
@@ -181,7 +183,7 @@ class Arguments:
         self._build_args(optionlist=['version'], parser=self.parser)
 
         from freqtrade.commands import (start_backtesting, start_backtesting_show,
-                                        start_convert_data, start_convert_trades,
+                                        start_convert_data, start_convert_db, start_convert_trades,
                                         start_create_userdir, start_download_data, start_edge,
                                         start_hyperopt, start_hyperopt_list, start_hyperopt_show,
                                         start_install_ui, start_list_data, start_list_exchanges,
@@ -374,6 +376,14 @@ class Arguments:
         test_pairlist_cmd.set_defaults(func=start_test_pairlist)
         self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd)
 
+        # Add db-convert subcommand
+        convert_db = subparsers.add_parser(
+            "convert-db",
+            help="Migrate database to different system",
+        )
+        convert_db.set_defaults(func=start_convert_db)
+        self._build_args(optionlist=ARGS_CONVERT_DB, parser=convert_db)
+
         # Add install-ui subcommand
         install_ui_cmd = subparsers.add_parser(
             'install-ui',

freqtrade/commands/cli_options.py

@@ -106,6 +106,11 @@ AVAILABLE_CLI_OPTIONS = {
         f'`{constants.DEFAULT_DB_DRYRUN_URL}` for Dry Run).',
         metavar='PATH',
     ),
+    "db_url_from": Arg(
+        '--db-url-from',
+        help='Source db url to use when migrating a database.',
+        metavar='PATH',
+    ),
     "sd_notify": Arg(
         '--sd-notify',
         help='Notify systemd service manager.',

freqtrade/commands/db_commands.py

@@ -0,0 +1,55 @@
+import logging
+from typing import Any, Dict
+
+from sqlalchemy import func
+
+from freqtrade.configuration.config_setup import setup_utils_configuration
+from freqtrade.enums.runmode import RunMode
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_convert_db(args: Dict[str, Any]) -> None:
+    from sqlalchemy.orm import make_transient
+
+    from freqtrade.persistence import Order, Trade, init_db
+    from freqtrade.persistence.migrations import set_sequence_ids
+    from freqtrade.persistence.pairlock import PairLock
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    init_db(config['db_url'], False)
+    session_target = Trade._session
+    init_db(config['db_url_from'], False)
+    logger.info("Starting db migration.")
+
+    trade_count = 0
+    pairlock_count = 0
+    for trade in Trade.get_trades():
+        trade_count += 1
+        make_transient(trade)
+        for o in trade.orders:
+            make_transient(o)
+
+        session_target.add(trade)
+
+    session_target.commit()
+
+    for pairlock in PairLock.query:
+        pairlock_count += 1
+        make_transient(pairlock)
+        session_target.add(pairlock)
+    session_target.commit()
+
+    # Update sequences
+    max_trade_id = session_target.query(func.max(Trade.id)).scalar()
+    max_order_id = session_target.query(func.max(Order.id)).scalar()
+    max_pairlock_id = session_target.query(func.max(PairLock.id)).scalar()
+
+    set_sequence_ids(session_target.get_bind(),
+                     trade_id=max_trade_id,
+                     order_id=max_order_id,
+                     pairlock_id=max_pairlock_id)
+
+    logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.")

freqtrade/configuration/configuration.py

@@ -147,6 +147,9 @@ class Configuration:
             config.update({'db_url': self.args['db_url']})
             logger.info('Parameter --db-url detected ...')
 
+        self._args_to_config(config, argname='db_url_from',
+                             logstring='Parameter --db-url-from detected ...')
+
         if config.get('force_entry_enable', False):
             logger.warning('`force_entry_enable` RPC message enabled.')
 

freqtrade/persistence/migrations.py

@@ -46,7 +46,7 @@ def get_last_sequence_ids(engine, trade_back_name, order_back_name):
     return order_id, trade_id
 
 
-def set_sequence_ids(engine, order_id, trade_id):
+def set_sequence_ids(engine, order_id, trade_id, pairlock_id=None):
 
     if engine.name == 'postgresql':
         with engine.begin() as connection:
@@ -54,6 +54,9 @@ def set_sequence_ids(engine, order_id, trade_id):
                 connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}"))
             if trade_id:
                 connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
+            if pairlock_id:
+                connection.execute(
+                    text(f"ALTER SEQUENCE pairlocks_id_seq RESTART WITH {pairlock_id}"))
 
 
 def drop_index_on_table(engine, inspector, table_bak_name):

tests/commands/test_commands.py

@@ -1,5 +1,6 @@
 import json
 import re
+from datetime import datetime
 from io import BytesIO
 from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
@@ -14,11 +15,14 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star
                                 start_list_exchanges, start_list_markets, start_list_strategies,
                                 start_list_timeframes, start_new_strategy, start_show_trades,
                                 start_test_pairlist, start_trading, start_webserver)
+from freqtrade.commands.db_commands import start_convert_db
 from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui,
                                                 get_ui_download_url, read_ui_version)
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
+from freqtrade.persistence.models import init_db
+from freqtrade.persistence.pairlock_middleware import PairLocks
 from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_args, log_has,
                             log_has_re, patch_exchange, patched_configuration_load_config_file)
 from tests.conftest_trades import MOCK_TRADE_COUNT
@@ -1458,3 +1462,33 @@ def test_backtesting_show(mocker, testdatadir, capsys):
     assert sbr.call_count == 1
     out, err = capsys.readouterr()
     assert "Pairs for Strategy" in out
+
+
+def test_start_convert_db(mocker, fee, tmpdir, caplog):
+    db_src_file = Path(f"{tmpdir}/db.sqlite")
+    db_from = f"sqlite:///{db_src_file}"
+    db_target_file = Path(f"{tmpdir}/db_target.sqlite")
+    db_to = f"sqlite:///{db_target_file}"
+    args = [
+        "convert-db",
+        "--db-url-from",
+        db_from,
+        "--db-url",
+        db_to,
+    ]
+
+    assert not db_src_file.is_file()
+    init_db(db_from, False)
+
+    create_mock_trades(fee)
+
+    PairLocks.timeframe = '5m'
+    PairLocks.lock_pair('XRP/USDT', datetime.now(), 'Random reason 125', side='long')
+    assert db_src_file.is_file()
+    assert not db_target_file.is_file()
+
+    pargs = get_args(args)
+    pargs['config'] = None
+    start_convert_db(pargs)
+
+    assert db_target_file.is_file()

tests/test_persistence.py

@@ -1416,14 +1416,14 @@ def test_migrate_set_sequence_ids():
     engine = MagicMock()
     engine.begin = MagicMock()
     engine.name = 'postgresql'
-    set_sequence_ids(engine, 22, 55)
+    set_sequence_ids(engine, 22, 55, 5)
 
     assert engine.begin.call_count == 1
     engine.reset_mock()
     engine.begin.reset_mock()
 
     engine.name = 'somethingelse'
-    set_sequence_ids(engine, 22, 55)
+    set_sequence_ids(engine, 22, 55, 6)
 
     assert engine.begin.call_count == 0