diff --git a/docs/utils.md b/docs/utils.md index 6c1b26b01..9b799e5fc 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -554,6 +554,27 @@ Show whitelist when using a [dynamic pairlist](plugins.md#pairlists). freqtrade test-pairlist --config config.json --quote USDT BTC ``` +## Convert database + +`freqtrade convert-db` can be used to convert your database from one system to another (sqlite -> postgres, postgres -> other postgres), migrating all trades, orders and Pairlocks. + +Please refer to the [SQL cheatsheet](sql_cheatsheet.md#use-a-different-database-system) to learn about requirements for different database systems. + +``` +usage: freqtrade convert-db [-h] [--db-url PATH] [--db-url-from PATH] + +optional arguments: + -h, --help show this help message and exit + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). + --db-url-from PATH Source db url to use when migrating a database. +``` + +!!! Warning + Please ensure to only use this on an empty target database. Freqtrade will perform a regular migration, but may fail if entries already existed. + ## Webserver mode !!! Warning "Experimental" diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 129836000..0e637c487 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -10,6 +10,7 @@ from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades, start_download_data, start_list_data) +from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index ff1d16590..815e28175 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -82,7 +82,9 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "timeframe", "plot_auto_open", ] -ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version'] +ARGS_CONVERT_DB = ["db_url", "db_url_from"] + +ARGS_INSTALL_UI = ["erase_ui_only", "ui_version"] ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] @@ -181,7 +183,7 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.commands import (start_backtesting, start_backtesting_show, - start_convert_data, start_convert_trades, + start_convert_data, start_convert_db, start_convert_trades, start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, @@ -374,6 +376,14 @@ class Arguments: test_pairlist_cmd.set_defaults(func=start_test_pairlist) self._build_args(optionlist=ARGS_TEST_PAIRLIST, parser=test_pairlist_cmd) + # Add db-convert subcommand + convert_db = subparsers.add_parser( + "convert-db", + help="Migrate database to different system", + ) + convert_db.set_defaults(func=start_convert_db) + self._build_args(optionlist=ARGS_CONVERT_DB, parser=convert_db) + # Add install-ui subcommand install_ui_cmd = subparsers.add_parser( 'install-ui', diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 58e208652..aac9f5713 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -106,6 +106,11 @@ AVAILABLE_CLI_OPTIONS = { f'`{constants.DEFAULT_DB_DRYRUN_URL}` for Dry Run).', metavar='PATH', ), + "db_url_from": Arg( + '--db-url-from', + help='Source db url to use when migrating a database.', + metavar='PATH', + ), "sd_notify": Arg( '--sd-notify', help='Notify systemd service manager.', diff --git a/freqtrade/commands/db_commands.py b/freqtrade/commands/db_commands.py new file mode 100644 index 000000000..d93aafcb6 --- /dev/null +++ b/freqtrade/commands/db_commands.py @@ -0,0 +1,55 @@ +import logging +from typing import Any, Dict + +from sqlalchemy import func + +from freqtrade.configuration.config_setup import setup_utils_configuration +from freqtrade.enums.runmode import RunMode + + +logger = logging.getLogger(__name__) + + +def start_convert_db(args: Dict[str, Any]) -> None: + from sqlalchemy.orm import make_transient + + from freqtrade.persistence import Order, Trade, init_db + from freqtrade.persistence.migrations import set_sequence_ids + from freqtrade.persistence.pairlock import PairLock + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + init_db(config['db_url'], False) + session_target = Trade._session + init_db(config['db_url_from'], False) + logger.info("Starting db migration.") + + trade_count = 0 + pairlock_count = 0 + for trade in Trade.get_trades(): + trade_count += 1 + make_transient(trade) + for o in trade.orders: + make_transient(o) + + session_target.add(trade) + + session_target.commit() + + for pairlock in PairLock.query: + pairlock_count += 1 + make_transient(pairlock) + session_target.add(pairlock) + session_target.commit() + + # Update sequences + max_trade_id = session_target.query(func.max(Trade.id)).scalar() + max_order_id = session_target.query(func.max(Order.id)).scalar() + max_pairlock_id = session_target.query(func.max(PairLock.id)).scalar() + + set_sequence_ids(session_target.get_bind(), + trade_id=max_trade_id, + order_id=max_order_id, + pairlock_id=max_pairlock_id) + + logger.info(f"Migrated {trade_count} Trades, and {pairlock_count} Pairlocks.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 80df6fb3f..96b585cd1 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -147,6 +147,9 @@ class Configuration: config.update({'db_url': self.args['db_url']}) logger.info('Parameter --db-url detected ...') + self._args_to_config(config, argname='db_url_from', + logstring='Parameter --db-url-from detected ...') + if config.get('force_entry_enable', False): logger.warning('`force_entry_enable` RPC message enabled.') diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 6a77b0b9a..53e35d9da 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -46,7 +46,7 @@ def get_last_sequence_ids(engine, trade_back_name, order_back_name): return order_id, trade_id -def set_sequence_ids(engine, order_id, trade_id): +def set_sequence_ids(engine, order_id, trade_id, pairlock_id=None): if engine.name == 'postgresql': with engine.begin() as connection: @@ -54,6 +54,9 @@ def set_sequence_ids(engine, order_id, trade_id): connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}")) if trade_id: connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}")) + if pairlock_id: + connection.execute( + text(f"ALTER SEQUENCE pairlocks_id_seq RESTART WITH {pairlock_id}")) def drop_index_on_table(engine, inspector, table_bak_name): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 37eeda86a..0932f4362 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,5 +1,6 @@ import json import re +from datetime import datetime from io import BytesIO from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -14,11 +15,14 @@ from freqtrade.commands import (start_backtesting_show, start_convert_data, star start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_strategy, start_show_trades, start_test_pairlist, start_trading, start_webserver) +from freqtrade.commands.db_commands import start_convert_db from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException +from freqtrade.persistence.models import init_db +from freqtrade.persistence.pairlock_middleware import PairLocks from tests.conftest import (CURRENT_TEST_STRATEGY, create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -1458,3 +1462,33 @@ def test_backtesting_show(mocker, testdatadir, capsys): assert sbr.call_count == 1 out, err = capsys.readouterr() assert "Pairs for Strategy" in out + + +def test_start_convert_db(mocker, fee, tmpdir, caplog): + db_src_file = Path(f"{tmpdir}/db.sqlite") + db_from = f"sqlite:///{db_src_file}" + db_target_file = Path(f"{tmpdir}/db_target.sqlite") + db_to = f"sqlite:///{db_target_file}" + args = [ + "convert-db", + "--db-url-from", + db_from, + "--db-url", + db_to, + ] + + assert not db_src_file.is_file() + init_db(db_from, False) + + create_mock_trades(fee) + + PairLocks.timeframe = '5m' + PairLocks.lock_pair('XRP/USDT', datetime.now(), 'Random reason 125', side='long') + assert db_src_file.is_file() + assert not db_target_file.is_file() + + pargs = get_args(args) + pargs['config'] = None + start_convert_db(pargs) + + assert db_target_file.is_file() diff --git a/tests/test_persistence.py b/tests/test_persistence.py index b66c12086..d84415938 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1416,14 +1416,14 @@ def test_migrate_set_sequence_ids(): engine = MagicMock() engine.begin = MagicMock() engine.name = 'postgresql' - set_sequence_ids(engine, 22, 55) + set_sequence_ids(engine, 22, 55, 5) assert engine.begin.call_count == 1 engine.reset_mock() engine.begin.reset_mock() engine.name = 'somethingelse' - set_sequence_ids(engine, 22, 55) + set_sequence_ids(engine, 22, 55, 6) assert engine.begin.call_count == 0