Merge pull request #864 from freqtrade/feature/overhaul-db-handling

Allow custom sqlite database path

config_full.json.example

@@ -45,6 +45,7 @@
         "token": "your_telegram_token",
         "chat_id": "your_telegram_chat_id"
     },
+    "db_url": "sqlite:///tradesv3.sqlite",
     "initial_state": "running",
     "internals": {
         "process_throttle_secs": 5

docs/bot-usage.md

@@ -9,9 +9,9 @@ it.
 
 ## Bot commands
 ```
-usage: main.py [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME]
+usage: freqtrade [-h] [-v] [--version] [-c PATH] [-d PATH] [-s NAME]
                  [--strategy-path PATH] [--dynamic-whitelist [INT]]
-               [--dry-run-db]
+                 [--db-url PATH]
                  {backtesting,hyperopt} ...
 
 Simple High Frequency Trading Bot for crypto currencies
@@ -28,17 +28,16 @@ optional arguments:
   -c PATH, --config PATH
                         specify configuration file (default: config.json)
   -d PATH, --datadir PATH
-                        path to backtest data (default:
+                        path to backtest data
-                        freqtrade/tests/testdata
   -s NAME, --strategy NAME
                         specify strategy class name (default: DefaultStrategy)
   --strategy-path PATH  specify additional strategy lookup path
   --dynamic-whitelist [INT]
                         dynamically generate and update whitelist based on 24h
-                        BaseVolume (Default 20 currencies)
+                        BaseVolume (default: 20)
-  --dry-run-db          Force dry run to use a local DB
+  --db-url PATH         Override trades database URL, this is useful if
-                        "tradesv3.dry_run.sqlite" instead of memory DB. Work
+                        dry_run is enabled or in custom deployments (default:
-                        only if dry_run is enabled.
+                        sqlite:///tradesv3.sqlite)
 ```
 
 ### How to use a different config file?
@@ -102,14 +101,14 @@ python3 ./freqtrade/main.py --dynamic-whitelist 30
 negative value (e.g -2), `--dynamic-whitelist` will use the default
 value (20).
 
-### How to use --dry-run-db?
+### How to use --db-url?
 When you run the bot in Dry-run mode, per default no transactions are 
 stored in a database. If you want to store your bot actions in a DB 
-using `--dry-run-db`. This command will use a separate database file 
+using `--db-url`. This can also be used to specify a custom database
-`tradesv3.dry_run.sqlite`
+in production mode. Example command:
 
 ```bash
-python3 ./freqtrade/main.py -c config.json --dry-run-db
+python3 ./freqtrade/main.py -c config.json --db-url sqlite:///tradesv3.dry_run.sqlite
 ```
 
 

docs/configuration.md

@@ -34,6 +34,7 @@ The table below will list all configuration parameters.
 | `telegram.enabled` | true | Yes | Enable or not the usage of Telegram.
 | `telegram.token` | token | No | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
 | `telegram.chat_id` | chat_id | No | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
+| `db_url` | `sqlite:///tradesv3.sqlite` | No | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `True`.
 | `initial_state` | running | No | Defines the initial application state. More information below.
 | `strategy` | DefaultStrategy | No | Defines Strategy class to use.
 | `strategy_path` | null | No | Adds an additional strategy lookup path (must be a folder).
@@ -111,9 +112,10 @@ creating trades.
 
 ### To switch your bot in Dry-run mode:
 1. Edit your `config.json`  file
-2. Switch dry-run to true
+2. Switch dry-run to true and specify db_url for a persistent db
 ```json
 "dry_run": true,
+"db_url": "sqlite///tradesv3.dryrun.sqlite",
 ```
 
 3. Remove your Exchange API key (change them by fake api credentials)
@@ -137,7 +139,7 @@ you run it in production mode.
 ### To switch your bot in production mode:
 1. Edit your `config.json`  file
 
-2. Switch dry-run to false
+2. Switch dry-run to false and don't forget to adapt your database URL if set
 ```json
 "dry_run": false,
 ```

docs/installation.md

@@ -162,10 +162,10 @@ docker run -d \
   -v /etc/localtime:/etc/localtime:ro \
   -v ~/.freqtrade/config.json:/freqtrade/config.json \
   -v ~/.freqtrade/tradesv3.sqlite:/freqtrade/tradesv3.sqlite \
-  freqtrade
+  freqtrade --db-url sqlite:///tradesv3.sqlite
 ```
-
+NOTE: db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used.
-If you are using `dry_run=True` it's not necessary to mount `tradesv3.sqlite`, but you can mount `tradesv3.dryrun.sqlite` if you plan to use the dry run mode with the param `--dry-run-db`.
+To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite`
 
 ### 6. Monitor your Docker instance
 

freqtrade/__init__.py

@@ -12,7 +12,8 @@ class DependencyException(BaseException):
 class OperationalException(BaseException):
     """
     Requires manual intervention.
-    This happens when an exchange returns an unexpected error during runtime.
+    This happens when an exchange returns an unexpected error during runtime
+    or given configuration is invalid.
     """
 
 

freqtrade/arguments.py

@@ -107,8 +107,8 @@ class Arguments(object):
         )
         self.parser.add_argument(
             '--dynamic-whitelist',
-            help='dynamically generate and update whitelist \
+            help='dynamically generate and update whitelist'
-                                  based on 24h BaseVolume (Default 20 currencies)',  # noqa
+                 ' based on 24h BaseVolume (default: %(const)s)',
             dest='dynamic_whitelist',
             const=constants.DYNAMIC_WHITELIST,
             type=int,
@@ -116,11 +116,13 @@ class Arguments(object):
             nargs='?',
         )
         self.parser.add_argument(
-            '--dry-run-db',
+            '--db-url',
-            help='Force dry run to use a local DB "tradesv3.dry_run.sqlite" \
+            help='Override trades database URL, this is useful if dry_run is enabled'
-                                  instead of memory DB. Work only if dry_run is enabled.',
+                 ' or in custom deployments (default: %(default)s)',
-            action='store_true',
+            dest='db_url',
-            dest='dry_run_db',
+            default=constants.DEFAULT_DB_PROD_URL,
+            type=str,
+            metavar='PATH',
         )
 
     @staticmethod
@@ -288,13 +290,6 @@ class Arguments(object):
             default=None
         )
 
-        self.parser.add_argument(
-            '-db', '--db-url',
-            help='Show trades stored in database.',
-            dest='db_url',
-            default=None
-        )
-
     def testdata_dl_options(self) -> None:
         """
         Parses given arguments for testdata download

freqtrade/configuration.py

@@ -97,16 +97,21 @@ class Configuration(object):
                 '(not applicable with Backtesting and Hyperopt)'
             )
 
-        # Add dry_run_db if found and the bot in dry run
+        if self.args.db_url != constants.DEFAULT_DB_PROD_URL:
-        if self.args.dry_run_db and config.get('dry_run', False):
+            config.update({'db_url': self.args.db_url})
-            config.update({'dry_run_db': True})
+            logger.info('Parameter --db-url detected ...')
-            logger.info('Parameter --dry-run-db detected ...')
 
-        if config.get('dry_run_db', False):
         if config.get('dry_run', False):
-                logger.info('Dry_run will use the DB file: "tradesv3.dry_run.sqlite"')
+            logger.info('Dry run is enabled')
+            if config.get('db_url') in [None, constants.DEFAULT_DB_PROD_URL]:
+                # Default to in-memory db for dry_run if not specified
+                config['db_url'] = constants.DEFAULT_DB_DRYRUN_URL
         else:
-                logger.info('Dry run is disabled. (--dry_run_db ignored)')
+            if not config.get('db_url', None):
+                config['db_url'] = constants.DEFAULT_DB_PROD_URL
+            logger.info('Dry run is disabled')
+
+        logger.info('Using DB: "{}"'.format(config['db_url']))
 
         # Check if the exchange set by the user is supported
         self.check_exchange(config)

freqtrade/constants.py

@@ -9,6 +9,8 @@ TICKER_INTERVAL = 5  # min
 HYPEROPT_EPOCH = 100  # epochs
 RETRY_TIMEOUT = 30  # sec
 DEFAULT_STRATEGY = 'DefaultStrategy'
+DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
+DEFAULT_DB_DRYRUN_URL = 'sqlite://'
 
 TICKER_INTERVAL_MINUTES = {
     '1m': 1,
@@ -83,6 +85,7 @@ CONF_SCHEMA = {
             },
             'required': ['enabled', 'token', 'chat_id']
         },
+        'db_url': {'type': 'string'},
         'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
         'internals': {
             'type': 'object',

freqtrade/freqtradebot.py

@@ -33,12 +33,11 @@ class FreqtradeBot(object):
     This is from here the bot start its logic.
     """
 
-    def __init__(self, config: Dict[str, Any], db_url: Optional[str] = None)-> None:
+    def __init__(self, config: Dict[str, Any])-> None:
         """
         Init all variables and object the bot need to work
         :param config: configuration dict, you can use the Configuration.get_config()
         method to get the config dict.
-        :param db_url: database connector string for sqlalchemy (Optional)
         """
 
         logger.info(
@@ -57,17 +56,16 @@ class FreqtradeBot(object):
         self.persistence = None
         self.exchange = None
 
-        self._init_modules(db_url=db_url)
+        self._init_modules()
 
-    def _init_modules(self, db_url: Optional[str] = None) -> None:
+    def _init_modules(self) -> None:
         """
         Initializes all modules and updates the config
-        :param db_url: database connector string for sqlalchemy (Optional)
         :return: None
         """
         # Initialize all modules
 
-        persistence.init(self.config, db_url)
+        persistence.init(self.config)
         exchange.init(self.config)
 
         # Set initial application state

freqtrade/main.py

@@ -7,6 +7,7 @@ import logging
 import sys
 from typing import List
 
+from freqtrade import OperationalException
 from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.freqtradebot import FreqtradeBot
@@ -47,6 +48,9 @@ def main(sysargv: List[str]) -> None:
     except KeyboardInterrupt:
         logger.info('SIGINT received, aborting ...')
         return_code = 0
+    except OperationalException as e:
+        logger.error(str(e))
+        return_code = 2
     except BaseException:
         logger.exception('Fatal exception!')
     finally:

freqtrade/persistence.py

@@ -10,13 +10,14 @@ from typing import Dict, Optional, Any
 import arrow
 from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
                         create_engine)
-from sqlalchemy.engine import Engine
+from sqlalchemy import inspect
+from sqlalchemy.exc import NoSuchModuleError
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm.scoping import scoped_session
 from sqlalchemy.orm.session import sessionmaker
 from sqlalchemy.pool import StaticPool
-from sqlalchemy import inspect
 
+from freqtrade import OperationalException
 
 logger = logging.getLogger(__name__)
 
@@ -24,29 +25,34 @@ _CONF = {}
 _DECL_BASE: Any = declarative_base()
 
 
-def init(config: dict, engine: Optional[Engine] = None) -> None:
+def init(config: Dict) -> None:
     """
     Initializes this module with the given config,
     registers all known command handlers
     and starts polling for message updates
     :param config: config to use
-    :param engine: database engine for sqlalchemy (Optional)
     :return: None
     """
     _CONF.update(config)
-    if not engine:
+
-        if _CONF.get('dry_run', False):
+    db_url = _CONF.get('db_url', None)
-            # the user wants dry run to use a DB
+    kwargs = {}
-            if _CONF.get('dry_run_db', False):
+
-                engine = create_engine('sqlite:///tradesv3.dry_run.sqlite')
+    # Take care of thread ownership if in-memory db
-            # Otherwise dry run will store in memory
+    if db_url == 'sqlite://':
-            else:
+        kwargs.update({
-                engine = create_engine('sqlite://',
+            'connect_args': {'check_same_thread': False},
-                                       connect_args={'check_same_thread': False},
+            'poolclass': StaticPool,
-                                       poolclass=StaticPool,
+            'echo': False,
-                                       echo=False)
+        })
-        else:
+
-            engine = create_engine('sqlite:///tradesv3.sqlite')
+    try:
+        engine = create_engine(db_url, **kwargs)
+    except NoSuchModuleError:
+        error = 'Given value for db_url: \'{}\' is no valid database URL! (See {}).'.format(
+            db_url, 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls'
+        )
+        raise OperationalException(error)
 
     session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
     Trade.session = session()
@@ -54,8 +60,8 @@ def init(config: dict, engine: Optional[Engine] = None) -> None:
     _DECL_BASE.metadata.create_all(engine)
     check_migrate(engine)
 
-    # Clean dry_run DB
+    # Clean dry_run DB if the db is not in-memory
-    if _CONF.get('dry_run', False) and _CONF.get('dry_run_db', False):
+    if _CONF.get('dry_run', False) and db_url != 'sqlite://':
         clean_dry_run_db()
 
 

freqtrade/tests/conftest.py

@@ -9,7 +9,6 @@ from unittest.mock import MagicMock
 import arrow
 import pytest
 from jsonschema import validate
-from sqlalchemy import create_engine
 from telegram import Chat, Message, Update
 
 from freqtrade.analyze import Analyze
@@ -45,7 +44,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
     mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock())
     mocker.patch('freqtrade.freqtradebot.Analyze.get_signal', MagicMock())
 
-    return FreqtradeBot(config, create_engine('sqlite://'))
+    return FreqtradeBot(config)
 
 
 def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None:
@@ -108,7 +107,8 @@ def default_conf():
             "chat_id": "0"
         },
         "initial_state": "running",
-        "loglevel": logging.DEBUG
+        "db_url": "sqlite://",
+        "loglevel": logging.DEBUG,
     }
     validate(configuration, constants.CONF_SCHEMA)
     return configuration

freqtrade/tests/rpc/test_rpc.py

@@ -7,8 +7,6 @@ Unit test file for rpc/rpc.py
 from datetime import datetime
 from unittest.mock import MagicMock
 
-from sqlalchemy import create_engine
-
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.persistence import Trade
 from freqtrade.rpc.rpc import RPC
@@ -39,7 +37,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -88,7 +86,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -123,7 +121,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee,
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     stake_currency = default_conf['stake_currency']
     fiat_display_currency = default_conf['fiat_display_currency']
 
@@ -180,7 +178,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee,
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     stake_currency = default_conf['stake_currency']
     fiat_display_currency = default_conf['fiat_display_currency']
 
@@ -258,7 +256,7 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee,
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     stake_currency = default_conf['stake_currency']
     fiat_display_currency = default_conf['fiat_display_currency']
 
@@ -329,7 +327,7 @@ def test_rpc_balance_handle(default_conf, mocker):
         get_balances=MagicMock(return_value=mock_balance)
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
 
     (error, res) = rpc.rpc_balance(default_conf['fiat_display_currency'])
@@ -359,7 +357,7 @@ def test_rpc_start(mocker, default_conf) -> None:
         get_ticker=MagicMock()
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
     freqtradebot.state = State.STOPPED
 
@@ -387,7 +385,7 @@ def test_rpc_stop(mocker, default_conf) -> None:
         get_ticker=MagicMock()
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
     freqtradebot.state = State.RUNNING
 
@@ -426,7 +424,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
         get_fee=fee,
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -536,7 +534,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee,
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
 
     # Create some test data
@@ -575,7 +573,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None:
         get_fee=fee,
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     rpc = RPC(freqtradebot)
 
     (error, trades) = rpc.rpc_count()

freqtrade/tests/rpc/test_rpc_telegram.py

@@ -11,7 +11,6 @@ from datetime import datetime
 from random import randint
 from unittest.mock import MagicMock
 
-from sqlalchemy import create_engine
 from telegram import Update, Message, Chat
 from telegram.error import NetworkError
 
@@ -156,7 +155,7 @@ def test_authorized_only(default_conf, mocker, caplog) -> None:
 
     conf = deepcopy(default_conf)
     conf['telegram']['enabled'] = False
-    dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://')))
+    dummy = DummyCls(FreqtradeBot(conf))
     dummy.dummy_handler(bot=MagicMock(), update=update)
     assert dummy.state['called'] is True
     assert log_has(
@@ -187,7 +186,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None:
 
     conf = deepcopy(default_conf)
     conf['telegram']['enabled'] = False
-    dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://')))
+    dummy = DummyCls(FreqtradeBot(conf))
     dummy.dummy_handler(bot=MagicMock(), update=update)
     assert dummy.state['called'] is False
     assert not log_has(
@@ -217,7 +216,7 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None:
 
     conf = deepcopy(default_conf)
     conf['telegram']['enabled'] = False
-    dummy = DummyCls(FreqtradeBot(conf, create_engine('sqlite://')))
+    dummy = DummyCls(FreqtradeBot(conf))
     dummy.dummy_exception(bot=MagicMock(), update=update)
     assert dummy.state['called'] is False
     assert not log_has(
@@ -263,7 +262,7 @@ def test_status(default_conf, update, mocker, fee, ticker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(conf)
     telegram = Telegram(freqtradebot)
 
     # Create some test data
@@ -301,7 +300,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -348,7 +347,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None:
 
     conf = deepcopy(default_conf)
     conf['stake_amount'] = 15.0
-    freqtradebot = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -402,7 +401,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee,
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Create some test data
@@ -470,7 +469,7 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Try invalid data
@@ -511,7 +510,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee,
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     telegram._profit(bot=MagicMock(), update=update)
@@ -608,7 +607,7 @@ def test_telegram_balance_handle(default_conf, update, mocker) -> None:
         send_msg=msg_mock
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     telegram._balance(bot=MagicMock(), update=update)
@@ -638,7 +637,7 @@ def test_zero_balance_handle(default_conf, update, mocker) -> None:
         send_msg=msg_mock
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     telegram._balance(bot=MagicMock(), update=update)
@@ -661,7 +660,7 @@ def test_start_handle(default_conf, update, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -685,7 +684,7 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.RUNNING
@@ -710,7 +709,7 @@ def test_stop_handle(default_conf, update, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.RUNNING
@@ -735,7 +734,7 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -762,7 +761,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, moc
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Create some test data
@@ -802,7 +801,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_do
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Create some test data
@@ -847,7 +846,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None
         get_fee=fee
     )
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Create some test data
@@ -880,7 +879,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None:
     )
     mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock())
 
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Trader is not running
@@ -927,7 +926,7 @@ def test_performance_handle(default_conf, update, ticker, fee,
         get_fee=fee
     )
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Create some test data
@@ -962,7 +961,7 @@ def test_performance_handle_invalid(default_conf, update, mocker) -> None:
         send_msg=msg_mock
     )
     mocker.patch('freqtrade.freqtradebot.exchange.validate_pairs', MagicMock())
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     # Trader is not running
@@ -991,7 +990,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None:
         buy=MagicMock(return_value={'id': 'mocked_order_id'})
     )
     mocker.patch('freqtrade.optimize.backtesting.exchange.get_fee', fee)
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     freqtradebot.state = State.STOPPED
@@ -1027,7 +1026,7 @@ def test_help_handle(default_conf, update, mocker) -> None:
         _init=MagicMock(),
         send_msg=msg_mock
     )
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     telegram._help(bot=MagicMock(), update=update)
@@ -1047,7 +1046,7 @@ def test_version_handle(default_conf, update, mocker) -> None:
         _init=MagicMock(),
         send_msg=msg_mock
     )
-    freqtradebot = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(default_conf)
     telegram = Telegram(freqtradebot)
 
     telegram._version(bot=MagicMock(), update=update)
@@ -1064,7 +1063,7 @@ def test_send_msg(default_conf, mocker) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
     conf = deepcopy(default_conf)
     bot = MagicMock()
-    freqtradebot = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(conf)
     telegram = Telegram(freqtradebot)
 
     telegram._config['telegram']['enabled'] = False
@@ -1087,7 +1086,7 @@ def test_send_msg_network_error(default_conf, mocker, caplog) -> None:
     conf = deepcopy(default_conf)
     bot = MagicMock()
     bot.send_message = MagicMock(side_effect=NetworkError('Oh snap'))
-    freqtradebot = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtradebot = FreqtradeBot(conf)
     telegram = Telegram(freqtradebot)
 
     telegram._config['telegram']['enabled'] = True

freqtrade/tests/test_arguments.py

@@ -46,6 +46,11 @@ def test_parse_args_config() -> None:
     assert args.config == '/dev/null'
 
 
+def test_parse_args_db_url() -> None:
+    args = Arguments(['--db-url', 'sqlite:///test.sqlite'], '').get_parsed_arg()
+    assert args.db_url == 'sqlite:///test.sqlite'
+
+
 def test_parse_args_verbose() -> None:
     args = Arguments(['-v'], '').get_parsed_arg()
     assert args.loglevel == logging.DEBUG

freqtrade/tests/test_configuration.py

@@ -118,7 +118,6 @@ def test_load_config(default_conf, mocker) -> None:
     assert validated_conf.get('strategy') == 'DefaultStrategy'
     assert validated_conf.get('strategy_path') is None
     assert 'dynamic_whitelist' not in validated_conf
-    assert 'dry_run_db' not in validated_conf
 
 
 def test_load_config_with_params(default_conf, mocker) -> None:
@@ -133,7 +132,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
         '--dynamic-whitelist', '10',
         '--strategy', 'TestStrategy',
         '--strategy-path', '/some/path',
-        '--dry-run-db',
+        '--db-url', 'sqlite:///someurl',
     ]
     args = Arguments(arglist, '').get_parsed_arg()
 
@@ -143,7 +142,7 @@ def test_load_config_with_params(default_conf, mocker) -> None:
     assert validated_conf.get('dynamic_whitelist') == 10
     assert validated_conf.get('strategy') == 'TestStrategy'
     assert validated_conf.get('strategy_path') == '/some/path'
-    assert validated_conf.get('dry_run_db') is True
+    assert validated_conf.get('db_url') == 'sqlite:///someurl'
 
 
 def test_load_custom_strategy(default_conf, mocker) -> None:
@@ -178,7 +177,7 @@ def test_show_info(default_conf, mocker, caplog) -> None:
     arglist = [
         '--dynamic-whitelist', '10',
         '--strategy', 'TestStrategy',
-        '--dry-run-db'
+        '--db-url', 'sqlite:///tmp/testdb',
     ]
     args = Arguments(arglist, '').get_parsed_arg()
 
@@ -192,23 +191,8 @@ def test_show_info(default_conf, mocker, caplog) -> None:
         caplog.record_tuples
     )
 
-    assert log_has(
+    assert log_has('Using DB: "sqlite:///tmp/testdb"', caplog.record_tuples)
-        'Parameter --dry-run-db detected ...',
+    assert log_has('Dry run is enabled', caplog.record_tuples)
-        caplog.record_tuples
-    )
-
-    assert log_has(
-        'Dry_run will use the DB file: "tradesv3.dry_run.sqlite"',
-        caplog.record_tuples
-    )
-
-    # Test the Dry run condition
-    configuration.config.update({'dry_run': False})  # type: ignore
-    configuration._load_common_config(configuration.config)  # type: ignore
-    assert log_has(
-        'Dry run is disabled. (--dry_run_db ignored)',
-        caplog.record_tuples
-    )
 
 
 def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None:

freqtrade/tests/test_freqtradebot.py

@@ -13,7 +13,6 @@ from unittest.mock import MagicMock
 import arrow
 import pytest
 import requests
-from sqlalchemy import create_engine
 
 from freqtrade import DependencyException, OperationalException, TemporaryError
 from freqtrade.freqtradebot import FreqtradeBot
@@ -36,7 +35,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot:
     mocker.patch('freqtrade.freqtradebot.exchange.init', MagicMock())
     patch_coinmarketcap(mocker)
 
-    return FreqtradeBot(config, create_engine('sqlite://'))
+    return FreqtradeBot(config)
 
 
 def patch_get_signal(mocker, value=(True, False)) -> None:
@@ -237,7 +236,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non
 
     # Save state of current whitelist
     whitelist = deepcopy(default_conf['exchange']['pair_whitelist'])
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -274,7 +273,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, fee,
 
     conf = deepcopy(default_conf)
     conf['stake_amount'] = 0.0005
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
 
     freqtrade.create_trade()
     rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2]
@@ -296,7 +295,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, fee
         get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5),
         get_fee=fee,
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     with pytest.raises(DependencyException, match=r'.*stake amount.*'):
         freqtrade.create_trade()
@@ -320,7 +319,7 @@ def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, mocke
     conf = deepcopy(default_conf)
     conf['exchange']['pair_whitelist'] = ["ETH/BTC"]
     conf['exchange']['pair_blacklist'] = ["ETH/BTC"]
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
 
     freqtrade.create_trade()
 
@@ -347,7 +346,7 @@ def test_create_trade_no_pairs_after_blacklist(default_conf, ticker,
     conf = deepcopy(default_conf)
     conf['exchange']['pair_whitelist'] = ["ETH/BTC"]
     conf['exchange']['pair_blacklist'] = ["ETH/BTC"]
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
 
     freqtrade.create_trade()
 
@@ -375,7 +374,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
 
     conf = deepcopy(default_conf)
     conf['stake_amount'] = 10
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
 
     Trade.query = MagicMock()
     Trade.query.filter = MagicMock()
@@ -399,7 +398,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order,
         get_order=MagicMock(return_value=limit_buy_order),
         get_fee=fee,
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trades = Trade.query.filter(Trade.is_open.is_(True)).all()
     assert not trades
@@ -440,7 +439,7 @@ def test_process_exchange_failures(default_conf, ticker, markets, mocker) -> Non
     )
     sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None)
 
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     result = freqtrade._process()
     assert result is False
     assert sleep_mock.has_calls()
@@ -460,7 +459,7 @@ def test_process_operational_exception(default_conf, ticker, markets, mocker) ->
         get_markets=markets,
         buy=MagicMock(side_effect=OperationalException)
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     assert freqtrade.state == State.RUNNING
 
     result = freqtrade._process()
@@ -486,7 +485,7 @@ def test_process_trade_handling(
         get_order=MagicMock(return_value=limit_buy_order),
         get_fee=fee,
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trades = Trade.query.filter(Trade.is_open.is_(True)).all()
     assert not trades
@@ -603,7 +602,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock
     )
     patch_coinmarketcap(mocker, value={'price_usd': 15000.0})
 
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     freqtrade.create_trade()
 
@@ -646,7 +645,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee,
         get_fee=fee,
     )
 
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
 
     freqtrade.create_trade()
 
@@ -705,7 +704,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, fee, mocker, ca
     )
 
     mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=True)
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -742,7 +741,7 @@ def test_handle_trade_experimental(
     )
     mocker.patch('freqtrade.freqtradebot.Analyze.min_roi_reached', return_value=False)
 
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -770,7 +769,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, fe
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         get_fee=fee,
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     # Create trade and sell it
     freqtrade.create_trade()
@@ -801,7 +800,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, fe
         cancel_order=cancel_order_mock,
         get_fee=fee
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trade_buy = Trade(
         pair='ETH/BTC',
@@ -841,7 +840,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old,
         get_order=MagicMock(return_value=limit_sell_order_old),
         cancel_order=cancel_order_mock
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trade_sell = Trade(
         pair='ETH/BTC',
@@ -881,7 +880,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old
         get_order=MagicMock(return_value=limit_buy_order_old_partial),
         cancel_order=cancel_order_mock
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trade_buy = Trade(
         pair='ETH/BTC',
@@ -929,7 +928,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, mocker, caplog) -
         get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')),
         cancel_order=cancel_order_mock
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trade_buy = Trade(
         pair='ETH/BTC',
@@ -968,7 +967,7 @@ def test_handle_timedout_limit_buy(mocker, default_conf) -> None:
         cancel_order=cancel_order_mock
     )
 
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     Trade.session = MagicMock()
     trade = MagicMock()
@@ -994,7 +993,7 @@ def test_handle_timedout_limit_sell(mocker, default_conf) -> None:
         cancel_order=cancel_order_mock
     )
 
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     trade = MagicMock()
     order = {'remaining': 1,
@@ -1021,7 +1020,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N
         get_fee=fee
     )
     mocker.patch('freqtrade.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0)
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     # Create some test data
     freqtrade.create_trade()
@@ -1062,7 +1061,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker)
         get_ticker=ticker,
         get_fee=fee
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     # Create some test data
     freqtrade.create_trade()
@@ -1102,7 +1101,7 @@ def test_execute_sell_without_conf_sell_up(default_conf, ticker, fee,
         get_ticker=ticker,
         get_fee=fee
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     # Create some test data
     freqtrade.create_trade()
@@ -1143,7 +1142,7 @@ def test_execute_sell_without_conf_sell_down(default_conf, ticker, fee,
         get_ticker=ticker,
         get_fee=fee
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
 
     # Create some test data
     freqtrade.create_trade()
@@ -1192,7 +1191,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, fee, mock
         'use_sell_signal': True,
         'sell_profit_only': True,
     }
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1225,7 +1224,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, fee, moc
         'use_sell_signal': True,
         'sell_profit_only': False,
     }
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1258,7 +1257,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker
         'use_sell_signal': True,
         'sell_profit_only': True,
     }
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1293,7 +1292,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke
         'sell_profit_only': False,
     }
 
-    freqtrade = FreqtradeBot(conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(conf)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -1321,7 +1320,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca
         open_rate=0.245441,
         open_order_id="123456"
         )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
@@ -1348,7 +1347,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker):
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
@@ -1375,7 +1374,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, ca
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount does not change
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
 
@@ -1401,7 +1400,7 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, mock
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount does not change
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount
 
@@ -1424,7 +1423,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001)
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
@@ -1452,7 +1451,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount is reduced by "fee"
     assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004
     assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
@@ -1480,7 +1479,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount does not change
     assert freqtrade.get_real_amount(trade, limit_buy_order) == amount
 
@@ -1505,6 +1504,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee,
         open_rate=0.245441,
         open_order_id="123456"
     )
-    freqtrade = FreqtradeBot(default_conf, create_engine('sqlite://'))
+    freqtrade = FreqtradeBot(default_conf)
     # Amount does not change
     assert freqtrade.get_real_amount(trade, buy_order_fee) == amount

freqtrade/tests/test_persistence.py

@@ -1,9 +1,11 @@
 # pragma pylint: disable=missing-docstring, C0103
-import os
+from copy import deepcopy
+from unittest.mock import MagicMock
 
 import pytest
 from sqlalchemy import create_engine
 
+from freqtrade import constants, OperationalException
 from freqtrade.persistence import Trade, init, clean_dry_run_db
 
 
@@ -21,77 +23,54 @@ def test_init_create_session(default_conf, mocker):
     assert 'Session' in type(Trade.session).__name__
 
 
-def test_init_dry_run_db(default_conf, mocker):
+def test_init_custom_db_url(default_conf, mocker):
-    default_conf.update({'dry_run_db': True})
+    conf = deepcopy(default_conf)
-    mocker.patch.dict('freqtrade.persistence._CONF', default_conf)
 
-    # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data)
+    # Update path to a value other than default, but still in-memory
-    dry_run_db = 'tradesv3.dry_run.sqlite'
+    conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'})
-    dry_run_db_swp = dry_run_db + '.swp'
+    create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock())
+    mocker.patch.dict('freqtrade.persistence._CONF', conf)
 
-    if os.path.isfile(dry_run_db):
+    init(conf)
-        os.rename(dry_run_db, dry_run_db_swp)
+    assert create_engine_mock.call_count == 1
-
+    assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite'
-    # Check if the new tradesv3.dry_run.sqlite was created
-    init(default_conf)
-    assert os.path.isfile(dry_run_db) is True
-
-    # Delete the file made for this unitest and rollback to the previous
-    # tradesv3.dry_run.sqlite file
-
-    # 1. Delete file from the test
-    if os.path.isfile(dry_run_db):
-        os.remove(dry_run_db)
-
-    # 2. Rollback to the initial file
-    if os.path.isfile(dry_run_db_swp):
-        os.rename(dry_run_db_swp, dry_run_db)
 
 
-def test_init_dry_run_without_db(default_conf, mocker):
+def test_init_invalid_db_url(default_conf, mocker):
-    default_conf.update({'dry_run_db': False})
+    conf = deepcopy(default_conf)
-    mocker.patch.dict('freqtrade.persistence._CONF', default_conf)
 
-    # First, protect the existing 'tradesv3.dry_run.sqlite' (Do not delete user data)
+    # Update path to a value other than default, but still in-memory
-    dry_run_db = 'tradesv3.dry_run.sqlite'
+    conf.update({'db_url': 'unknown:///some.url'})
-    dry_run_db_swp = dry_run_db + '.swp'
+    mocker.patch.dict('freqtrade.persistence._CONF', conf)
 
-    if os.path.isfile(dry_run_db):
+    with pytest.raises(OperationalException, match=r'.*no valid database URL*'):
-        os.rename(dry_run_db, dry_run_db_swp)
+        init(conf)
-
-    # Check if the new tradesv3.dry_run.sqlite was created
-    init(default_conf)
-    assert os.path.isfile(dry_run_db) is False
-
-    # Rollback to the initial 'tradesv3.dry_run.sqlite' file
-    if os.path.isfile(dry_run_db_swp):
-        os.rename(dry_run_db_swp, dry_run_db)
 
 
 def test_init_prod_db(default_conf, mocker):
-    default_conf.update({'dry_run': False})
+    conf = deepcopy(default_conf)
-    mocker.patch.dict('freqtrade.persistence._CONF', default_conf)
+    conf.update({'dry_run': False})
+    conf.update({'db_url': constants.DEFAULT_DB_PROD_URL})
 
-    # First, protect the existing 'tradesv3.sqlite' (Do not delete user data)
+    create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock())
-    prod_db = 'tradesv3.sqlite'
+    mocker.patch.dict('freqtrade.persistence._CONF', conf)
-    prod_db_swp = prod_db + '.swp'
 
-    if os.path.isfile(prod_db):
+    init(conf)
-        os.rename(prod_db, prod_db_swp)
+    assert create_engine_mock.call_count == 1
+    assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite'
 
-    # Check if the new tradesv3.sqlite was created
-    init(default_conf)
-    assert os.path.isfile(prod_db) is True
 
-    # Delete the file made for this unitest and rollback to the previous tradesv3.sqlite file
+def test_init_dryrun_db(default_conf, mocker):
+    conf = deepcopy(default_conf)
+    conf.update({'dry_run': True})
+    conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL})
 
-    # 1. Delete file from the test
+    create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock())
-    if os.path.isfile(prod_db):
+    mocker.patch.dict('freqtrade.persistence._CONF', conf)
-        os.remove(prod_db)
 
-    # Rollback to the initial 'tradesv3.sqlite' file
+    init(conf)
-    if os.path.isfile(prod_db_swp):
+    assert create_engine_mock.call_count == 1
-        os.rename(prod_db_swp, prod_db)
+    assert create_engine_mock.mock_calls[0][1][0] == 'sqlite://'
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -328,7 +307,7 @@ def test_calc_profit_percent(limit_buy_order, limit_sell_order, fee):
 
 
 def test_clean_dry_run_db(default_conf, fee):
-    init(default_conf, create_engine('sqlite://'))
+    init(default_conf)
 
     # Simulate dry_run entries
     trade = Trade(
@@ -377,7 +356,7 @@ def test_clean_dry_run_db(default_conf, fee):
     assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 1
 
 
-def test_migrate_old(default_conf, fee):
+def test_migrate_old(mocker, default_conf, fee):
     """
     Test Database migration(starting with old pairformat)
     """
@@ -409,11 +388,13 @@ def test_migrate_old(default_conf, fee):
                                      amount=amount
                                      )
     engine = create_engine('sqlite://')
+    mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine)
+
     # Create table using the old format
     engine.execute(create_table_old)
     engine.execute(insert_table_old)
     # Run init to test migration
-    init(default_conf, engine)
+    init(default_conf)
 
     assert len(Trade.query.filter(Trade.id == 1).all()) == 1
     trade = Trade.query.filter(Trade.id == 1).first()
@@ -428,7 +409,7 @@ def test_migrate_old(default_conf, fee):
     assert trade.exchange == "bittrex"
 
 
-def test_migrate_new(default_conf, fee):
+def test_migrate_new(mocker, default_conf, fee):
     """
     Test Database migration (starting with new pairformat)
     """
@@ -460,11 +441,13 @@ def test_migrate_new(default_conf, fee):
                                      amount=amount
                                      )
     engine = create_engine('sqlite://')
+    mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine)
+
     # Create table using the old format
     engine.execute(create_table_old)
     engine.execute(insert_table_old)
     # Run init to test migration
-    init(default_conf, engine)
+    init(default_conf)
 
     assert len(Trade.query.filter(Trade.id == 1).all()) == 1
     trade = Trade.query.filter(Trade.id == 1).first()

scripts/plot_dataframe.py

@@ -29,16 +29,17 @@ import os
 import sys
 from argparse import Namespace
 from typing import Dict, List, Any
-from sqlalchemy import create_engine
+
+import plotly.graph_objs as go
 from plotly import tools
 from plotly.offline import plot
-import plotly.graph_objs as go
+
-from freqtrade.arguments import Arguments
-from freqtrade.analyze import Analyze
-from freqtrade.optimize.backtesting import setup_configuration
-from freqtrade import exchange
 import freqtrade.optimize as optimize
+from freqtrade import exchange
 from freqtrade import persistence
+from freqtrade.analyze import Analyze
+from freqtrade.arguments import Arguments
+from freqtrade.optimize.backtesting import setup_configuration
 from freqtrade.persistence import Trade
 
 logger = logging.getLogger(__name__)
@@ -50,9 +51,10 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
     Calls analyze() and plots the returned dataframe
     :return: None
     """
+    global _CONF
 
     # Load the configuration
-    config = setup_configuration(args)
+    _CONF.update(setup_configuration(args))
 
     # Set the pair to audit
     pair = args.pair
@@ -65,14 +67,13 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
         logger.critical('--pair format must be XXX/YYY')
         exit()
 
-
     # Set timerange to use
     timerange = Arguments.parse_timerange(args.timerange)
 
     # Load the strategy
     try:
-        analyze = Analyze(config)
+        analyze = Analyze(_CONF)
-        exchange.init(config)
+        exchange.init(_CONF)
     except AttributeError:
         logger.critical(
             'Impossible to load the strategy. Please check the file "user_data/strategies/%s.py"',
@@ -93,7 +94,7 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
             datadir=args.datadir,
             pairs=[pair],
             ticker_interval=tick_interval,
-            refresh_pairs=config.get('refresh_pairs', False),
+            refresh_pairs=_CONF.get('refresh_pairs', False),
             timerange=timerange
         )
 
@@ -102,10 +103,9 @@ def plot_analyzed_dataframe(args: Namespace) -> None:
             exit()
 
     # Get trades already made from the DB
-    trades = []
+    trades: List[Trade] = []
     if args.db_url:
-        engine = create_engine('sqlite:///' + args.db_url)
+        persistence.init(_CONF)
-        persistence.init(_CONF, engine)
         trades = Trade.query.filter(Trade.pair.is_(pair)).all()
 
     dataframes = analyze.tickerdata_to_dataframe(tickers)