multiple --config options

freqtrade/arguments.py

@@ -6,9 +6,7 @@ import argparse
 import os
 import re
 from typing import List, NamedTuple, Optional
-
 import arrow
-
 from freqtrade import __version__, constants
 
 
@@ -55,6 +53,11 @@ class Arguments(object):
         """
         parsed_arg = self.parser.parse_args(self.args)
 
+        # Workaround issue in argparse with action='append' and default value
+        # (see https://bugs.python.org/issue16399)
+        if parsed_arg.config is None:
+            parsed_arg.config = [constants.DEFAULT_CONFIG]
+
         return parsed_arg
 
     def common_args_parser(self) -> None:
@@ -63,7 +66,7 @@ class Arguments(object):
         """
         self.parser.add_argument(
             '-v', '--verbose',
-            help='verbose mode (-vv for more, -vvv to get all messages)',
+            help='Verbose mode (-vv for more, -vvv to get all messages).',
             action='count',
             dest='loglevel',
             default=0,
@@ -75,15 +78,15 @@ class Arguments(object):
         )
         self.parser.add_argument(
             '-c', '--config',
-            help='specify configuration file (default: %(default)s)',
+            help='Specify configuration file (default: %(default)s).',
             dest='config',
-            default='config.json',
+            action='append',
             type=str,
             metavar='PATH',
         )
         self.parser.add_argument(
             '-d', '--datadir',
-            help='path to backtest data',
+            help='Path to backtest data.',
             dest='datadir',
             default=None,
             type=str,
@@ -91,7 +94,7 @@ class Arguments(object):
         )
         self.parser.add_argument(
             '-s', '--strategy',
-            help='specify strategy class name (default: %(default)s)',
+            help='Specify strategy class name (default: %(default)s).',
             dest='strategy',
             default='DefaultStrategy',
             type=str,
@@ -99,14 +102,14 @@ class Arguments(object):
         )
         self.parser.add_argument(
             '--strategy-path',
-            help='specify additional strategy lookup path',
+            help='Specify additional strategy lookup path.',
             dest='strategy_path',
             type=str,
             metavar='PATH',
         )
         self.parser.add_argument(
             '--customhyperopt',
-            help='specify hyperopt class name (default: %(default)s)',
+            help='Specify hyperopt class name (default: %(default)s).',
             dest='hyperopt',
             default=constants.DEFAULT_HYPEROPT,
             type=str,
@@ -114,8 +117,8 @@ class Arguments(object):
         )
         self.parser.add_argument(
             '--dynamic-whitelist',
-            help='dynamically generate and update whitelist'
+            help='Dynamically generate and update whitelist'
-                 ' based on 24h BaseVolume (default: %(const)s)'
+                 ' based on 24h BaseVolume (default: %(const)s).'
                  ' DEPRECATED.',
             dest='dynamic_whitelist',
             const=constants.DYNAMIC_WHITELIST,
@@ -126,7 +129,7 @@ class Arguments(object):
         self.parser.add_argument(
             '--db-url',
             help='Override trades database URL, this is useful if dry_run is enabled'
-                 ' or in custom deployments (default: %(default)s)',
+                 ' or in custom deployments (default: %(default)s).',
             dest='db_url',
             type=str,
             metavar='PATH',
@@ -139,7 +142,7 @@ class Arguments(object):
         """
         parser.add_argument(
             '--eps', '--enable-position-stacking',
-            help='Allow buying the same pair multiple times (position stacking)',
+            help='Allow buying the same pair multiple times (position stacking).',
             action='store_true',
             dest='position_stacking',
             default=False
@@ -148,20 +151,20 @@ class Arguments(object):
         parser.add_argument(
             '--dmmp', '--disable-max-market-positions',
             help='Disable applying `max_open_trades` during backtest '
-                 '(same as setting `max_open_trades` to a very high number)',
+                 '(same as setting `max_open_trades` to a very high number).',
             action='store_false',
             dest='use_max_market_positions',
             default=True
         )
         parser.add_argument(
             '-l', '--live',
-            help='using live data',
+            help='Use live data.',
             action='store_true',
             dest='live',
         )
         parser.add_argument(
             '-r', '--refresh-pairs-cached',
-            help='refresh the pairs files in tests/testdata with the latest data from the '
+            help='Refresh the pairs files in tests/testdata with the latest data from the '
                  'exchange. Use it if you want to run your backtesting with up-to-date data.',
             action='store_true',
             dest='refresh_pairs',
@@ -178,8 +181,8 @@ class Arguments(object):
         )
         parser.add_argument(
             '--export',
-            help='export backtest results, argument are: trades\
+            help='Export backtest results, argument are: trades. '
-                  Example --export=trades',
+                 'Example --export=trades',
             type=str,
             default=None,
             dest='export',
@@ -203,14 +206,14 @@ class Arguments(object):
         """
         parser.add_argument(
             '-r', '--refresh-pairs-cached',
-            help='refresh the pairs files in tests/testdata with the latest data from the '
+            help='Refresh the pairs files in tests/testdata with the latest data from the '
                  'exchange. Use it if you want to run your edge with up-to-date data.',
             action='store_true',
             dest='refresh_pairs',
         )
         parser.add_argument(
             '--stoplosses',
-            help='defines a range of stoploss against which edge will assess the strategy '
+            help='Defines a range of stoploss against which edge will assess the strategy '
                  'the format is "min,max,step" (without any space).'
                  'example: --stoplosses=-0.01,-0.1,-0.001',
             type=str,
@@ -226,14 +229,14 @@ class Arguments(object):
         """
         parser.add_argument(
             '-i', '--ticker-interval',
-            help='specify ticker interval (1m, 5m, 30m, 1h, 1d)',
+            help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).',
             dest='ticker_interval',
             type=str,
         )
 
         parser.add_argument(
             '--timerange',
-            help='specify what timerange of data to use.',
+            help='Specify what timerange of data to use.',
             default=None,
             type=str,
             dest='timerange',
@@ -246,7 +249,7 @@ class Arguments(object):
         """
         parser.add_argument(
             '--eps', '--enable-position-stacking',
-            help='Allow buying the same pair multiple times (position stacking)',
+            help='Allow buying the same pair multiple times (position stacking).',
             action='store_true',
             dest='position_stacking',
             default=False
@@ -255,14 +258,14 @@ class Arguments(object):
         parser.add_argument(
             '--dmmp', '--disable-max-market-positions',
             help='Disable applying `max_open_trades` during backtest '
-                 '(same as setting `max_open_trades` to a very high number)',
+                 '(same as setting `max_open_trades` to a very high number).',
             action='store_false',
             dest='use_max_market_positions',
             default=True
         )
         parser.add_argument(
             '-e', '--epochs',
-            help='specify number of epochs (default: %(default)d)',
+            help='Specify number of epochs (default: %(default)d).',
             dest='epochs',
             default=constants.HYPEROPT_EPOCH,
             type=int,
@@ -271,7 +274,7 @@ class Arguments(object):
         parser.add_argument(
             '-s', '--spaces',
             help='Specify which parameters to hyperopt. Space separate list. \
-                  Default: %(default)s',
+                  Default: %(default)s.',
             choices=['all', 'buy', 'sell', 'roi', 'stoploss'],
             default='all',
             nargs='+',
@@ -288,19 +291,19 @@ class Arguments(object):
         subparsers = self.parser.add_subparsers(dest='subparser')
 
         # Add backtesting subcommand
-        backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
+        backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.')
         backtesting_cmd.set_defaults(func=backtesting.start)
         self.optimizer_shared_options(backtesting_cmd)
         self.backtesting_options(backtesting_cmd)
 
         # Add edge subcommand
-        edge_cmd = subparsers.add_parser('edge', help='edge module')
+        edge_cmd = subparsers.add_parser('edge', help='Edge module.')
         edge_cmd.set_defaults(func=edge_cli.start)
         self.optimizer_shared_options(edge_cmd)
         self.edge_options(edge_cmd)
 
         # Add hyperopt subcommand
-        hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
+        hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.')
         hyperopt_cmd.set_defaults(func=hyperopt.start)
         self.optimizer_shared_options(hyperopt_cmd)
         self.hyperopt_options(hyperopt_cmd)
@@ -364,7 +367,7 @@ class Arguments(object):
         """
         self.parser.add_argument(
             '--pairs-file',
-            help='File containing a list of pairs to download',
+            help='File containing a list of pairs to download.',
             dest='pairs_file',
             default=None,
             metavar='PATH',
@@ -372,7 +375,7 @@ class Arguments(object):
 
         self.parser.add_argument(
             '--export',
-            help='Export files to given dir',
+            help='Export files to given dir.',
             dest='export',
             default=None,
             metavar='PATH',
@@ -380,7 +383,8 @@ class Arguments(object):
 
         self.parser.add_argument(
             '-c', '--config',
-            help='specify configuration file, used for additional exchange parameters',
+            help='Specify configuration file, used for additional exchange parameters. '
+                 'Multiple --config options may be used.',
             dest='config',
             default=None,
             type=str,
@@ -389,7 +393,7 @@ class Arguments(object):
 
         self.parser.add_argument(
             '--days',
-            help='Download data for number of days',
+            help='Download data for given number of days.',
             dest='days',
             type=int,
             metavar='INT',
@@ -398,7 +402,7 @@ class Arguments(object):
 
         self.parser.add_argument(
             '--exchange',
-            help='Exchange name (default: %(default)s). Only valid if no config is provided',
+            help='Exchange name (default: %(default)s). Only valid if no config is provided.',
             dest='exchange',
             type=str,
             default='bittrex'
@@ -407,7 +411,7 @@ class Arguments(object):
         self.parser.add_argument(
             '-t', '--timeframes',
             help='Specify which tickers to download. Space separated list. \
-                  Default: %(default)s',
+                  Default: %(default)s.',
             choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
                      '6h', '8h', '12h', '1d', '3d', '1w'],
             default=['1m', '5m'],
@@ -417,7 +421,7 @@ class Arguments(object):
 
         self.parser.add_argument(
             '--erase',
-            help='Clean all existing data for the selected exchange/pairs/timeframes',
+            help='Clean all existing data for the selected exchange/pairs/timeframes.',
             dest='erase',
             action='store_true'
         )

freqtrade/configuration.py

@@ -13,6 +13,8 @@ from jsonschema.exceptions import ValidationError, best_match
 
 from freqtrade import OperationalException, constants
 from freqtrade.state import RunMode
+from freqtrade.misc import deep_merge_dicts
+
 logger = logging.getLogger(__name__)
 
 
@@ -45,8 +47,18 @@ class Configuration(object):
         Extract information for sys.argv and load the bot configuration
         :return: Configuration dictionary
         """
-        logger.info('Using config: %s ...', self.args.config)
+        config: Dict[str, Any] = {}
-        config = self._load_config_file(self.args.config)
+        # Now expecting a list of config filenames here, not a string
+        for path in self.args.config:
+            logger.info('Using config: %s ...', path)
+            # Merge config options, overwriting old values
+            config = deep_merge_dicts(self._load_config_file(path), config)
+
+        if 'internals' not in config:
+            config['internals'] = {}
+
+        logger.info('Validating configuration ...')
+        self._validate_config(config)
 
         # Set strategy if not specified in config and or if it's non default
         if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'):
@@ -93,11 +105,7 @@ class Configuration(object):
                 f'Config file "{path}" not found!'
                 ' Please create a config file or check whether it exists.')
 
-        if 'internals' not in conf:
+        return conf
-            conf['internals'] = {}
-        logger.info('Validating configuration ...')
-
-        return self._validate_config(conf)
 
     def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         """

freqtrade/constants.py

@@ -3,6 +3,7 @@
 """
 bot constants
 """
+DEFAULT_CONFIG = 'config.json'
 DYNAMIC_WHITELIST = 20  # pairs
 PROCESS_THROTTLE_SECS = 5  # sec
 TICKER_INTERVAL = 5  # min

freqtrade/misc.py

@@ -113,3 +113,21 @@ def format_ms_time(date: int) -> str:
     : epoch-string in ms
     """
     return datetime.fromtimestamp(date/1000.0).strftime('%Y-%m-%dT%H:%M:%S')
+
+
+def deep_merge_dicts(source, destination):
+    """
+    >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } }
+    >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } }
+    >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }
+    True
+    """
+    for key, value in source.items():
+        if isinstance(value, dict):
+            # get node or create one
+            node = destination.setdefault(key, {})
+            deep_merge_dicts(value, node)
+        else:
+            destination[key] = value
+
+    return destination