From 2f225e23408bc6c47118840bfddfb829ba6d6776 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 19 Feb 2019 15:14:47 +0300 Subject: [PATCH] multiple --config options --- freqtrade/arguments.py | 76 ++++++++++++++++++++------------------ freqtrade/configuration.py | 22 +++++++---- freqtrade/constants.py | 1 + freqtrade/misc.py | 18 +++++++++ 4 files changed, 74 insertions(+), 43 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 9b1b9a925..53a46d141 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -6,9 +6,7 @@ import argparse import os import re from typing import List, NamedTuple, Optional - import arrow - from freqtrade import __version__, constants @@ -55,6 +53,11 @@ class Arguments(object): """ parsed_arg = self.parser.parse_args(self.args) + # Workaround issue in argparse with action='append' and default value + # (see https://bugs.python.org/issue16399) + if parsed_arg.config is None: + parsed_arg.config = [constants.DEFAULT_CONFIG] + return parsed_arg def common_args_parser(self) -> None: @@ -63,7 +66,7 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='verbose mode (-vv for more, -vvv to get all messages)', + help='Verbose mode (-vv for more, -vvv to get all messages).', action='count', dest='loglevel', default=0, @@ -75,15 +78,15 @@ class Arguments(object): ) self.parser.add_argument( '-c', '--config', - help='specify configuration file (default: %(default)s)', + help='Specify configuration file (default: %(default)s).', dest='config', - default='config.json', + action='append', type=str, metavar='PATH', ) self.parser.add_argument( '-d', '--datadir', - help='path to backtest data', + help='Path to backtest data.', dest='datadir', default=None, type=str, @@ -91,7 +94,7 @@ class Arguments(object): ) self.parser.add_argument( '-s', '--strategy', - help='specify strategy class name (default: %(default)s)', + help='Specify strategy class name (default: %(default)s).', dest='strategy', default='DefaultStrategy', type=str, @@ -99,14 +102,14 @@ class Arguments(object): ) self.parser.add_argument( '--strategy-path', - help='specify additional strategy lookup path', + help='Specify additional strategy lookup path.', dest='strategy_path', type=str, metavar='PATH', ) self.parser.add_argument( '--customhyperopt', - help='specify hyperopt class name (default: %(default)s)', + help='Specify hyperopt class name (default: %(default)s).', dest='hyperopt', default=constants.DEFAULT_HYPEROPT, type=str, @@ -114,8 +117,8 @@ class Arguments(object): ) self.parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist' - ' based on 24h BaseVolume (default: %(const)s)' + help='Dynamically generate and update whitelist' + ' based on 24h BaseVolume (default: %(const)s).' ' DEPRECATED.', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, @@ -126,7 +129,7 @@ class Arguments(object): self.parser.add_argument( '--db-url', help='Override trades database URL, this is useful if dry_run is enabled' - ' or in custom deployments (default: %(default)s)', + ' or in custom deployments (default: %(default)s).', dest='db_url', type=str, metavar='PATH', @@ -139,7 +142,7 @@ class Arguments(object): """ parser.add_argument( '--eps', '--enable-position-stacking', - help='Allow buying the same pair multiple times (position stacking)', + help='Allow buying the same pair multiple times (position stacking).', action='store_true', dest='position_stacking', default=False @@ -148,20 +151,20 @@ class Arguments(object): parser.add_argument( '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' - '(same as setting `max_open_trades` to a very high number)', + '(same as setting `max_open_trades` to a very high number).', action='store_false', dest='use_max_market_positions', default=True ) parser.add_argument( '-l', '--live', - help='using live data', + help='Use live data.', action='store_true', dest='live', ) parser.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from the ' + help='Refresh the pairs files in tests/testdata with the latest data from the ' 'exchange. Use it if you want to run your backtesting with up-to-date data.', action='store_true', dest='refresh_pairs', @@ -178,8 +181,8 @@ class Arguments(object): ) parser.add_argument( '--export', - help='export backtest results, argument are: trades\ - Example --export=trades', + help='Export backtest results, argument are: trades. ' + 'Example --export=trades', type=str, default=None, dest='export', @@ -203,14 +206,14 @@ class Arguments(object): """ parser.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from the ' + help='Refresh the pairs files in tests/testdata with the latest data from the ' 'exchange. Use it if you want to run your edge with up-to-date data.', action='store_true', dest='refresh_pairs', ) parser.add_argument( '--stoplosses', - help='defines a range of stoploss against which edge will assess the strategy ' + help='Defines a range of stoploss against which edge will assess the strategy ' 'the format is "min,max,step" (without any space).' 'example: --stoplosses=-0.01,-0.1,-0.001', type=str, @@ -226,14 +229,14 @@ class Arguments(object): """ parser.add_argument( '-i', '--ticker-interval', - help='specify ticker interval (1m, 5m, 30m, 1h, 1d)', + help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).', dest='ticker_interval', type=str, ) parser.add_argument( '--timerange', - help='specify what timerange of data to use.', + help='Specify what timerange of data to use.', default=None, type=str, dest='timerange', @@ -246,7 +249,7 @@ class Arguments(object): """ parser.add_argument( '--eps', '--enable-position-stacking', - help='Allow buying the same pair multiple times (position stacking)', + help='Allow buying the same pair multiple times (position stacking).', action='store_true', dest='position_stacking', default=False @@ -255,14 +258,14 @@ class Arguments(object): parser.add_argument( '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' - '(same as setting `max_open_trades` to a very high number)', + '(same as setting `max_open_trades` to a very high number).', action='store_false', dest='use_max_market_positions', default=True ) parser.add_argument( '-e', '--epochs', - help='specify number of epochs (default: %(default)d)', + help='Specify number of epochs (default: %(default)d).', dest='epochs', default=constants.HYPEROPT_EPOCH, type=int, @@ -271,7 +274,7 @@ class Arguments(object): parser.add_argument( '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ - Default: %(default)s', + Default: %(default)s.', choices=['all', 'buy', 'sell', 'roi', 'stoploss'], default='all', nargs='+', @@ -288,19 +291,19 @@ class Arguments(object): subparsers = self.parser.add_subparsers(dest='subparser') # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') + backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.') backtesting_cmd.set_defaults(func=backtesting.start) self.optimizer_shared_options(backtesting_cmd) self.backtesting_options(backtesting_cmd) # Add edge subcommand - edge_cmd = subparsers.add_parser('edge', help='edge module') + edge_cmd = subparsers.add_parser('edge', help='Edge module.') edge_cmd.set_defaults(func=edge_cli.start) self.optimizer_shared_options(edge_cmd) self.edge_options(edge_cmd) # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') + hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.') hyperopt_cmd.set_defaults(func=hyperopt.start) self.optimizer_shared_options(hyperopt_cmd) self.hyperopt_options(hyperopt_cmd) @@ -364,7 +367,7 @@ class Arguments(object): """ self.parser.add_argument( '--pairs-file', - help='File containing a list of pairs to download', + help='File containing a list of pairs to download.', dest='pairs_file', default=None, metavar='PATH', @@ -372,7 +375,7 @@ class Arguments(object): self.parser.add_argument( '--export', - help='Export files to given dir', + help='Export files to given dir.', dest='export', default=None, metavar='PATH', @@ -380,7 +383,8 @@ class Arguments(object): self.parser.add_argument( '-c', '--config', - help='specify configuration file, used for additional exchange parameters', + help='Specify configuration file, used for additional exchange parameters. ' + 'Multiple --config options may be used.', dest='config', default=None, type=str, @@ -389,7 +393,7 @@ class Arguments(object): self.parser.add_argument( '--days', - help='Download data for number of days', + help='Download data for given number of days.', dest='days', type=int, metavar='INT', @@ -398,7 +402,7 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name (default: %(default)s). Only valid if no config is provided', + help='Exchange name (default: %(default)s). Only valid if no config is provided.', dest='exchange', type=str, default='bittrex' @@ -407,7 +411,7 @@ class Arguments(object): self.parser.add_argument( '-t', '--timeframes', help='Specify which tickers to download. Space separated list. \ - Default: %(default)s', + Default: %(default)s.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], default=['1m', '5m'], @@ -417,7 +421,7 @@ class Arguments(object): self.parser.add_argument( '--erase', - help='Clean all existing data for the selected exchange/pairs/timeframes', + help='Clean all existing data for the selected exchange/pairs/timeframes.', dest='erase', action='store_true' ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index d972f50b8..bddf60028 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -13,6 +13,8 @@ from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants from freqtrade.state import RunMode +from freqtrade.misc import deep_merge_dicts + logger = logging.getLogger(__name__) @@ -45,8 +47,18 @@ class Configuration(object): Extract information for sys.argv and load the bot configuration :return: Configuration dictionary """ - logger.info('Using config: %s ...', self.args.config) - config = self._load_config_file(self.args.config) + config: Dict[str, Any] = {} + # Now expecting a list of config filenames here, not a string + for path in self.args.config: + logger.info('Using config: %s ...', path) + # Merge config options, overwriting old values + config = deep_merge_dicts(self._load_config_file(path), config) + + if 'internals' not in config: + config['internals'] = {} + + logger.info('Validating configuration ...') + self._validate_config(config) # Set strategy if not specified in config and or if it's non default if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'): @@ -93,11 +105,7 @@ class Configuration(object): f'Config file "{path}" not found!' ' Please create a config file or check whether it exists.') - if 'internals' not in conf: - conf['internals'] = {} - logger.info('Validating configuration ...') - - return self._validate_config(conf) + return conf def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 8fbcdfed7..c214f0de9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -3,6 +3,7 @@ """ bot constants """ +DEFAULT_CONFIG = 'config.json' DYNAMIC_WHITELIST = 20 # pairs PROCESS_THROTTLE_SECS = 5 # sec TICKER_INTERVAL = 5 # min diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d03187d77..38f758669 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -113,3 +113,21 @@ def format_ms_time(date: int) -> str: : epoch-string in ms """ return datetime.fromtimestamp(date/1000.0).strftime('%Y-%m-%dT%H:%M:%S') + + +def deep_merge_dicts(source, destination): + """ + >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } } + >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } } + >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } } + True + """ + for key, value in source.items(): + if isinstance(value, dict): + # get node or create one + node = destination.setdefault(key, {}) + deep_merge_dicts(value, node) + else: + destination[key] = value + + return destination