From 2f225e23408bc6c47118840bfddfb829ba6d6776 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 19 Feb 2019 15:14:47 +0300 Subject: [PATCH 1/6] multiple --config options --- freqtrade/arguments.py | 76 ++++++++++++++++++++------------------ freqtrade/configuration.py | 22 +++++++---- freqtrade/constants.py | 1 + freqtrade/misc.py | 18 +++++++++ 4 files changed, 74 insertions(+), 43 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 9b1b9a925..53a46d141 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -6,9 +6,7 @@ import argparse import os import re from typing import List, NamedTuple, Optional - import arrow - from freqtrade import __version__, constants @@ -55,6 +53,11 @@ class Arguments(object): """ parsed_arg = self.parser.parse_args(self.args) + # Workaround issue in argparse with action='append' and default value + # (see https://bugs.python.org/issue16399) + if parsed_arg.config is None: + parsed_arg.config = [constants.DEFAULT_CONFIG] + return parsed_arg def common_args_parser(self) -> None: @@ -63,7 +66,7 @@ class Arguments(object): """ self.parser.add_argument( '-v', '--verbose', - help='verbose mode (-vv for more, -vvv to get all messages)', + help='Verbose mode (-vv for more, -vvv to get all messages).', action='count', dest='loglevel', default=0, @@ -75,15 +78,15 @@ class Arguments(object): ) self.parser.add_argument( '-c', '--config', - help='specify configuration file (default: %(default)s)', + help='Specify configuration file (default: %(default)s).', dest='config', - default='config.json', + action='append', type=str, metavar='PATH', ) self.parser.add_argument( '-d', '--datadir', - help='path to backtest data', + help='Path to backtest data.', dest='datadir', default=None, type=str, @@ -91,7 +94,7 @@ class Arguments(object): ) self.parser.add_argument( '-s', '--strategy', - help='specify strategy class name (default: %(default)s)', + help='Specify strategy class name (default: %(default)s).', dest='strategy', default='DefaultStrategy', type=str, @@ -99,14 +102,14 @@ class Arguments(object): ) self.parser.add_argument( '--strategy-path', - help='specify additional strategy lookup path', + help='Specify additional strategy lookup path.', dest='strategy_path', type=str, metavar='PATH', ) self.parser.add_argument( '--customhyperopt', - help='specify hyperopt class name (default: %(default)s)', + help='Specify hyperopt class name (default: %(default)s).', dest='hyperopt', default=constants.DEFAULT_HYPEROPT, type=str, @@ -114,8 +117,8 @@ class Arguments(object): ) self.parser.add_argument( '--dynamic-whitelist', - help='dynamically generate and update whitelist' - ' based on 24h BaseVolume (default: %(const)s)' + help='Dynamically generate and update whitelist' + ' based on 24h BaseVolume (default: %(const)s).' ' DEPRECATED.', dest='dynamic_whitelist', const=constants.DYNAMIC_WHITELIST, @@ -126,7 +129,7 @@ class Arguments(object): self.parser.add_argument( '--db-url', help='Override trades database URL, this is useful if dry_run is enabled' - ' or in custom deployments (default: %(default)s)', + ' or in custom deployments (default: %(default)s).', dest='db_url', type=str, metavar='PATH', @@ -139,7 +142,7 @@ class Arguments(object): """ parser.add_argument( '--eps', '--enable-position-stacking', - help='Allow buying the same pair multiple times (position stacking)', + help='Allow buying the same pair multiple times (position stacking).', action='store_true', dest='position_stacking', default=False @@ -148,20 +151,20 @@ class Arguments(object): parser.add_argument( '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' - '(same as setting `max_open_trades` to a very high number)', + '(same as setting `max_open_trades` to a very high number).', action='store_false', dest='use_max_market_positions', default=True ) parser.add_argument( '-l', '--live', - help='using live data', + help='Use live data.', action='store_true', dest='live', ) parser.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from the ' + help='Refresh the pairs files in tests/testdata with the latest data from the ' 'exchange. Use it if you want to run your backtesting with up-to-date data.', action='store_true', dest='refresh_pairs', @@ -178,8 +181,8 @@ class Arguments(object): ) parser.add_argument( '--export', - help='export backtest results, argument are: trades\ - Example --export=trades', + help='Export backtest results, argument are: trades. ' + 'Example --export=trades', type=str, default=None, dest='export', @@ -203,14 +206,14 @@ class Arguments(object): """ parser.add_argument( '-r', '--refresh-pairs-cached', - help='refresh the pairs files in tests/testdata with the latest data from the ' + help='Refresh the pairs files in tests/testdata with the latest data from the ' 'exchange. Use it if you want to run your edge with up-to-date data.', action='store_true', dest='refresh_pairs', ) parser.add_argument( '--stoplosses', - help='defines a range of stoploss against which edge will assess the strategy ' + help='Defines a range of stoploss against which edge will assess the strategy ' 'the format is "min,max,step" (without any space).' 'example: --stoplosses=-0.01,-0.1,-0.001', type=str, @@ -226,14 +229,14 @@ class Arguments(object): """ parser.add_argument( '-i', '--ticker-interval', - help='specify ticker interval (1m, 5m, 30m, 1h, 1d)', + help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).', dest='ticker_interval', type=str, ) parser.add_argument( '--timerange', - help='specify what timerange of data to use.', + help='Specify what timerange of data to use.', default=None, type=str, dest='timerange', @@ -246,7 +249,7 @@ class Arguments(object): """ parser.add_argument( '--eps', '--enable-position-stacking', - help='Allow buying the same pair multiple times (position stacking)', + help='Allow buying the same pair multiple times (position stacking).', action='store_true', dest='position_stacking', default=False @@ -255,14 +258,14 @@ class Arguments(object): parser.add_argument( '--dmmp', '--disable-max-market-positions', help='Disable applying `max_open_trades` during backtest ' - '(same as setting `max_open_trades` to a very high number)', + '(same as setting `max_open_trades` to a very high number).', action='store_false', dest='use_max_market_positions', default=True ) parser.add_argument( '-e', '--epochs', - help='specify number of epochs (default: %(default)d)', + help='Specify number of epochs (default: %(default)d).', dest='epochs', default=constants.HYPEROPT_EPOCH, type=int, @@ -271,7 +274,7 @@ class Arguments(object): parser.add_argument( '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ - Default: %(default)s', + Default: %(default)s.', choices=['all', 'buy', 'sell', 'roi', 'stoploss'], default='all', nargs='+', @@ -288,19 +291,19 @@ class Arguments(object): subparsers = self.parser.add_subparsers(dest='subparser') # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module') + backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.') backtesting_cmd.set_defaults(func=backtesting.start) self.optimizer_shared_options(backtesting_cmd) self.backtesting_options(backtesting_cmd) # Add edge subcommand - edge_cmd = subparsers.add_parser('edge', help='edge module') + edge_cmd = subparsers.add_parser('edge', help='Edge module.') edge_cmd.set_defaults(func=edge_cli.start) self.optimizer_shared_options(edge_cmd) self.edge_options(edge_cmd) # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module') + hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.') hyperopt_cmd.set_defaults(func=hyperopt.start) self.optimizer_shared_options(hyperopt_cmd) self.hyperopt_options(hyperopt_cmd) @@ -364,7 +367,7 @@ class Arguments(object): """ self.parser.add_argument( '--pairs-file', - help='File containing a list of pairs to download', + help='File containing a list of pairs to download.', dest='pairs_file', default=None, metavar='PATH', @@ -372,7 +375,7 @@ class Arguments(object): self.parser.add_argument( '--export', - help='Export files to given dir', + help='Export files to given dir.', dest='export', default=None, metavar='PATH', @@ -380,7 +383,8 @@ class Arguments(object): self.parser.add_argument( '-c', '--config', - help='specify configuration file, used for additional exchange parameters', + help='Specify configuration file, used for additional exchange parameters. ' + 'Multiple --config options may be used.', dest='config', default=None, type=str, @@ -389,7 +393,7 @@ class Arguments(object): self.parser.add_argument( '--days', - help='Download data for number of days', + help='Download data for given number of days.', dest='days', type=int, metavar='INT', @@ -398,7 +402,7 @@ class Arguments(object): self.parser.add_argument( '--exchange', - help='Exchange name (default: %(default)s). Only valid if no config is provided', + help='Exchange name (default: %(default)s). Only valid if no config is provided.', dest='exchange', type=str, default='bittrex' @@ -407,7 +411,7 @@ class Arguments(object): self.parser.add_argument( '-t', '--timeframes', help='Specify which tickers to download. Space separated list. \ - Default: %(default)s', + Default: %(default)s.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], default=['1m', '5m'], @@ -417,7 +421,7 @@ class Arguments(object): self.parser.add_argument( '--erase', - help='Clean all existing data for the selected exchange/pairs/timeframes', + help='Clean all existing data for the selected exchange/pairs/timeframes.', dest='erase', action='store_true' ) diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index d972f50b8..bddf60028 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -13,6 +13,8 @@ from jsonschema.exceptions import ValidationError, best_match from freqtrade import OperationalException, constants from freqtrade.state import RunMode +from freqtrade.misc import deep_merge_dicts + logger = logging.getLogger(__name__) @@ -45,8 +47,18 @@ class Configuration(object): Extract information for sys.argv and load the bot configuration :return: Configuration dictionary """ - logger.info('Using config: %s ...', self.args.config) - config = self._load_config_file(self.args.config) + config: Dict[str, Any] = {} + # Now expecting a list of config filenames here, not a string + for path in self.args.config: + logger.info('Using config: %s ...', path) + # Merge config options, overwriting old values + config = deep_merge_dicts(self._load_config_file(path), config) + + if 'internals' not in config: + config['internals'] = {} + + logger.info('Validating configuration ...') + self._validate_config(config) # Set strategy if not specified in config and or if it's non default if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'): @@ -93,11 +105,7 @@ class Configuration(object): f'Config file "{path}" not found!' ' Please create a config file or check whether it exists.') - if 'internals' not in conf: - conf['internals'] = {} - logger.info('Validating configuration ...') - - return self._validate_config(conf) + return conf def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 8fbcdfed7..c214f0de9 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -3,6 +3,7 @@ """ bot constants """ +DEFAULT_CONFIG = 'config.json' DYNAMIC_WHITELIST = 20 # pairs PROCESS_THROTTLE_SECS = 5 # sec TICKER_INTERVAL = 5 # min diff --git a/freqtrade/misc.py b/freqtrade/misc.py index d03187d77..38f758669 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -113,3 +113,21 @@ def format_ms_time(date: int) -> str: : epoch-string in ms """ return datetime.fromtimestamp(date/1000.0).strftime('%Y-%m-%dT%H:%M:%S') + + +def deep_merge_dicts(source, destination): + """ + >>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } } + >>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } } + >>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } } + True + """ + for key, value in source.items(): + if isinstance(value, dict): + # get node or create one + node = destination.setdefault(key, {}) + deep_merge_dicts(value, node) + else: + destination[key] = value + + return destination From c08a2b6638f735c0f188b2f94d59f539d714cf0d Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 20 Feb 2019 16:23:09 +0300 Subject: [PATCH 2/6] help message fixed --- freqtrade/arguments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 53a46d141..f462926f9 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -78,7 +78,8 @@ class Arguments(object): ) self.parser.add_argument( '-c', '--config', - help='Specify configuration file (default: %(default)s).', + help='Specify configuration file (default: %(default)s). ' + 'Multiple --config options may be used.', dest='config', action='append', type=str, From 87c82dea3d93a8321a6594e024e48ad3b30c5b17 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 20 Feb 2019 17:00:35 +0300 Subject: [PATCH 3/6] support for multiple --config in the download_backtest_data.py utility --- freqtrade/arguments.py | 4 ++-- scripts/download_backtest_data.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index f462926f9..62f22befc 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -384,10 +384,10 @@ class Arguments(object): self.parser.add_argument( '-c', '--config', - help='Specify configuration file, used for additional exchange parameters. ' + help='Specify configuration file (default: %(default)s). ' 'Multiple --config options may be used.', dest='config', - default=None, + action='append', type=str, metavar='PATH', ) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index c8fd08747..df3e4bb0c 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -5,12 +5,14 @@ import json import sys from pathlib import Path import arrow +from typing import Any, Dict -from freqtrade import arguments +from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.exchange import Exchange from freqtrade.data.history import download_pair_history from freqtrade.configuration import Configuration, set_loggers +from freqtrade.misc import deep_merge_dicts import logging logging.basicConfig( @@ -21,7 +23,7 @@ set_loggers(0) DEFAULT_DL_PATH = 'user_data/data' -arguments = arguments.Arguments(sys.argv[1:], 'download utility') +arguments = Arguments(sys.argv[1:], 'download utility') arguments.testdata_dl_options() args = arguments.parse_args() @@ -29,7 +31,15 @@ timeframes = args.timeframes if args.config: configuration = Configuration(args) - config = configuration._load_config_file(args.config) + + config: Dict[str, Any] = {} + # Now expecting a list of config filenames here, not a string + for path in args.config: + print('Using config: %s ...', path) + # Merge config options, overwriting old values + config = deep_merge_dicts(configuration._load_config_file(path), config) + +### config = configuration._load_config_file(args.config) config['stake_currency'] = '' # Ensure we do not use Exchange credentials From 4fbba98168dfaca60733b055dc21e10f22ea5b12 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 20 Feb 2019 17:54:20 +0300 Subject: [PATCH 4/6] tests adjusted for multiple --config options --- freqtrade/tests/test_arguments.py | 15 ++++++++++----- freqtrade/tests/test_configuration.py | 12 +++++++----- freqtrade/tests/test_main.py | 4 ++-- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 042d43ed2..0952d1c5d 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -16,7 +16,7 @@ def test_parse_args_none() -> None: def test_parse_args_defaults() -> None: args = Arguments([], '').get_parsed_arg() - assert args.config == 'config.json' + assert args.config == ['config.json'] assert args.strategy_path is None assert args.datadir is None assert args.loglevel == 0 @@ -24,10 +24,15 @@ def test_parse_args_defaults() -> None: def test_parse_args_config() -> None: args = Arguments(['-c', '/dev/null'], '').get_parsed_arg() - assert args.config == '/dev/null' + assert args.config == ['/dev/null'] args = Arguments(['--config', '/dev/null'], '').get_parsed_arg() - assert args.config == '/dev/null' + assert args.config == ['/dev/null'] + + args = Arguments(['--config', '/dev/null', + '--config', '/dev/zero'], + '').get_parsed_arg() + assert args.config == ['/dev/null', '/dev/zero'] def test_parse_args_db_url() -> None: @@ -139,7 +144,7 @@ def test_parse_args_backtesting_custom() -> None: 'TestStrategy' ] call_args = Arguments(args, '').get_parsed_arg() - assert call_args.config == 'test_conf.json' + assert call_args.config == ['test_conf.json'] assert call_args.live is True assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' @@ -158,7 +163,7 @@ def test_parse_args_hyperopt_custom() -> None: '--spaces', 'buy' ] call_args = Arguments(args, '').get_parsed_arg() - assert call_args.config == 'test_conf.json' + assert call_args.config == ['test_conf.json'] assert call_args.epochs == 20 assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 67445238b..9f11e8ac2 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -50,18 +50,20 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: validated_conf = configuration._load_config_file('somefile') assert file_mock.call_count == 1 assert validated_conf.items() >= default_conf.items() - assert 'internals' in validated_conf - assert log_has('Validating configuration ...', caplog.record_tuples) def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: default_conf['max_open_trades'] = 0 - file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open( + mocker.patch('freqtrade.configuration.open', mocker.mock_open( read_data=json.dumps(default_conf) )) - Configuration(Namespace())._load_config_file('somefile') - assert file_mock.call_count == 1 + args = Arguments([], '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + + assert validated_conf['max_open_trades'] == 0 + assert 'internals' in validated_conf assert log_has('Validating configuration ...', caplog.record_tuples) diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 7aae98ebe..51c95a4a9 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -22,7 +22,7 @@ def test_parse_args_backtesting(mocker) -> None: main(['backtesting']) assert backtesting_mock.call_count == 1 call_args = backtesting_mock.call_args[0][0] - assert call_args.config == 'config.json' + assert call_args.config == ['config.json'] assert call_args.live is False assert call_args.loglevel == 0 assert call_args.subparser == 'backtesting' @@ -35,7 +35,7 @@ def test_main_start_hyperopt(mocker) -> None: main(['hyperopt']) assert hyperopt_mock.call_count == 1 call_args = hyperopt_mock.call_args[0][0] - assert call_args.config == 'config.json' + assert call_args.config == ['config.json'] assert call_args.loglevel == 0 assert call_args.subparser == 'hyperopt' assert call_args.func is not None From da5bef501e4a3f7330beac5c1d44b3192385515b Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Wed, 20 Feb 2019 17:55:20 +0300 Subject: [PATCH 5/6] cleanup --- scripts/download_backtest_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py index df3e4bb0c..5dee41bdd 100755 --- a/scripts/download_backtest_data.py +++ b/scripts/download_backtest_data.py @@ -39,8 +39,6 @@ if args.config: # Merge config options, overwriting old values config = deep_merge_dicts(configuration._load_config_file(path), config) -### config = configuration._load_config_file(args.config) - config['stake_currency'] = '' # Ensure we do not use Exchange credentials config['exchange']['key'] = '' From 9b288c6933792353c1db7209df0fffd7153b435e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Feb 2019 13:29:22 +0100 Subject: [PATCH 6/6] Add test to specifically test for merged dict --- freqtrade/tests/test_configuration.py | 37 ++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index 9f11e8ac2..51098baaa 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -1,15 +1,15 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name import json -from argparse import Namespace import logging +from argparse import Namespace +from copy import deepcopy from unittest.mock import MagicMock import pytest -from jsonschema import validate, ValidationError, Draft4Validator +from jsonschema import Draft4Validator, ValidationError, validate -from freqtrade import constants -from freqtrade import OperationalException +from freqtrade import OperationalException, constants from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration, set_loggers from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL @@ -67,6 +67,35 @@ def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None: assert log_has('Validating configuration ...', caplog.record_tuples) +def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None: + conf1 = deepcopy(default_conf) + conf2 = deepcopy(default_conf) + del conf1['exchange']['key'] + del conf1['exchange']['secret'] + del conf2['exchange']['name'] + conf2['exchange']['pair_whitelist'] += ['NANO/BTC'] + + config_files = [conf1, conf2] + + configsmock = MagicMock(side_effect=config_files) + mocker.patch('freqtrade.configuration.Configuration._load_config_file', configsmock) + + arg_list = ['-c', 'test_conf.json', '--config', 'test2_conf.json', ] + args = Arguments(arg_list, '').get_parsed_arg() + configuration = Configuration(args) + validated_conf = configuration.load_config() + + exchange_conf = default_conf['exchange'] + assert validated_conf['exchange']['name'] == exchange_conf['name'] + assert validated_conf['exchange']['key'] == exchange_conf['key'] + assert validated_conf['exchange']['secret'] == exchange_conf['secret'] + assert validated_conf['exchange']['pair_whitelist'] != conf1['exchange']['pair_whitelist'] + assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist'] + + assert 'internals' in validated_conf + assert log_has('Validating configuration ...', caplog.record_tuples) + + def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> None: default_conf['max_open_trades'] = -1 mocker.patch('freqtrade.configuration.open', mocker.mock_open(