adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1571 from hroff-1902/patch-9 

multiple --config options
This commit is contained in:
Matthias 2019-02-24 13:50:39 +01:00 committed by GitHub
parent c7b6e19872 9b288c6933
commit 2531961bf8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 139 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
freqtrade/arguments.py
View File

@ -6,9 +6,7 @@ import argparse
import os
import re
from typing import List, NamedTuple, Optional
import arrow
from freqtrade import __version__, constants
@ -55,6 +53,11 @@ class Arguments(object):
"""
parsed_arg = self.parser.parse_args(self.args)
# Workaround issue in argparse with action='append' and default value
# (see https://bugs.python.org/issue16399)
if parsed_arg.config is None:
parsed_arg.config = [constants.DEFAULT_CONFIG]
return parsed_arg
def common_args_parser(self) -> None:
@ -63,7 +66,7 @@ class Arguments(object):
"""
self.parser.add_argument(
'-v', '--verbose',
help='verbose mode (-vv for more, -vvv to get all messages)',
help='Verbose mode (-vv for more, -vvv to get all messages).',
action='count',
dest='loglevel',
default=0,
@ -75,15 +78,16 @@ class Arguments(object):
)
self.parser.add_argument(
'-c', '--config',
help='specify configuration file (default: %(default)s)',
help='Specify configuration file (default: %(default)s). '
'Multiple --config options may be used.',
dest='config',
default='config.json',
action='append',
type=str,
metavar='PATH',
)
self.parser.add_argument(
'-d', '--datadir',
help='path to backtest data',
help='Path to backtest data.',
dest='datadir',
default=None,
type=str,
@ -91,7 +95,7 @@ class Arguments(object):
)
self.parser.add_argument(
'-s', '--strategy',
help='specify strategy class name (default: %(default)s)',
help='Specify strategy class name (default: %(default)s).',
dest='strategy',
default='DefaultStrategy',
type=str,
@ -99,14 +103,14 @@ class Arguments(object):
)
self.parser.add_argument(
'--strategy-path',
help='specify additional strategy lookup path',
help='Specify additional strategy lookup path.',
dest='strategy_path',
type=str,
metavar='PATH',
)
self.parser.add_argument(
'--customhyperopt',
help='specify hyperopt class name (default: %(default)s)',
help='Specify hyperopt class name (default: %(default)s).',
dest='hyperopt',
default=constants.DEFAULT_HYPEROPT,
type=str,
@ -114,8 +118,8 @@ class Arguments(object):
)
self.parser.add_argument(
'--dynamic-whitelist',
help='dynamically generate and update whitelist'
' based on 24h BaseVolume (default: %(const)s)'
help='Dynamically generate and update whitelist'
' based on 24h BaseVolume (default: %(const)s).'
' DEPRECATED.',
dest='dynamic_whitelist',
const=constants.DYNAMIC_WHITELIST,
@ -126,7 +130,7 @@ class Arguments(object):
self.parser.add_argument(
'--db-url',
help='Override trades database URL, this is useful if dry_run is enabled'
' or in custom deployments (default: %(default)s)',
' or in custom deployments (default: %(default)s).',
dest='db_url',
type=str,
metavar='PATH',
@ -139,7 +143,7 @@ class Arguments(object):
"""
parser.add_argument(
'--eps', '--enable-position-stacking',
help='Allow buying the same pair multiple times (position stacking)',
help='Allow buying the same pair multiple times (position stacking).',
action='store_true',
dest='position_stacking',
default=False
@ -148,20 +152,20 @@ class Arguments(object):
parser.add_argument(
'--dmmp', '--disable-max-market-positions',
help='Disable applying `max_open_trades` during backtest '
'(same as setting `max_open_trades` to a very high number)',
'(same as setting `max_open_trades` to a very high number).',
action='store_false',
dest='use_max_market_positions',
default=True
)
parser.add_argument(
'-l', '--live',
help='using live data',
help='Use live data.',
action='store_true',
dest='live',
)
parser.add_argument(
'-r', '--refresh-pairs-cached',
help='refresh the pairs files in tests/testdata with the latest data from the '
help='Refresh the pairs files in tests/testdata with the latest data from the '
'exchange. Use it if you want to run your backtesting with up-to-date data.',
action='store_true',
dest='refresh_pairs',
@ -178,8 +182,8 @@ class Arguments(object):
)
parser.add_argument(
'--export',
help='export backtest results, argument are: trades\
Example --export=trades',
help='Export backtest results, argument are: trades. '
'Example --export=trades',
type=str,
default=None,
dest='export',
@ -203,14 +207,14 @@ class Arguments(object):
"""
parser.add_argument(
'-r', '--refresh-pairs-cached',
help='refresh the pairs files in tests/testdata with the latest data from the '
help='Refresh the pairs files in tests/testdata with the latest data from the '
'exchange. Use it if you want to run your edge with up-to-date data.',
action='store_true',
dest='refresh_pairs',
)
parser.add_argument(
'--stoplosses',
help='defines a range of stoploss against which edge will assess the strategy '
help='Defines a range of stoploss against which edge will assess the strategy '
'the format is "min,max,step" (without any space).'
'example: --stoplosses=-0.01,-0.1,-0.001',
type=str,
@ -226,14 +230,14 @@ class Arguments(object):
"""
parser.add_argument(
'-i', '--ticker-interval',
help='specify ticker interval (1m, 5m, 30m, 1h, 1d)',
help='Specify ticker interval (1m, 5m, 30m, 1h, 1d).',
dest='ticker_interval',
type=str,
)
parser.add_argument(
'--timerange',
help='specify what timerange of data to use.',
help='Specify what timerange of data to use.',
default=None,
type=str,
dest='timerange',
@ -246,7 +250,7 @@ class Arguments(object):
"""
parser.add_argument(
'--eps', '--enable-position-stacking',
help='Allow buying the same pair multiple times (position stacking)',
help='Allow buying the same pair multiple times (position stacking).',
action='store_true',
dest='position_stacking',
default=False
@ -255,14 +259,14 @@ class Arguments(object):
parser.add_argument(
'--dmmp', '--disable-max-market-positions',
help='Disable applying `max_open_trades` during backtest '
'(same as setting `max_open_trades` to a very high number)',
'(same as setting `max_open_trades` to a very high number).',
action='store_false',
dest='use_max_market_positions',
default=True
)
parser.add_argument(
'-e', '--epochs',
help='specify number of epochs (default: %(default)d)',
help='Specify number of epochs (default: %(default)d).',
dest='epochs',
default=constants.HYPEROPT_EPOCH,
type=int,
@ -271,7 +275,7 @@ class Arguments(object):
parser.add_argument(
'-s', '--spaces',
help='Specify which parameters to hyperopt. Space separate list. \
Default: %(default)s',
Default: %(default)s.',
choices=['all', 'buy', 'sell', 'roi', 'stoploss'],
default='all',
nargs='+',
@ -288,19 +292,19 @@ class Arguments(object):
subparsers = self.parser.add_subparsers(dest='subparser')
# Add backtesting subcommand
backtesting_cmd = subparsers.add_parser('backtesting', help='backtesting module')
backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.')
backtesting_cmd.set_defaults(func=backtesting.start)
self.optimizer_shared_options(backtesting_cmd)
self.backtesting_options(backtesting_cmd)
# Add edge subcommand
edge_cmd = subparsers.add_parser('edge', help='edge module')
edge_cmd = subparsers.add_parser('edge', help='Edge module.')
edge_cmd.set_defaults(func=edge_cli.start)
self.optimizer_shared_options(edge_cmd)
self.edge_options(edge_cmd)
# Add hyperopt subcommand
hyperopt_cmd = subparsers.add_parser('hyperopt', help='hyperopt module')
hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.')
hyperopt_cmd.set_defaults(func=hyperopt.start)
self.optimizer_shared_options(hyperopt_cmd)
self.hyperopt_options(hyperopt_cmd)
@ -364,7 +368,7 @@ class Arguments(object):
"""
self.parser.add_argument(
'--pairs-file',
help='File containing a list of pairs to download',
help='File containing a list of pairs to download.',
dest='pairs_file',
default=None,
metavar='PATH',
@ -372,7 +376,7 @@ class Arguments(object):
self.parser.add_argument(
'--export',
help='Export files to given dir',
help='Export files to given dir.',
dest='export',
default=None,
metavar='PATH',
@ -380,16 +384,17 @@ class Arguments(object):
self.parser.add_argument(
'-c', '--config',
help='specify configuration file, used for additional exchange parameters',
help='Specify configuration file (default: %(default)s). '
'Multiple --config options may be used.',
dest='config',
default=None,
action='append',
type=str,
metavar='PATH',
)
self.parser.add_argument(
'--days',
help='Download data for number of days',
help='Download data for given number of days.',
dest='days',
type=int,
metavar='INT',
@ -398,7 +403,7 @@ class Arguments(object):
self.parser.add_argument(
'--exchange',
help='Exchange name (default: %(default)s). Only valid if no config is provided',
help='Exchange name (default: %(default)s). Only valid if no config is provided.',
dest='exchange',
type=str,
default='bittrex'
@ -407,7 +412,7 @@ class Arguments(object):
self.parser.add_argument(
'-t', '--timeframes',
help='Specify which tickers to download. Space separated list. \
Default: %(default)s',
Default: %(default)s.',
choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h',
'6h', '8h', '12h', '1d', '3d', '1w'],
default=['1m', '5m'],
@ -417,7 +422,7 @@ class Arguments(object):
self.parser.add_argument(
'--erase',
help='Clean all existing data for the selected exchange/pairs/timeframes',
help='Clean all existing data for the selected exchange/pairs/timeframes.',
dest='erase',
action='store_true'
)

22
freqtrade/configuration.py
View File

@ -13,6 +13,8 @@ from jsonschema.exceptions import ValidationError, best_match
from freqtrade import OperationalException, constants
from freqtrade.state import RunMode
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
@ -45,8 +47,18 @@ class Configuration(object):
Extract information for sys.argv and load the bot configuration
:return: Configuration dictionary
"""
logger.info('Using config: %s ...', self.args.config)
config = self._load_config_file(self.args.config)
config: Dict[str, Any] = {}
# Now expecting a list of config filenames here, not a string
for path in self.args.config:
logger.info('Using config: %s ...', path)
# Merge config options, overwriting old values
config = deep_merge_dicts(self._load_config_file(path), config)
if 'internals' not in config:
config['internals'] = {}
logger.info('Validating configuration ...')
self._validate_config(config)
# Set strategy if not specified in config and or if it's non default
if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'):
@ -93,11 +105,7 @@ class Configuration(object):
f'Config file "{path}" not found!'
' Please create a config file or check whether it exists.')
if 'internals' not in conf:
conf['internals'] = {}
logger.info('Validating configuration ...')
return self._validate_config(conf)
return conf
def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
"""

1
freqtrade/constants.py
View File

@ -3,6 +3,7 @@
"""
bot constants
"""
DEFAULT_CONFIG = 'config.json'
DYNAMIC_WHITELIST = 20 # pairs
PROCESS_THROTTLE_SECS = 5 # sec
TICKER_INTERVAL = 5 # min

18
freqtrade/misc.py
View File

@ -113,3 +113,21 @@ def format_ms_time(date: int) -> str:
: epoch-string in ms
"""
return datetime.fromtimestamp(date/1000.0).strftime('%Y-%m-%dT%H:%M:%S')
def deep_merge_dicts(source, destination):
"""
>>> a = { 'first' : { 'rows' : { 'pass' : 'dog', 'number' : '1' } } }
>>> b = { 'first' : { 'rows' : { 'fail' : 'cat', 'number' : '5' } } }
>>> merge(b, a) == { 'first' : { 'rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }
True
"""
for key, value in source.items():
if isinstance(value, dict):
# get node or create one
node = destination.setdefault(key, {})
deep_merge_dicts(value, node)
else:
destination[key] = value
return destination

15
freqtrade/tests/test_arguments.py
View File

@ -16,7 +16,7 @@ def test_parse_args_none() -> None:
def test_parse_args_defaults() -> None:
args = Arguments([], '').get_parsed_arg()
assert args.config == 'config.json'
assert args.config == ['config.json']
assert args.strategy_path is None
assert args.datadir is None
assert args.loglevel == 0
@ -24,10 +24,15 @@ def test_parse_args_defaults() -> None:
def test_parse_args_config() -> None:
args = Arguments(['-c', '/dev/null'], '').get_parsed_arg()
assert args.config == '/dev/null'
assert args.config == ['/dev/null']
args = Arguments(['--config', '/dev/null'], '').get_parsed_arg()
assert args.config == '/dev/null'
assert args.config == ['/dev/null']
args = Arguments(['--config', '/dev/null',
'--config', '/dev/zero'],
'').get_parsed_arg()
assert args.config == ['/dev/null', '/dev/zero']
def test_parse_args_db_url() -> None:
@ -139,7 +144,7 @@ def test_parse_args_backtesting_custom() -> None:
'TestStrategy'
]
call_args = Arguments(args, '').get_parsed_arg()
assert call_args.config == 'test_conf.json'
assert call_args.config == ['test_conf.json']
assert call_args.live is True
assert call_args.loglevel == 0
assert call_args.subparser == 'backtesting'
@ -158,7 +163,7 @@ def test_parse_args_hyperopt_custom() -> None:
'--spaces', 'buy'
]
call_args = Arguments(args, '').get_parsed_arg()
assert call_args.config == 'test_conf.json'
assert call_args.config == ['test_conf.json']
assert call_args.epochs == 20
assert call_args.loglevel == 0
assert call_args.subparser == 'hyperopt'

49
freqtrade/tests/test_configuration.py
View File

@ -1,15 +1,15 @@
# pragma pylint: disable=missing-docstring, protected-access, invalid-name
import json
from argparse import Namespace
import logging
from argparse import Namespace
from copy import deepcopy
from unittest.mock import MagicMock
import pytest
from jsonschema import validate, ValidationError, Draft4Validator
from jsonschema import Draft4Validator, ValidationError, validate
from freqtrade import constants
from freqtrade import OperationalException
from freqtrade import OperationalException, constants
from freqtrade.arguments import Arguments
from freqtrade.configuration import Configuration, set_loggers
from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
@ -50,18 +50,49 @@ def test_load_config_file(default_conf, mocker, caplog) -> None:
validated_conf = configuration._load_config_file('somefile')
assert file_mock.call_count == 1
assert validated_conf.items() >= default_conf.items()
assert 'internals' in validated_conf
assert log_has('Validating configuration ...', caplog.record_tuples)
def test_load_config_max_open_trades_zero(default_conf, mocker, caplog) -> None:
default_conf['max_open_trades'] = 0
file_mock = mocker.patch('freqtrade.configuration.open', mocker.mock_open(
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
Configuration(Namespace())._load_config_file('somefile')
assert file_mock.call_count == 1
args = Arguments([], '').get_parsed_arg()
configuration = Configuration(args)
validated_conf = configuration.load_config()
assert validated_conf['max_open_trades'] == 0
assert 'internals' in validated_conf
assert log_has('Validating configuration ...', caplog.record_tuples)
def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None:
conf1 = deepcopy(default_conf)
conf2 = deepcopy(default_conf)
del conf1['exchange']['key']
del conf1['exchange']['secret']
del conf2['exchange']['name']
conf2['exchange']['pair_whitelist'] += ['NANO/BTC']
config_files = [conf1, conf2]
configsmock = MagicMock(side_effect=config_files)
mocker.patch('freqtrade.configuration.Configuration._load_config_file', configsmock)
arg_list = ['-c', 'test_conf.json', '--config', 'test2_conf.json', ]
args = Arguments(arg_list, '').get_parsed_arg()
configuration = Configuration(args)
validated_conf = configuration.load_config()
exchange_conf = default_conf['exchange']
assert validated_conf['exchange']['name'] == exchange_conf['name']
assert validated_conf['exchange']['key'] == exchange_conf['key']
assert validated_conf['exchange']['secret'] == exchange_conf['secret']
assert validated_conf['exchange']['pair_whitelist'] != conf1['exchange']['pair_whitelist']
assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist']
assert 'internals' in validated_conf
assert log_has('Validating configuration ...', caplog.record_tuples)

4
freqtrade/tests/test_main.py
View File

@ -22,7 +22,7 @@ def test_parse_args_backtesting(mocker) -> None:
main(['backtesting'])
assert backtesting_mock.call_count == 1
call_args = backtesting_mock.call_args[0][0]
assert call_args.config == 'config.json'
assert call_args.config == ['config.json']
assert call_args.live is False
assert call_args.loglevel == 0
assert call_args.subparser == 'backtesting'
@ -35,7 +35,7 @@ def test_main_start_hyperopt(mocker) -> None:
main(['hyperopt'])
assert hyperopt_mock.call_count == 1
call_args = hyperopt_mock.call_args[0][0]
assert call_args.config == 'config.json'
assert call_args.config == ['config.json']
assert call_args.loglevel == 0
assert call_args.subparser == 'hyperopt'
assert call_args.func is not None

14
scripts/download_backtest_data.py
View File

@ -5,12 +5,14 @@ import json
import sys
from pathlib import Path
import arrow
from typing import Any, Dict
from freqtrade import arguments
from freqtrade.arguments import Arguments
from freqtrade.arguments import TimeRange
from freqtrade.exchange import Exchange
from freqtrade.data.history import download_pair_history
from freqtrade.configuration import Configuration, set_loggers
from freqtrade.misc import deep_merge_dicts
import logging
logging.basicConfig(
@ -21,7 +23,7 @@ set_loggers(0)
DEFAULT_DL_PATH = 'user_data/data'
arguments = arguments.Arguments(sys.argv[1:], 'download utility')
arguments = Arguments(sys.argv[1:], 'download utility')
arguments.testdata_dl_options()
args = arguments.parse_args()
@ -29,7 +31,13 @@ timeframes = args.timeframes
if args.config:
configuration = Configuration(args)
config = configuration._load_config_file(args.config)
config: Dict[str, Any] = {}
# Now expecting a list of config filenames here, not a string
for path in args.config:
print('Using config: %s ...', path)
# Merge config options, overwriting old values
config = deep_merge_dicts(configuration._load_config_file(path), config)
config['stake_currency'] = ''
# Ensure we do not use Exchange credentials