Merge pull request #2260 from freqtrade/args_vars

Configuration/Arguments refactoing (don't pass Namespace around).
This commit is contained in:
Matthias 2019-09-14 10:11:02 +02:00 committed by GitHub
commit 19ce7180be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 133 additions and 124 deletions

View File

@ -91,7 +91,8 @@ df.groupby("pair")["sell_reason"].value_counts()
### Load multiple configuration files
This option can be useful to inspect the results of passing in multiple configs
This option can be useful to inspect the results of passing in multiple configs.
This will also run through the whole Configuration initialization, so the configuration is completely initialized to be passed to other methods.
``` python
import json
@ -101,7 +102,16 @@ from freqtrade.configuration import Configuration
config = Configuration.from_files(["config1.json", "config2.json"])
# Show the config in memory
print(json.dumps(config, indent=1))
print(json.dumps(config, indent=2))
```
For Interactive environments, have an additional configuration specifying `user_data_dir` and pass this in last, so you don't have to change directories while running the bot.
Best avoid relative paths, since this starts at the storage location of the jupyter notebook, unless the directory is changed.
``` json
{
"user_data_dir": "~/.freqtrade/"
}
```
### Load exchange data to a pandas dataframe

View File

@ -2,11 +2,11 @@
This module contains the argument manager class
"""
import argparse
from typing import List, Optional
from pathlib import Path
from typing import Any, Dict, List, Optional
from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS
from freqtrade import constants
from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS
ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
@ -41,7 +41,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "ticker_interval"]
NO_CONF_REQURIED = ["download-data", "plot-dataframe", "plot-profit"]
NO_CONF_REQURIED = ["create-userdir", "download-data", "plot-dataframe", "plot-profit"]
class Arguments:
@ -57,7 +57,7 @@ class Arguments:
self._build_args(optionlist=ARGS_MAIN)
self._build_subcommands()
def get_parsed_arg(self) -> argparse.Namespace:
def get_parsed_arg(self) -> Dict[str, Any]:
"""
Return the list of arguments
:return: List[str] List of arguments
@ -66,7 +66,7 @@ class Arguments:
self._load_args()
self._parsed_arg = self._parse_args()
return self._parsed_arg
return vars(self._parsed_arg)
def _parse_args(self) -> argparse.Namespace:
"""

View File

@ -3,7 +3,6 @@ This module contains the configuration class
"""
import logging
import warnings
from argparse import Namespace
from copy import deepcopy
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
@ -28,7 +27,7 @@ class Configuration:
Reuse this class for the bot, backtesting, hyperopt and every script that required configuration
"""
def __init__(self, args: Namespace, runmode: RunMode = None) -> None:
def __init__(self, args: Dict[str, Any], runmode: RunMode = None) -> None:
self.args = args
self.config: Optional[Dict[str, Any]] = None
self.runmode = runmode
@ -50,9 +49,16 @@ class Configuration:
and merging their contents.
Files are loaded in sequence, parameters in later configuration files
override the same parameter from an earlier file (last definition wins).
Runs through the whole Configuration initialization, so all expected config entries
are available to interactive environments.
:param files: List of file paths
:return: configuration dictionary
"""
c = Configuration({"config": files}, RunMode.OTHER)
return c.get_config()
def load_from_files(self, files: List[str]) -> Dict[str, Any]:
# Keep this method as staticmethod, so it can be used from interactive environments
config: Dict[str, Any] = {}
@ -82,7 +88,7 @@ class Configuration:
:return: Configuration dictionary
"""
# Load all configs
config: Dict[str, Any] = Configuration.from_files(self.args.config)
config: Dict[str, Any] = self.load_from_files(self.args["config"])
self._process_common_options(config)
@ -107,13 +113,10 @@ class Configuration:
the -v/--verbose, --logfile options
"""
# Log level
if 'verbosity' in self.args and self.args.verbosity:
config.update({'verbosity': self.args.verbosity})
else:
config.update({'verbosity': 0})
config.update({'verbosity': self.args.get("verbosity", 0)})
if 'logfile' in self.args and self.args.logfile:
config.update({'logfile': self.args.logfile})
if 'logfile' in self.args and self.args["logfile"]:
config.update({'logfile': self.args["logfile"]})
setup_logging(config)
@ -122,15 +125,15 @@ class Configuration:
self._process_logging_options(config)
# Set strategy if not specified in config and or if it's non default
if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'):
config.update({'strategy': self.args.strategy})
if self.args.get("strategy") != constants.DEFAULT_STRATEGY or not config.get('strategy'):
config.update({'strategy': self.args.get("strategy")})
self._args_to_config(config, argname='strategy_path',
logstring='Using additional Strategy lookup path: {}')
if ('db_url' in self.args and self.args.db_url and
self.args.db_url != constants.DEFAULT_DB_PROD_URL):
config.update({'db_url': self.args.db_url})
if ('db_url' in self.args and self.args["db_url"] and
self.args["db_url"] != constants.DEFAULT_DB_PROD_URL):
config.update({'db_url': self.args["db_url"]})
logger.info('Parameter --db-url detected ...')
if config.get('dry_run', False):
@ -153,7 +156,7 @@ class Configuration:
config['max_open_trades'] = float('inf')
# Support for sd_notify
if 'sd_notify' in self.args and self.args.sd_notify:
if 'sd_notify' in self.args and self.args["sd_notify"]:
config['internals'].update({'sd_notify': True})
def _process_datadir_options(self, config: Dict[str, Any]) -> None:
@ -162,12 +165,12 @@ class Configuration:
--user-data, --datadir
"""
# Check exchange parameter here - otherwise `datadir` might be wrong.
if "exchange" in self.args and self.args.exchange:
config['exchange']['name'] = self.args.exchange
if "exchange" in self.args and self.args["exchange"]:
config['exchange']['name'] = self.args["exchange"]
logger.info(f"Using exchange {config['exchange']['name']}")
if 'user_data_dir' in self.args and self.args.user_data_dir:
config.update({'user_data_dir': self.args.user_data_dir})
if 'user_data_dir' in self.args and self.args["user_data_dir"]:
config.update({'user_data_dir': self.args["user_data_dir"]})
elif 'user_data_dir' not in config:
# Default to cwd/user_data (legacy option ...)
config.update({'user_data_dir': str(Path.cwd() / "user_data")})
@ -176,10 +179,7 @@ class Configuration:
config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False)
logger.info('Using user-data directory: %s ...', config['user_data_dir'])
if 'datadir' in self.args and self.args.datadir:
config.update({'datadir': create_datadir(config, self.args.datadir)})
else:
config.update({'datadir': create_datadir(config, None)})
config.update({'datadir': create_datadir(config, self.args.get("datadir", None))})
logger.info('Using data directory: %s ...', config.get('datadir'))
def _process_optimize_options(self, config: Dict[str, Any]) -> None:
@ -192,12 +192,12 @@ class Configuration:
self._args_to_config(config, argname='position_stacking',
logstring='Parameter --enable-position-stacking detected ...')
if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions:
if 'use_max_market_positions' in self.args and not self.args["use_max_market_positions"]:
config.update({'use_max_market_positions': False})
logger.info('Parameter --disable-max-market-positions detected ...')
logger.info('max_open_trades set to unlimited ...')
elif 'max_open_trades' in self.args and self.args.max_open_trades:
config.update({'max_open_trades': self.args.max_open_trades})
elif 'max_open_trades' in self.args and self.args["max_open_trades"]:
config.update({'max_open_trades': self.args["max_open_trades"]})
logger.info('Parameter --max_open_trades detected, '
'overriding max_open_trades to: %s ...', config.get('max_open_trades'))
else:
@ -229,12 +229,12 @@ class Configuration:
logstring='Storing backtest results to {} ...')
# Edge section:
if 'stoploss_range' in self.args and self.args.stoploss_range:
txt_range = eval(self.args.stoploss_range)
if 'stoploss_range' in self.args and self.args["stoploss_range"]:
txt_range = eval(self.args["stoploss_range"])
config['edge'].update({'stoploss_range_min': txt_range[0]})
config['edge'].update({'stoploss_range_max': txt_range[1]})
config['edge'].update({'stoploss_range_step': txt_range[2]})
logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range)
logger.info('Parameter --stoplosses detected: %s ...', self.args["stoploss_range"])
# Hyperopt section
self._args_to_config(config, argname='hyperopt',
@ -254,7 +254,7 @@ class Configuration:
self._args_to_config(config, argname='print_all',
logstring='Parameter --print-all detected ...')
if 'print_colorized' in self.args and not self.args.print_colorized:
if 'print_colorized' in self.args and not self.args["print_colorized"]:
logger.info('Parameter --no-color detected ...')
config.update({'print_colorized': False})
else:
@ -324,9 +324,9 @@ class Configuration:
sample: logfun=len (prints the length of the found
configuration instead of the content)
"""
if argname in self.args and getattr(self.args, argname):
if argname in self.args and self.args[argname]:
config.update({argname: getattr(self.args, argname)})
config.update({argname: self.args[argname]})
if logfun:
logger.info(logstring.format(logfun(config[argname])))
else:
@ -346,8 +346,8 @@ class Configuration:
if "pairs" in config:
return
if "pairs_file" in self.args and self.args.pairs_file:
pairs_file = Path(self.args.pairs_file)
if "pairs_file" in self.args and self.args["pairs_file"]:
pairs_file = Path(self.args["pairs_file"])
logger.info(f'Reading pairs file "{pairs_file}".')
# Download pairs from the pairs file if no config is specified
# or if pairs file is specified explicitely
@ -358,7 +358,7 @@ class Configuration:
config['pairs'].sort()
return
if "config" in self.args and self.args.config:
if "config" in self.args and self.args["config"]:
logger.info("Using pairlist from configuration.")
config['pairs'] = config.get('exchange', {}).get('pair_whitelist')
else:

View File

@ -11,7 +11,6 @@ if sys.version_info < (3, 6):
# flake8: noqa E402
import logging
from argparse import Namespace
from typing import Any, List
from freqtrade import OperationalException
@ -32,12 +31,12 @@ def main(sysargv: List[str] = None) -> None:
worker = None
try:
arguments = Arguments(sysargv)
args: Namespace = arguments.get_parsed_arg()
args = arguments.get_parsed_arg()
# A subcommand has been issued.
# Means if Backtesting or Hyperopt have been called we exit the bot
if hasattr(args, 'func'):
args.func(args)
if 'func' in args:
args['func'](args)
# TODO: fetch return_code as returned by the command function here
return_code = 0
else:

View File

@ -1,5 +1,4 @@
import logging
from argparse import Namespace
from typing import Any, Dict
from filelock import FileLock, Timeout
@ -12,7 +11,7 @@ from freqtrade.utils import setup_utils_configuration
logger = logging.getLogger(__name__)
def setup_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]:
def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
"""
Prepare the configuration for the Hyperopt module
:param args: Cli args from Arguments()
@ -28,7 +27,7 @@ def setup_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]:
return config
def start_backtesting(args: Namespace) -> None:
def start_backtesting(args: Dict[str, Any]) -> None:
"""
Start Backtesting script
:param args: Cli args from Arguments()
@ -47,7 +46,7 @@ def start_backtesting(args: Namespace) -> None:
backtesting.start()
def start_hyperopt(args: Namespace) -> None:
def start_hyperopt(args: Dict[str, Any]) -> None:
"""
Start hyperopt script
:param args: Cli args from Arguments()
@ -85,7 +84,7 @@ def start_hyperopt(args: Namespace) -> None:
# Same in Edge and Backtesting start() functions.
def start_edge(args: Namespace) -> None:
def start_edge(args: Dict[str, Any]) -> None:
"""
Start Edge script
:param args: Cli args from Arguments()

View File

@ -1,18 +1,18 @@
from argparse import Namespace
from typing import Any, Dict
from freqtrade import OperationalException
from freqtrade.state import RunMode
from freqtrade.utils import setup_utils_configuration
def validate_plot_args(args: Namespace):
args_tmp = vars(args)
if not args_tmp.get('datadir') and not args_tmp.get('config'):
def validate_plot_args(args: Dict[str, Any]):
if not args.get('datadir') and not args.get('config'):
raise OperationalException(
"You need to specify either `--datadir` or `--config` "
"for plot-profit and plot-dataframe.")
def start_plot_dataframe(args: Namespace) -> None:
def start_plot_dataframe(args: Dict[str, Any]) -> None:
"""
Entrypoint for dataframe plotting
"""
@ -24,7 +24,7 @@ def start_plot_dataframe(args: Namespace) -> None:
load_and_plot_trades(config)
def start_plot_profit(args: Namespace) -> None:
def start_plot_profit(args: Dict[str, Any]) -> None:
"""
Entrypoint for plot_profit
"""

View File

@ -1,6 +1,5 @@
import logging
import sys
from argparse import Namespace
from pathlib import Path
from typing import Any, Dict, List
@ -16,7 +15,7 @@ from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any]:
def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
"""
Prepare the configuration for utils subcommands
:param args: Cli args from Arguments()
@ -33,34 +32,34 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any
return config
def start_list_exchanges(args: Namespace) -> None:
def start_list_exchanges(args: Dict[str, Any]) -> None:
"""
Print available exchanges
:param args: Cli args from Arguments()
:return: None
"""
if args.print_one_column:
if args['print_one_column']:
print('\n'.join(available_exchanges()))
else:
print(f"Exchanges supported by ccxt and available for Freqtrade: "
f"{', '.join(available_exchanges())}")
def start_create_userdir(args: Namespace) -> None:
def start_create_userdir(args: Dict[str, Any]) -> None:
"""
Create "user_data" directory to contain user data strategies, hyperopts, ...)
:param args: Cli args from Arguments()
:return: None
"""
if "user_data_dir" in args and args.user_data_dir:
create_userdata_dir(args.user_data_dir, create_dir=True)
if "user_data_dir" in args and args["user_data_dir"]:
create_userdata_dir(args["user_data_dir"], create_dir=True)
else:
logger.warning("`create-userdir` requires --userdir to be set.")
sys.exit(1)
def start_download_data(args: Namespace) -> None:
def start_download_data(args: Dict[str, Any]) -> None:
"""
Download data (former download_backtest_data.py script)
"""

View File

@ -4,17 +4,16 @@ Main Freqtrade worker class.
import logging
import time
import traceback
from argparse import Namespace
from typing import Any, Callable, Optional
from typing import Any, Callable, Dict, Optional
import sdnotify
from freqtrade import (constants, OperationalException, TemporaryError,
__version__)
from freqtrade import (OperationalException, TemporaryError, __version__,
constants)
from freqtrade.configuration import Configuration
from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.state import State
from freqtrade.rpc import RPCMessageType
from freqtrade.state import State
logger = logging.getLogger(__name__)
@ -24,7 +23,7 @@ class Worker:
Freqtradebot worker class
"""
def __init__(self, args: Namespace, config=None) -> None:
def __init__(self, args: Dict[str, Any], config=None) -> None:
"""
Init all variables and objects the bot needs to work
"""

View File

@ -12,48 +12,48 @@ def test_parse_args_none() -> None:
arguments = Arguments([])
assert isinstance(arguments, Arguments)
x = arguments.get_parsed_arg()
assert isinstance(x, argparse.Namespace)
assert isinstance(x, dict)
assert isinstance(arguments.parser, argparse.ArgumentParser)
def test_parse_args_defaults() -> None:
args = Arguments([]).get_parsed_arg()
assert args.config == ['config.json']
assert args.strategy_path is None
assert args.datadir is None
assert args.verbosity == 0
assert args["config"] == ['config.json']
assert args["strategy_path"] is None
assert args["datadir"] is None
assert args["verbosity"] == 0
def test_parse_args_config() -> None:
args = Arguments(['-c', '/dev/null']).get_parsed_arg()
assert args.config == ['/dev/null']
assert args["config"] == ['/dev/null']
args = Arguments(['--config', '/dev/null']).get_parsed_arg()
assert args.config == ['/dev/null']
assert args["config"] == ['/dev/null']
args = Arguments(['--config', '/dev/null',
'--config', '/dev/zero'],).get_parsed_arg()
assert args.config == ['/dev/null', '/dev/zero']
assert args["config"] == ['/dev/null', '/dev/zero']
def test_parse_args_db_url() -> None:
args = Arguments(['--db-url', 'sqlite:///test.sqlite']).get_parsed_arg()
assert args.db_url == 'sqlite:///test.sqlite'
assert args["db_url"] == 'sqlite:///test.sqlite'
def test_parse_args_verbose() -> None:
args = Arguments(['-v']).get_parsed_arg()
assert args.verbosity == 1
assert args["verbosity"] == 1
args = Arguments(['--verbose']).get_parsed_arg()
assert args.verbosity == 1
assert args["verbosity"] == 1
def test_common_scripts_options() -> None:
args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC']).get_parsed_arg()
assert args.pairs == ['ETH/BTC', 'XRP/BTC']
assert hasattr(args, "func")
assert args["pairs"] == ['ETH/BTC', 'XRP/BTC']
assert "func" in args
def test_parse_args_version() -> None:
@ -68,7 +68,7 @@ def test_parse_args_invalid() -> None:
def test_parse_args_strategy() -> None:
args = Arguments(['--strategy', 'SomeStrategy']).get_parsed_arg()
assert args.strategy == 'SomeStrategy'
assert args["strategy"] == 'SomeStrategy'
def test_parse_args_strategy_invalid() -> None:
@ -78,7 +78,7 @@ def test_parse_args_strategy_invalid() -> None:
def test_parse_args_strategy_path() -> None:
args = Arguments(['--strategy-path', '/some/path']).get_parsed_arg()
assert args.strategy_path == '/some/path'
assert args["strategy_path"] == '/some/path'
def test_parse_args_strategy_path_invalid() -> None:
@ -105,14 +105,14 @@ def test_parse_args_backtesting_custom() -> None:
'SampleStrategy'
]
call_args = Arguments(args).get_parsed_arg()
assert call_args.config == ['test_conf.json']
assert call_args.verbosity == 0
assert call_args.subparser == 'backtesting'
assert call_args.func is not None
assert call_args.ticker_interval == '1m'
assert call_args.refresh_pairs is True
assert type(call_args.strategy_list) is list
assert len(call_args.strategy_list) == 2
assert call_args["config"] == ['test_conf.json']
assert call_args["verbosity"] == 0
assert call_args["subparser"] == 'backtesting'
assert call_args["func"] is not None
assert call_args["ticker_interval"] == '1m'
assert call_args["refresh_pairs"] is True
assert type(call_args["strategy_list"]) is list
assert len(call_args["strategy_list"]) == 2
def test_parse_args_hyperopt_custom() -> None:
@ -123,12 +123,13 @@ def test_parse_args_hyperopt_custom() -> None:
'--spaces', 'buy'
]
call_args = Arguments(args).get_parsed_arg()
assert call_args.config == ['test_conf.json']
assert call_args.epochs == 20
assert call_args.verbosity == 0
assert call_args.subparser == 'hyperopt'
assert call_args.spaces == ['buy']
assert call_args.func is not None
assert call_args["config"] == ['test_conf.json']
assert call_args["epochs"] == 20
assert call_args["verbosity"] == 0
assert call_args["subparser"] == 'hyperopt'
assert call_args["spaces"] == ['buy']
assert call_args["func"] is not None
assert callable(call_args["func"])
def test_download_data_options() -> None:
@ -139,12 +140,12 @@ def test_download_data_options() -> None:
'--days', '30',
'--exchange', 'binance'
]
args = Arguments(args).get_parsed_arg()
pargs = Arguments(args).get_parsed_arg()
assert args.pairs_file == 'file_with_pairs'
assert args.datadir == 'datadir/directory'
assert args.days == 30
assert args.exchange == 'binance'
assert pargs["pairs_file"] == 'file_with_pairs'
assert pargs["datadir"] == 'datadir/directory'
assert pargs["days"] == 30
assert pargs["exchange"] == 'binance'
def test_plot_dataframe_options() -> None:
@ -158,10 +159,10 @@ def test_plot_dataframe_options() -> None:
]
pargs = Arguments(args).get_parsed_arg()
assert pargs.indicators1 == ["sma10", "sma100"]
assert pargs.indicators2 == ["macd", "fastd", "fastk"]
assert pargs.plot_limit == 30
assert pargs.pairs == ["UNITTEST/BTC"]
assert pargs["indicators1"] == ["sma10", "sma100"]
assert pargs["indicators2"] == ["macd", "fastd", "fastk"]
assert pargs["plot_limit"] == 30
assert pargs["pairs"] == ["UNITTEST/BTC"]
def test_plot_profit_options() -> None:
@ -173,9 +174,9 @@ def test_plot_profit_options() -> None:
]
pargs = Arguments(args).get_parsed_arg()
assert pargs.trade_source == "DB"
assert pargs.pairs == ["UNITTEST/BTC"]
assert pargs.db_url == "sqlite:///whatever.sqlite"
assert pargs["trade_source"] == "DB"
assert pargs["pairs"] == ["UNITTEST/BTC"]
assert pargs["db_url"] == "sqlite:///whatever.sqlite"
def test_check_int_positive() -> None:

View File

@ -871,7 +871,7 @@ def test_pairlist_resolving_fallback(mocker):
args = Arguments(arglist).get_parsed_arg()
# Fix flaky tests if config.json exists
args.config = None
args["config"] = None
configuration = Configuration(args)
config = configuration.get_config()

View File

@ -27,11 +27,12 @@ def test_parse_args_backtesting(mocker) -> None:
main(['backtesting'])
assert backtesting_mock.call_count == 1
call_args = backtesting_mock.call_args[0][0]
assert call_args.config == ['config.json']
assert call_args.verbosity == 0
assert call_args.subparser == 'backtesting'
assert call_args.func is not None
assert call_args.ticker_interval is None
assert call_args["config"] == ['config.json']
assert call_args["verbosity"] == 0
assert call_args["subparser"] == 'backtesting'
assert call_args["func"] is not None
assert callable(call_args["func"])
assert call_args["ticker_interval"] is None
def test_main_start_hyperopt(mocker) -> None:
@ -42,10 +43,11 @@ def test_main_start_hyperopt(mocker) -> None:
main(['hyperopt'])
assert hyperopt_mock.call_count == 1
call_args = hyperopt_mock.call_args[0][0]
assert call_args.config == ['config.json']
assert call_args.verbosity == 0
assert call_args.subparser == 'hyperopt'
assert call_args.func is not None
assert call_args["config"] == ['config.json']
assert call_args["verbosity"] == 0
assert call_args["subparser"] == 'hyperopt'
assert call_args["func"] is not None
assert callable(call_args["func"])
def test_main_fatal_exception(mocker, default_conf, caplog) -> None:

View File

@ -344,7 +344,7 @@ def test_start_plot_profit_error(mocker):
argsp = get_args(args)
# Make sure we use no config. Details: #2241
# not resetting config causes random failures if config.json exists
argsp.config = []
argsp["config"] = []
with pytest.raises(OperationalException):
start_plot_profit(argsp)