Merge pull request #2002 from hroff-1902/refactor/arguments2

minor: refactoring arguments and configuration

freqtrade/arguments.py

@@ -1,7 +1,6 @@
 """
 This module contains the argument manager class
 """
-
 import argparse
 import os
 import re
@@ -29,10 +28,10 @@ class Arg:
         self.kwargs = kwargs
 
 
-# List of available command line arguments
+# List of available command line options
 AVAILABLE_CLI_OPTIONS = {
-    # Common arguments
+    # Common options
-    "loglevel": Arg(
+    "verbosity": Arg(
         '-v', '--verbose',
         help='Verbose mode (-vv for more, -vvv to get all messages).',
         action='count',
@@ -43,11 +42,10 @@ AVAILABLE_CLI_OPTIONS = {
         help='Log to the file specified.',
         metavar='FILE',
     ),
-
     "version": Arg(
         '--version',
         action='version',
-        version=f'%(prog)s {__version__}'
+        version=f'%(prog)s {__version__}',
     ),
     "config": Arg(
         '-c', '--config',
@@ -55,17 +53,19 @@ AVAILABLE_CLI_OPTIONS = {
         f'Multiple --config options may be used. '
         f'Can be set to `-` to read config from stdin.',
         action='append',
-        metavar='PATH',),
+        metavar='PATH',
+    ),
     "datadir": Arg(
         '-d', '--datadir',
         help='Path to backtest data.',
-        metavar='PATH',),
+        metavar='PATH',
+    ),
     # Main options
     "strategy": Arg(
         '-s', '--strategy',
         help='Specify strategy class name (default: `%(default)s`).',
-        default='DefaultStrategy',
         metavar='NAME',
+        default='DefaultStrategy',
     ),
     "strategy_path": Arg(
         '--strategy-path',
@@ -107,6 +107,7 @@ AVAILABLE_CLI_OPTIONS = {
         '--max_open_trades',
         help='Specify max_open_trades to use.',
         type=int,
+        metavar='INT',
     ),
     "stake_amount": Arg(
         '--stake_amount',
@@ -120,19 +121,19 @@ AVAILABLE_CLI_OPTIONS = {
         'up-to-date data.',
         action='store_true',
     ),
-    # backtesting
+    # Backtesting
     "position_stacking": Arg(
         '--eps', '--enable-position-stacking',
         help='Allow buying the same pair multiple times (position stacking).',
         action='store_true',
-        default=False
+        default=False,
     ),
     "use_max_market_positions": Arg(
         '--dmmp', '--disable-max-market-positions',
         help='Disable applying `max_open_trades` during backtest '
         '(same as setting `max_open_trades` to a very high number).',
         action='store_false',
-        default=True
+        default=True,
     ),
     "live": Arg(
         '-l', '--live',
@@ -158,9 +159,9 @@ AVAILABLE_CLI_OPTIONS = {
         help='Save backtest results to the file with this filename (default: `%(default)s`). '
         'Requires `--export` to be set as well. '
         'Example: `--export-filename=user_data/backtest_data/backtest_today.json`',
+        metavar='PATH',
         default=os.path.join('user_data', 'backtest_data',
                              'backtest-result.json'),
-        metavar='PATH',
     ),
     # Edge
     "stoploss_range": Arg(
@@ -169,33 +170,33 @@ AVAILABLE_CLI_OPTIONS = {
         'The format is "min,max,step" (without any space). '
         'Example: `--stoplosses=-0.01,-0.1,-0.001`',
     ),
-    # hyperopt
+    # Hyperopt
     "hyperopt": Arg(
         '--customhyperopt',
         help='Specify hyperopt class name (default: `%(default)s`).',
-        default=constants.DEFAULT_HYPEROPT,
         metavar='NAME',
+        default=constants.DEFAULT_HYPEROPT,
     ),
     "epochs": Arg(
         '-e', '--epochs',
         help='Specify number of epochs (default: %(default)d).',
-        default=constants.HYPEROPT_EPOCH,
+        type=check_int_positive,
-        type=int,
         metavar='INT',
+        default=constants.HYPEROPT_EPOCH,
     ),
     "spaces": Arg(
         '-s', '--spaces',
         help='Specify which parameters to hyperopt. Space-separated list. '
         'Default: `%(default)s`.',
         choices=['all', 'buy', 'sell', 'roi', 'stoploss'],
-        default='all',
         nargs='+',
+        default='all',
     ),
     "print_all": Arg(
         '--print-all',
         help='Print all results, not only the best ones.',
         action='store_true',
-        default=False
+        default=False,
     ),
     "hyperopt_jobs": Arg(
         '-j', '--job-workers',
@@ -203,9 +204,9 @@ AVAILABLE_CLI_OPTIONS = {
         '(hyperopt worker processes). '
         'If -1 (default), all CPUs are used, for -2, all CPUs but one are used, etc. '
         'If 1 is given, no parallel computing code is used at all.',
-        default=-1,
         type=int,
         metavar='JOBS',
+        default=-1,
     ),
     "hyperopt_random_state": Arg(
         '--random-state',
@@ -217,23 +218,22 @@ AVAILABLE_CLI_OPTIONS = {
         '--min-trades',
         help="Set minimal desired number of trades for evaluations in the hyperopt "
         "optimization path (default: 1).",
-        default=1,
         type=check_int_positive,
         metavar='INT',
+        default=1,
     ),
-    # List_exchange
+    # List exchanges
     "print_one_column": Arg(
         '-1', '--one-column',
         help='Print exchanges in one column.',
         action='store_true',
     ),
-    # script_options
+    # Script options
     "pairs": Arg(
         '-p', '--pairs',
         help='Show profits for only these pairs. Pairs are comma-separated.',
     ),
     # Download data
-
     "pairs_file": Arg(
         '--pairs-file',
         help='File containing a list of pairs to download.',
@@ -263,7 +263,7 @@ AVAILABLE_CLI_OPTIONS = {
         help='Clean all existing data for the selected exchange/pairs/timeframes.',
         action='store_true',
     ),
-    # Plot_df_options
+    # Plot dataframe
     "indicators1": Arg(
         '--indicators1',
         help='Set indicators from your strategy you want in the first row of the graph. '
@@ -280,24 +280,27 @@ AVAILABLE_CLI_OPTIONS = {
         '--plot-limit',
         help='Specify tick limit for plotting. Notice: too high values cause huge files. '
         'Default: %(default)s.',
+        type=check_int_positive,
+        metavar='INT',
         default=750,
-        type=int,
     ),
     "trade_source": Arg(
         '--trade-source',
         help='Specify the source for trades (Can be DB or file (backtest file)) '
         'Default: %(default)s',
+        choices=["DB", "file"],
         default="file",
-        choices=["DB", "file"]
+    ),
-    )
 }
 
-ARGS_COMMON = ["loglevel", "logfile", "version", "config", "datadir"]
+
+ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir"]
+
 ARGS_STRATEGY = ["strategy", "strategy_path"]
 
 ARGS_MAIN = ARGS_COMMON + ARGS_STRATEGY + ["dynamic_whitelist", "db_url", "sd_notify"]
 
-ARGS_COMMON_OPTIMIZE = ["loglevel", "ticker_interval", "timerange",
+ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange",
                         "max_open_trades", "stake_amount", "refresh_pairs"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
@@ -309,8 +312,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "position_stacking", "epochs
 
 ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 
-
+ARGS_LIST_EXCHANGES = ["print_one_column"]
-ARGS_LIST_EXCHANGE = ["print_one_column"]
 
 ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
 
@@ -325,9 +327,9 @@ ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY +
 
 class TimeRange(NamedTuple):
     """
-    NamedTuple Defining timerange inputs.
+    NamedTuple defining timerange inputs.
     [start/stop]type defines if [start/stop]ts shall be used.
-    if *type is none, don't use corresponding startvalue.
+    if *type is None, don't use corresponding startvalue.
     """
     starttype: Optional[str] = None
     stoptype: Optional[str] = None
@@ -339,7 +341,6 @@ class Arguments(object):
     """
     Arguments Class. Manage the arguments received by the cli
     """
-
     def __init__(self, args: Optional[List[str]], description: str) -> None:
         self.args = args
         self.parsed_arg: Optional[argparse.Namespace] = None
@@ -412,7 +413,7 @@ class Arguments(object):
             help='Print available exchanges.'
         )
         list_exchanges_cmd.set_defaults(func=start_list_exchanges)
-        self.build_args(optionlist=ARGS_LIST_EXCHANGE, parser=list_exchanges_cmd)
+        self.build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd)
 
     @staticmethod
     def parse_timerange(text: Optional[str]) -> TimeRange:

freqtrade/configuration.py

@@ -6,8 +6,7 @@ import logging
 import os
 import sys
 from argparse import Namespace
-from logging.handlers import RotatingFileHandler
+from typing import Any, Callable, Dict, Optional
-from typing import Any, Callable, Dict, List, Optional
 
 from jsonschema import Draft4Validator, validators
 from jsonschema.exceptions import ValidationError, best_match
@@ -15,25 +14,14 @@ from jsonschema.exceptions import ValidationError, best_match
 from freqtrade import OperationalException, constants
 from freqtrade.exchange import (is_exchange_bad, is_exchange_available,
                                 is_exchange_officially_supported, available_exchanges)
+from freqtrade.loggers import setup_logging
 from freqtrade.misc import deep_merge_dicts
 from freqtrade.state import RunMode
 
+
 logger = logging.getLogger(__name__)
 
 
-def set_loggers(log_level: int = 0) -> None:
-    """
-    Set the logger level for Third party libs
-    :return: None
-    """
-
-    logging.getLogger('requests').setLevel(logging.INFO if log_level <= 1 else logging.DEBUG)
-    logging.getLogger("urllib3").setLevel(logging.INFO if log_level <= 1 else logging.DEBUG)
-    logging.getLogger('ccxt.base.exchange').setLevel(
-        logging.INFO if log_level <= 2 else logging.DEBUG)
-    logging.getLogger('telegram').setLevel(logging.INFO)
-
-
 def _extend_validator(validator_class):
     """
     Extended validator for the Freqtrade configuration JSON Schema.
@@ -135,32 +123,18 @@ class Configuration(object):
     def _load_logging_config(self, config: Dict[str, Any]) -> None:
         """
         Extract information for sys.argv and load logging configuration:
-        the --loglevel, --logfile options
+        the -v/--verbose, --logfile options
         """
         # Log level
-        if 'loglevel' in self.args and self.args.loglevel:
+        if 'verbosity' in self.args and self.args.verbosity:
-            config.update({'verbosity': self.args.loglevel})
+            config.update({'verbosity': self.args.verbosity})
         else:
             config.update({'verbosity': 0})
 
-        # Log to stdout, not stderr
-        log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stdout)]
         if 'logfile' in self.args and self.args.logfile:
             config.update({'logfile': self.args.logfile})
 
-        # Allow setting this as either configuration or argument
+        setup_logging(config)
-        if 'logfile' in config:
-            log_handlers.append(RotatingFileHandler(config['logfile'],
-                                                    maxBytes=1024 * 1024,  # 1Mb
-                                                    backupCount=10))
-
-        logging.basicConfig(
-            level=logging.INFO if config['verbosity'] < 1 else logging.DEBUG,
-            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-            handlers=log_handlers
-        )
-        set_loggers(config['verbosity'])
-        logger.info('Verbosity set to %s', config['verbosity'])
 
     def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         """

freqtrade/loggers.py

@@ -0,0 +1,50 @@
+import logging
+import sys
+
+from logging.handlers import RotatingFileHandler
+from typing import Any, Dict, List
+
+
+logger = logging.getLogger(__name__)
+
+
+def _set_loggers(verbosity: int = 0) -> None:
+    """
+    Set the logging level for third party libraries
+    :return: None
+    """
+
+    logging.getLogger('requests').setLevel(
+            logging.INFO if verbosity <= 1 else logging.DEBUG
+    )
+    logging.getLogger("urllib3").setLevel(
+            logging.INFO if verbosity <= 1 else logging.DEBUG
+    )
+    logging.getLogger('ccxt.base.exchange').setLevel(
+            logging.INFO if verbosity <= 2 else logging.DEBUG
+    )
+    logging.getLogger('telegram').setLevel(logging.INFO)
+
+
+def setup_logging(config: Dict[str, Any]) -> None:
+    """
+    Process -v/--verbose, --logfile options
+    """
+    # Log level
+    verbosity = config['verbosity']
+
+    # Log to stdout, not stderr
+    log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stdout)]
+
+    if config.get('logfile'):
+        log_handlers.append(RotatingFileHandler(config['logfile'],
+                                                maxBytes=1024 * 1024,  # 1Mb
+                                                backupCount=10))
+
+    logging.basicConfig(
+        level=logging.INFO if verbosity < 1 else logging.DEBUG,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        handlers=log_handlers
+    )
+    _set_loggers(verbosity)
+    logger.info('Verbosity set to %s', verbosity)

freqtrade/main.py

@@ -16,7 +16,6 @@ from typing import Any, List
 
 from freqtrade import OperationalException
 from freqtrade.arguments import Arguments
-from freqtrade.configuration import set_loggers
 from freqtrade.worker import Worker
 
 
@@ -32,8 +31,6 @@ def main(sysargv: List[str] = None) -> None:
     return_code: Any = 1
     worker = None
     try:
-        set_loggers()
-
         arguments = Arguments(
             sysargv,
             'Free, open source crypto trading bot'

freqtrade/tests/conftest.py

@@ -227,7 +227,7 @@ def default_conf():
         },
         "initial_state": "running",
         "db_url": "sqlite://",
-        "loglevel": logging.DEBUG,
+        "verbosity": 3,
     }
     return configuration
 

freqtrade/tests/test_arguments.py

@@ -19,7 +19,7 @@ def test_parse_args_defaults() -> None:
     assert args.config == ['config.json']
     assert args.strategy_path is None
     assert args.datadir is None
-    assert args.loglevel == 0
+    assert args.verbosity == 0
 
 
 def test_parse_args_config() -> None:
@@ -42,10 +42,10 @@ def test_parse_args_db_url() -> None:
 
 def test_parse_args_verbose() -> None:
     args = Arguments(['-v'], '').get_parsed_arg()
-    assert args.loglevel == 1
+    assert args.verbosity == 1
 
     args = Arguments(['--verbose'], '').get_parsed_arg()
-    assert args.loglevel == 1
+    assert args.verbosity == 1
 
 
 def test_common_scripts_options() -> None:
@@ -146,7 +146,7 @@ def test_parse_args_backtesting_custom() -> None:
     call_args = Arguments(args, '').get_parsed_arg()
     assert call_args.config == ['test_conf.json']
     assert call_args.live is True
-    assert call_args.loglevel == 0
+    assert call_args.verbosity == 0
     assert call_args.subparser == 'backtesting'
     assert call_args.func is not None
     assert call_args.ticker_interval == '1m'
@@ -165,7 +165,7 @@ def test_parse_args_hyperopt_custom() -> None:
     call_args = Arguments(args, '').get_parsed_arg()
     assert call_args.config == ['test_conf.json']
     assert call_args.epochs == 20
-    assert call_args.loglevel == 0
+    assert call_args.verbosity == 0
     assert call_args.subparser == 'hyperopt'
     assert call_args.spaces == ['buy']
     assert call_args.func is not None

freqtrade/tests/test_configuration.py

@@ -12,8 +12,9 @@ from jsonschema import Draft4Validator, ValidationError, validate
 
 from freqtrade import OperationalException, constants
 from freqtrade.arguments import Arguments
-from freqtrade.configuration import Configuration, set_loggers
+from freqtrade.configuration import Configuration
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
+from freqtrade.loggers import _set_loggers
 from freqtrade.state import RunMode
 from freqtrade.tests.conftest import log_has, log_has_re
 
@@ -524,7 +525,7 @@ def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None:
     mocker.patch('freqtrade.configuration.open', mocker.mock_open(
         read_data=json.dumps(default_conf)))
     # Prevent setting loggers
-    mocker.patch('freqtrade.configuration.set_loggers', MagicMock)
+    mocker.patch('freqtrade.loggers._set_loggers', MagicMock)
     arglist = ['-vvv']
     args = Arguments(arglist, '').get_parsed_arg()
 
@@ -546,7 +547,7 @@ def test_set_loggers() -> None:
     previous_value2 = logging.getLogger('ccxt.base.exchange').level
     previous_value3 = logging.getLogger('telegram').level
 
-    set_loggers()
+    _set_loggers()
 
     value1 = logging.getLogger('requests').level
     assert previous_value1 is not value1
@@ -560,13 +561,13 @@ def test_set_loggers() -> None:
     assert previous_value3 is not value3
     assert value3 is logging.INFO
 
-    set_loggers(log_level=2)
+    _set_loggers(verbosity=2)
 
     assert logging.getLogger('requests').level is logging.DEBUG
     assert logging.getLogger('ccxt.base.exchange').level is logging.INFO
     assert logging.getLogger('telegram').level is logging.INFO
 
-    set_loggers(log_level=3)
+    _set_loggers(verbosity=3)
 
     assert logging.getLogger('requests').level is logging.DEBUG
     assert logging.getLogger('ccxt.base.exchange').level is logging.DEBUG

freqtrade/tests/test_main.py

@@ -27,7 +27,7 @@ def test_parse_args_backtesting(mocker) -> None:
     call_args = backtesting_mock.call_args[0][0]
     assert call_args.config == ['config.json']
     assert call_args.live is False
-    assert call_args.loglevel == 0
+    assert call_args.verbosity == 0
     assert call_args.subparser == 'backtesting'
     assert call_args.func is not None
     assert call_args.ticker_interval is None
@@ -41,7 +41,7 @@ def test_main_start_hyperopt(mocker) -> None:
     assert hyperopt_mock.call_count == 1
     call_args = hyperopt_mock.call_args[0][0]
     assert call_args.config == ['config.json']
-    assert call_args.loglevel == 0
+    assert call_args.verbosity == 0
     assert call_args.subparser == 'hyperopt'
     assert call_args.func is not None