Merge branch 'develop' into stoploss_market

freqtrade/commands/__init__.py

@@ -0,0 +1,25 @@
+# flake8: noqa: F401
+"""
+Commands module.
+Contains all start-commands, subcommands and CLI Interface creation.
+
+Note: Be careful with file-scoped imports in these subfiles.
+    as they are parsed on startup, nothing containing optional modules should be loaded.
+"""
+from freqtrade.commands.arguments import Arguments
+from freqtrade.commands.data_commands import start_download_data
+from freqtrade.commands.deploy_commands import (start_create_userdir,
+                                                start_new_hyperopt,
+                                                start_new_strategy)
+from freqtrade.commands.hyperopt_commands import (start_hyperopt_list,
+                                                  start_hyperopt_show)
+from freqtrade.commands.list_commands import (start_list_exchanges,
+                                              start_list_markets,
+                                              start_list_strategies,
+                                              start_list_timeframes)
+from freqtrade.commands.optimize_commands import (start_backtesting,
+                                                  start_edge, start_hyperopt)
+from freqtrade.commands.pairlist_commands import start_test_pairlist
+from freqtrade.commands.plot_commands import (start_plot_dataframe,
+                                              start_plot_profit)
+from freqtrade.commands.trade_commands import start_trading

freqtrade/commands/arguments.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from freqtrade import constants
-from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS
+from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 
 ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
 
@@ -130,14 +130,14 @@ class Arguments:
         self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
         self._build_args(optionlist=['version'], parser=self.parser)
 
-        from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge
+        from freqtrade.commands import (start_create_userdir, start_download_data,
-        from freqtrade.utils import (start_create_userdir, start_download_data,
+                                        start_hyperopt_list, start_hyperopt_show,
-                                     start_hyperopt_list, start_hyperopt_show,
+                                        start_list_exchanges, start_list_markets,
-                                     start_list_exchanges, start_list_markets,
+                                        start_list_strategies, start_new_hyperopt,
-                                     start_list_strategies, start_new_hyperopt,
+                                        start_new_strategy, start_list_timeframes,
-                                     start_new_strategy, start_list_timeframes,
+                                        start_plot_dataframe, start_plot_profit,
-                                     start_test_pairlist, start_trading)
+                                        start_backtesting, start_hyperopt, start_edge,
-        from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit
+                                        start_test_pairlist, start_trading)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added

freqtrade/commands/cli_options.py

@@ -1,7 +1,7 @@
 """
 Definition of cli arguments used in arguments.py
 """
-import argparse
+from argparse import ArgumentTypeError
 
 from freqtrade import __version__, constants
 
@@ -12,7 +12,7 @@ def check_int_positive(value: str) -> int:
         if uint <= 0:
             raise ValueError
     except ValueError:
-        raise argparse.ArgumentTypeError(
+        raise ArgumentTypeError(
             f"{value} is invalid for this parameter, should be a positive integer value"
         )
     return uint
@@ -24,7 +24,7 @@ def check_int_nonzero(value: str) -> int:
         if uint == 0:
             raise ValueError
     except ValueError:
-        raise argparse.ArgumentTypeError(
+        raise ArgumentTypeError(
             f"{value} is invalid for this parameter, should be a non-zero integer value"
         )
     return uint

freqtrade/commands/data_commands.py

@@ -0,0 +1,63 @@
+import logging
+import sys
+from typing import Any, Dict, List
+
+import arrow
+
+from freqtrade.configuration import TimeRange, setup_utils_configuration
+from freqtrade.data.history import (convert_trades_to_ohlcv,
+                                    refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data)
+from freqtrade.exceptions import OperationalException
+from freqtrade.resolvers import ExchangeResolver
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_download_data(args: Dict[str, Any]) -> None:
+    """
+    Download data (former download_backtest_data.py script)
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+    logger.info(f'About to download pairs: {config["pairs"]}, '
+                f'intervals: {config["timeframes"]} to {config["datadir"]}')
+
+    pairs_not_available: List[str] = []
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
+    try:
+
+        if config.get('download_trades'):
+            pairs_not_available = refresh_backtest_trades_data(
+                exchange, pairs=config["pairs"], datadir=config['datadir'],
+                timerange=timerange, erase=config.get("erase"))
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
+        else:
+            pairs_not_available = refresh_backtest_ohlcv_data(
+                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
+
+    except KeyboardInterrupt:
+        sys.exit("SIGINT received, aborting ...")
+
+    finally:
+        if pairs_not_available:
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {exchange.name}.")

freqtrade/commands/deploy_commands.py

@@ -0,0 +1,112 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.configuration.directory_operations import (copy_sample_files,
+                                                          create_userdata_dir)
+from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
+from freqtrade.exceptions import OperationalException
+from freqtrade.misc import render_template
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_create_userdir(args: Dict[str, Any]) -> None:
+    """
+    Create "user_data" directory to contain user data strategies, hyperopt, ...)
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    if "user_data_dir" in args and args["user_data_dir"]:
+        userdir = create_userdata_dir(args["user_data_dir"], create_dir=True)
+        copy_sample_files(userdir, overwrite=args["reset"])
+    else:
+        logger.warning("`create-userdir` requires --userdir to be set.")
+        sys.exit(1)
+
+
+def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str):
+    """
+    Deploy new strategy from template to strategy_path
+    """
+    indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",)
+    buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",)
+    sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",)
+    plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",)
+
+    strategy_text = render_template(templatefile='base_strategy.py.j2',
+                                    arguments={"strategy": strategy_name,
+                                               "indicators": indicators,
+                                               "buy_trend": buy_trend,
+                                               "sell_trend": sell_trend,
+                                               "plot_config": plot_config,
+                                               })
+
+    logger.info(f"Writing strategy to `{strategy_path}`.")
+    strategy_path.write_text(strategy_text)
+
+
+def start_new_strategy(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    if "strategy" in args and args["strategy"]:
+        if args["strategy"] == "DefaultStrategy":
+            raise OperationalException("DefaultStrategy is not allowed as name.")
+
+        new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py")
+
+        if new_path.exists():
+            raise OperationalException(f"`{new_path}` already exists. "
+                                       "Please choose another Strategy Name.")
+
+        deploy_new_strategy(args['strategy'], new_path, args['template'])
+
+    else:
+        raise OperationalException("`new-strategy` requires --strategy to be set.")
+
+
+def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str):
+    """
+    Deploys a new hyperopt template to hyperopt_path
+    """
+    buy_guards = render_template(
+        templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",)
+    sell_guards = render_template(
+        templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",)
+    buy_space = render_template(
+        templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",)
+    sell_space = render_template(
+        templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",)
+
+    strategy_text = render_template(templatefile='base_hyperopt.py.j2',
+                                    arguments={"hyperopt": hyperopt_name,
+                                               "buy_guards": buy_guards,
+                                               "sell_guards": sell_guards,
+                                               "buy_space": buy_space,
+                                               "sell_space": sell_space,
+                                               })
+
+    logger.info(f"Writing hyperopt to `{hyperopt_path}`.")
+    hyperopt_path.write_text(strategy_text)
+
+
+def start_new_hyperopt(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    if "hyperopt" in args and args["hyperopt"]:
+        if args["hyperopt"] == "DefaultHyperopt":
+            raise OperationalException("DefaultHyperopt is not allowed as name.")
+
+        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py")
+
+        if new_path.exists():
+            raise OperationalException(f"`{new_path}` already exists. "
+                                       "Please choose another Strategy Name.")
+        deploy_new_hyperopt(args['hyperopt'], new_path, args['template'])
+    else:
+        raise OperationalException("`new-hyperopt` requires --hyperopt to be set.")

freqtrade/commands/hyperopt_commands.py

@@ -0,0 +1,114 @@
+import logging
+from operator import itemgetter
+from typing import Any, Dict, List
+
+from colorama import init as colorama_init
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.exceptions import OperationalException
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_hyperopt_list(args: Dict[str, Any]) -> None:
+    """
+    List hyperopt epochs previously evaluated
+    """
+    from freqtrade.optimize.hyperopt import Hyperopt
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    only_best = config.get('hyperopt_list_best', False)
+    only_profitable = config.get('hyperopt_list_profitable', False)
+    print_colorized = config.get('print_colorized', False)
+    print_json = config.get('print_json', False)
+    no_details = config.get('hyperopt_list_no_details', False)
+    no_header = False
+
+    trials_file = (config['user_data_dir'] /
+                   'hyperopt_results' / 'hyperopt_results.pickle')
+
+    # Previous evaluations
+    trials = Hyperopt.load_previous_results(trials_file)
+    total_epochs = len(trials)
+
+    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
+
+    # TODO: fetch the interval for epochs to print from the cli option
+    epoch_start, epoch_stop = 0, None
+
+    if print_colorized:
+        colorama_init(autoreset=True)
+
+    try:
+        # Human-friendly indexes used here (starting from 1)
+        for val in trials[epoch_start:epoch_stop]:
+            Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized)
+
+    except KeyboardInterrupt:
+        print('User interrupted..')
+
+    if trials and not no_details:
+        sorted_trials = sorted(trials, key=itemgetter('loss'))
+        results = sorted_trials[0]
+        Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
+
+
+def start_hyperopt_show(args: Dict[str, Any]) -> None:
+    """
+    Show details of a hyperopt epoch previously evaluated
+    """
+    from freqtrade.optimize.hyperopt import Hyperopt
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    only_best = config.get('hyperopt_list_best', False)
+    only_profitable = config.get('hyperopt_list_profitable', False)
+    no_header = config.get('hyperopt_show_no_header', False)
+
+    trials_file = (config['user_data_dir'] /
+                   'hyperopt_results' / 'hyperopt_results.pickle')
+
+    # Previous evaluations
+    trials = Hyperopt.load_previous_results(trials_file)
+    total_epochs = len(trials)
+
+    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
+    trials_epochs = len(trials)
+
+    n = config.get('hyperopt_show_index', -1)
+    if n > trials_epochs:
+        raise OperationalException(
+                f"The index of the epoch to show should be less than {trials_epochs + 1}.")
+    if n < -trials_epochs:
+        raise OperationalException(
+                f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
+
+    # Translate epoch index from human-readable format to pythonic
+    if n > 0:
+        n -= 1
+
+    print_json = config.get('print_json', False)
+
+    if trials:
+        val = trials[n]
+        Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
+                                     header_str="Epoch details")
+
+
+def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List:
+    """
+    Filter our items from the list of hyperopt results
+    """
+    if only_best:
+        trials = [x for x in trials if x['is_best']]
+    if only_profitable:
+        trials = [x for x in trials if x['results_metrics']['profit'] > 0]
+
+    logger.info(f"{len(trials)} " +
+                ("best " if only_best else "") +
+                ("profitable " if only_profitable else "") +
+                "epochs found.")
+
+    return trials

freqtrade/commands/list_commands.py

@@ -0,0 +1,156 @@
+import csv
+import logging
+import sys
+from collections import OrderedDict
+from pathlib import Path
+from typing import Any, Dict
+
+import rapidjson
+from tabulate import tabulate
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.constants import USERPATH_STRATEGY
+from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
+                                market_is_active, symbol_is_pair)
+from freqtrade.misc import plural
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_list_exchanges(args: Dict[str, Any]) -> None:
+    """
+    Print available exchanges
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
+    if args['print_one_column']:
+        print('\n'.join(exchanges))
+    else:
+        if args['list_exchanges_all']:
+            print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
+        else:
+            print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
+
+
+def start_list_strategies(args: Dict[str, Any]) -> None:
+    """
+    Print Strategies available in a directory
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY))
+    strategies = StrategyResolver.search_all_objects(directory)
+    # Sort alphabetically
+    strategies = sorted(strategies, key=lambda x: x['name'])
+    strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies]
+
+    if args['print_one_column']:
+        print('\n'.join([s['name'] for s in strategies]))
+    else:
+        print(tabulate(strats_to_print, headers='keys', tablefmt='pipe'))
+
+
+def start_list_timeframes(args: Dict[str, Any]) -> None:
+    """
+    Print ticker intervals (timeframes) available on Exchange
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+    # Do not use ticker_interval set in the config
+    config['ticker_interval'] = None
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    if args['print_one_column']:
+        print('\n'.join(exchange.timeframes))
+    else:
+        print(f"Timeframes available for the exchange `{exchange.name}`: "
+              f"{', '.join(exchange.timeframes)}")
+
+
+def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
+    """
+    Print pairs/markets on the exchange
+    :param args: Cli args from Arguments()
+    :param pairs_only: if True print only pairs, otherwise print all instruments (markets)
+    :return: None
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    # By default only active pairs/markets are to be shown
+    active_only = not args.get('list_pairs_all', False)
+
+    base_currencies = args.get('base_currencies', [])
+    quote_currencies = args.get('quote_currencies', [])
+
+    try:
+        pairs = exchange.get_markets(base_currencies=base_currencies,
+                                     quote_currencies=quote_currencies,
+                                     pairs_only=pairs_only,
+                                     active_only=active_only)
+        # Sort the pairs/markets by symbol
+        pairs = OrderedDict(sorted(pairs.items()))
+    except Exception as e:
+        raise OperationalException(f"Cannot get markets. Reason: {e}") from e
+
+    else:
+        summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") +
+                       ("active " if active_only else "") +
+                       (plural(len(pairs), "pair" if pairs_only else "market")) +
+                       (f" with {', '.join(base_currencies)} as base "
+                        f"{plural(len(base_currencies), 'currency', 'currencies')}"
+                        if base_currencies else "") +
+                       (" and" if base_currencies and quote_currencies else "") +
+                       (f" with {', '.join(quote_currencies)} as quote "
+                        f"{plural(len(quote_currencies), 'currency', 'currencies')}"
+                        if quote_currencies else ""))
+
+        headers = ["Id", "Symbol", "Base", "Quote", "Active",
+                   *(['Is pair'] if not pairs_only else [])]
+
+        tabular_data = []
+        for _, v in pairs.items():
+            tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
+                                 'Base': v['base'], 'Quote': v['quote'],
+                                 'Active': market_is_active(v),
+                                 **({'Is pair': symbol_is_pair(v['symbol'])}
+                                    if not pairs_only else {})})
+
+        if (args.get('print_one_column', False) or
+                args.get('list_pairs_print_json', False) or
+                args.get('print_csv', False)):
+            # Print summary string in the log in case of machine-readable
+            # regular formats.
+            logger.info(f"{summary_str}.")
+        else:
+            # Print empty string separating leading logs and output in case of
+            # human-readable formats.
+            print()
+
+        if len(pairs):
+            if args.get('print_list', False):
+                # print data as a list, with human-readable summary
+                print(f"{summary_str}: {', '.join(pairs.keys())}.")
+            elif args.get('print_one_column', False):
+                print('\n'.join(pairs.keys()))
+            elif args.get('list_pairs_print_json', False):
+                print(rapidjson.dumps(list(pairs.keys()), default=str))
+            elif args.get('print_csv', False):
+                writer = csv.DictWriter(sys.stdout, fieldnames=headers)
+                writer.writeheader()
+                writer.writerows(tabular_data)
+            else:
+                # print data as a table, with the human-readable summary
+                print(f"{summary_str}:")
+                print(tabulate(tabular_data, headers='keys', tablefmt='pipe'))
+        elif not (args.get('print_one_column', False) or
+                  args.get('list_pairs_print_json', False) or
+                  args.get('print_csv', False)):
+            print(f"{summary_str}.")

freqtrade/commands/optimize_commands.py

@@ -0,0 +1,102 @@
+import logging
+from typing import Any, Dict
+
+from freqtrade import constants
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.exceptions import DependencyException, OperationalException
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
+    """
+    Prepare the configuration for the Hyperopt module
+    :param args: Cli args from Arguments()
+    :return: Configuration
+    """
+    config = setup_utils_configuration(args, method)
+
+    if method == RunMode.BACKTEST:
+        if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
+            raise DependencyException('stake amount could not be "%s" for backtesting' %
+                                      constants.UNLIMITED_STAKE_AMOUNT)
+
+    return config
+
+
+def start_backtesting(args: Dict[str, Any]) -> None:
+    """
+    Start Backtesting script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    # Import here to avoid loading backtesting module when it's not used
+    from freqtrade.optimize.backtesting import Backtesting
+
+    # Initialize configuration
+    config = setup_optimize_configuration(args, RunMode.BACKTEST)
+
+    logger.info('Starting freqtrade in Backtesting mode')
+
+    # Initialize backtesting object
+    backtesting = Backtesting(config)
+    backtesting.start()
+
+
+def start_hyperopt(args: Dict[str, Any]) -> None:
+    """
+    Start hyperopt script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    # Import here to avoid loading hyperopt module when it's not used
+    try:
+        from filelock import FileLock, Timeout
+        from freqtrade.optimize.hyperopt import Hyperopt
+    except ImportError as e:
+        raise OperationalException(
+            f"{e}. Please ensure that the hyperopt dependencies are installed.") from e
+    # Initialize configuration
+    config = setup_optimize_configuration(args, RunMode.HYPEROPT)
+
+    logger.info('Starting freqtrade in Hyperopt mode')
+
+    lock = FileLock(Hyperopt.get_lock_filename(config))
+
+    try:
+        with lock.acquire(timeout=1):
+
+            # Remove noisy log messages
+            logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
+            logging.getLogger('filelock').setLevel(logging.WARNING)
+
+            # Initialize backtesting object
+            hyperopt = Hyperopt(config)
+            hyperopt.start()
+
+    except Timeout:
+        logger.info("Another running instance of freqtrade Hyperopt detected.")
+        logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. "
+                    "Hyperopt module is resource hungry. Please run your Hyperopt sequentially "
+                    "or on separate machines.")
+        logger.info("Quitting now.")
+        # TODO: return False here in order to help freqtrade to exit
+        # with non-zero exit code...
+        # Same in Edge and Backtesting start() functions.
+
+
+def start_edge(args: Dict[str, Any]) -> None:
+    """
+    Start Edge script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    from freqtrade.optimize.edge_cli import EdgeCli
+    # Initialize configuration
+    config = setup_optimize_configuration(args, RunMode.EDGE)
+    logger.info('Starting freqtrade in Edge mode')
+
+    # Initialize Edge object
+    edge_cli = EdgeCli(config)
+    edge_cli.start()

freqtrade/commands/pairlist_commands.py

@@ -0,0 +1,42 @@
+import logging
+from typing import Any, Dict
+
+import rapidjson
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.resolvers import ExchangeResolver
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_test_pairlist(args: Dict[str, Any]) -> None:
+    """
+    Test Pairlist configuration
+    """
+    from freqtrade.pairlist.pairlistmanager import PairListManager
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    quote_currencies = args.get('quote_currencies')
+    if not quote_currencies:
+        quote_currencies = [config.get('stake_currency')]
+    results = {}
+    for curr in quote_currencies:
+        config['stake_currency'] = curr
+        # Do not use ticker_interval set in the config
+        pairlists = PairListManager(exchange, config)
+        pairlists.refresh_pairlist()
+        results[curr] = pairlists.whitelist
+
+    for curr, pairlist in results.items():
+        if not args.get('print_one_column', False):
+            print(f"Pairs for {curr}: ")
+
+        if args.get('print_one_column', False):
+            print('\n'.join(pairlist))
+        elif args.get('list_pairs_print_json', False):
+            print(rapidjson.dumps(list(pairlist), default=str))
+        else:
+            print(pairlist)

freqtrade/commands/plot_commands.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict
 
+from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
-from freqtrade.utils import setup_utils_configuration
 
 
 def validate_plot_args(args: Dict[str, Any]):

freqtrade/commands/trade_commands.py

@@ -0,0 +1,25 @@
+import logging
+
+from typing import Any, Dict
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_trading(args: Dict[str, Any]) -> int:
+    """
+    Main entry point for trading mode
+    """
+    from freqtrade.worker import Worker
+    # Load and run worker
+    worker = None
+    try:
+        worker = Worker(args)
+        worker.run()
+    except KeyboardInterrupt:
+        logger.info('SIGINT received, aborting ...')
+    finally:
+        if worker:
+            logger.info("worker found ... calling exit")
+            worker.exit()
+    return 0

freqtrade/configuration/__init__.py

@@ -1,5 +1,7 @@
-from freqtrade.configuration.arguments import Arguments  # noqa: F401
+# flake8: noqa: F401
-from freqtrade.configuration.check_exchange import check_exchange, remove_credentials  # noqa: F401
+
-from freqtrade.configuration.timerange import TimeRange  # noqa: F401
+from freqtrade.configuration.config_setup import setup_utils_configuration
-from freqtrade.configuration.configuration import Configuration  # noqa: F401
+from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
-from freqtrade.configuration.config_validation import validate_config_consistency  # noqa: F401
+from freqtrade.configuration.timerange import TimeRange
+from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.config_validation import validate_config_consistency

freqtrade/configuration/config_setup.py

@@ -0,0 +1,25 @@
+import logging
+from typing import Any, Dict
+
+from .config_validation import validate_config_consistency
+from .configuration import Configuration
+from .check_exchange import remove_credentials
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
+    """
+    Prepare the configuration for utils subcommands
+    :param args: Cli args from Arguments()
+    :return: Configuration
+    """
+    configuration = Configuration(args, method)
+    config = configuration.get_config()
+
+    # Ensure we do not use Exchange credentials
+    remove_credentials(config)
+    validate_config_consistency(config)
+
+    return config

freqtrade/edge/__init__.py

@@ -1,464 +1 @@
-# pragma pylint: disable=W0603
+from .edge_positioning import Edge, PairInfo  # noqa: F401
-""" Edge positioning package """
-import logging
-from typing import Any, Dict, NamedTuple
-
-import arrow
-import numpy as np
-import utils_find_1st as utf1st
-from pandas import DataFrame
-
-from freqtrade import constants
-from freqtrade.configuration import TimeRange
-from freqtrade.data import history
-from freqtrade.exceptions import OperationalException
-from freqtrade.strategy.interface import SellType
-
-logger = logging.getLogger(__name__)
-
-
-class PairInfo(NamedTuple):
-    stoploss: float
-    winrate: float
-    risk_reward_ratio: float
-    required_risk_reward: float
-    expectancy: float
-    nb_trades: int
-    avg_trade_duration: float
-
-
-class Edge:
-    """
-    Calculates Win Rate, Risk Reward Ratio, Expectancy
-    against historical data for a give set of markets and a strategy
-    it then adjusts stoploss and position size accordingly
-    and force it into the strategy
-    Author: https://github.com/mishaker
-    """
-
-    config: Dict = {}
-    _cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
-
-    def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:
-
-        self.config = config
-        self.exchange = exchange
-        self.strategy = strategy
-
-        self.edge_config = self.config.get('edge', {})
-        self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
-        self._final_pairs: list = []
-
-        # checking max_open_trades. it should be -1 as with Edge
-        # the number of trades is determined by position size
-        if self.config['max_open_trades'] != float('inf'):
-            logger.critical('max_open_trades should be -1 in config !')
-
-        if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT:
-            raise OperationalException('Edge works only with unlimited stake amount')
-
-        # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future.
-        self._capital_percentage: float = self.edge_config.get(
-            'capital_available_percentage', self.config['tradable_balance_ratio'])
-        self._allowed_risk: float = self.edge_config.get('allowed_risk')
-        self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14)
-        self._last_updated: int = 0  # Timestamp of pairs last updated time
-        self._refresh_pairs = True
-
-        self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01))
-        self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05))
-        self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001))
-
-        # calculating stoploss range
-        self._stoploss_range = np.arange(
-            self._stoploss_range_min,
-            self._stoploss_range_max,
-            self._stoploss_range_step
-        )
-
-        self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
-            days=-1 * self._since_number_of_days).format('YYYYMMDD'))
-        if config.get('fee'):
-            self.fee = config['fee']
-        else:
-            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
-
-    def calculate(self) -> bool:
-        pairs = self.config['exchange']['pair_whitelist']
-        heartbeat = self.edge_config.get('process_throttle_secs')
-
-        if (self._last_updated > 0) and (
-                self._last_updated + heartbeat > arrow.utcnow().timestamp):
-            return False
-
-        data: Dict[str, Any] = {}
-        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
-        logger.info('Using local backtesting data (using whitelist in given config) ...')
-
-        if self._refresh_pairs:
-            history.refresh_data(
-                datadir=self.config['datadir'],
-                pairs=pairs,
-                exchange=self.exchange,
-                timeframe=self.strategy.ticker_interval,
-                timerange=self._timerange,
-            )
-
-        data = history.load_data(
-            datadir=self.config['datadir'],
-            pairs=pairs,
-            timeframe=self.strategy.ticker_interval,
-            timerange=self._timerange,
-            startup_candles=self.strategy.startup_candle_count,
-        )
-
-        if not data:
-            # Reinitializing cached pairs
-            self._cached_pairs = {}
-            logger.critical("No data found. Edge is stopped ...")
-            return False
-
-        preprocessed = self.strategy.tickerdata_to_dataframe(data)
-
-        # Print timeframe
-        min_date, max_date = history.get_timerange(preprocessed)
-        logger.info(
-            'Measuring data from %s up to %s (%s days) ...',
-            min_date.isoformat(),
-            max_date.isoformat(),
-            (max_date - min_date).days
-        )
-        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
-
-        trades: list = []
-        for pair, pair_data in preprocessed.items():
-            # Sorting dataframe by date and reset index
-            pair_data = pair_data.sort_values(by=['date'])
-            pair_data = pair_data.reset_index(drop=True)
-
-            ticker_data = self.strategy.advise_sell(
-                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
-
-            trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range)
-
-        # If no trade found then exit
-        if len(trades) == 0:
-            logger.info("No trades found.")
-            return False
-
-        # Fill missing, calculable columns, profit, duration , abs etc.
-        trades_df = self._fill_calculable_fields(DataFrame(trades))
-        self._cached_pairs = self._process_expectancy(trades_df)
-        self._last_updated = arrow.utcnow().timestamp
-
-        return True
-
-    def stake_amount(self, pair: str, free_capital: float,
-                     total_capital: float, capital_in_trade: float) -> float:
-        stoploss = self.stoploss(pair)
-        available_capital = (total_capital + capital_in_trade) * self._capital_percentage
-        allowed_capital_at_risk = available_capital * self._allowed_risk
-        max_position_size = abs(allowed_capital_at_risk / stoploss)
-        position_size = min(max_position_size, free_capital)
-        if pair in self._cached_pairs:
-            logger.info(
-                'winrate: %s, expectancy: %s, position size: %s, pair: %s,'
-                ' capital in trade: %s, free capital: %s, total capital: %s,'
-                ' stoploss: %s, available capital: %s.',
-                self._cached_pairs[pair].winrate,
-                self._cached_pairs[pair].expectancy,
-                position_size, pair,
-                capital_in_trade, free_capital, total_capital,
-                stoploss, available_capital
-            )
-        return round(position_size, 15)
-
-    def stoploss(self, pair: str) -> float:
-        if pair in self._cached_pairs:
-            return self._cached_pairs[pair].stoploss
-        else:
-            logger.warning('tried to access stoploss of a non-existing pair, '
-                           'strategy stoploss is returned instead.')
-            return self.strategy.stoploss
-
-    def adjust(self, pairs) -> list:
-        """
-        Filters out and sorts "pairs" according to Edge calculated pairs
-        """
-        final = []
-        for pair, info in self._cached_pairs.items():
-            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
-                info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
-                    pair in pairs:
-                final.append(pair)
-
-        if self._final_pairs != final:
-            self._final_pairs = final
-            if self._final_pairs:
-                logger.info(
-                    'Minimum expectancy and minimum winrate are met only for %s,'
-                    ' so other pairs are filtered out.',
-                    self._final_pairs
-                    )
-            else:
-                logger.info(
-                    'Edge removed all pairs as no pair with minimum expectancy '
-                    'and minimum winrate was found !'
-                    )
-
-        return self._final_pairs
-
-    def accepted_pairs(self) -> list:
-        """
-        return a list of accepted pairs along with their winrate, expectancy and stoploss
-        """
-        final = []
-        for pair, info in self._cached_pairs.items():
-            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
-                 info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
-                final.append({
-                    'Pair': pair,
-                    'Winrate': info.winrate,
-                    'Expectancy': info.expectancy,
-                    'Stoploss': info.stoploss,
-                })
-        return final
-
-    def _fill_calculable_fields(self, result: DataFrame) -> DataFrame:
-        """
-        The result frame contains a number of columns that are calculable
-        from other columns. These are left blank till all rows are added,
-        to be populated in single vector calls.
-
-        Columns to be populated are:
-        - Profit
-        - trade duration
-        - profit abs
-        :param result Dataframe
-        :return: result Dataframe
-        """
-
-        # stake and fees
-        # stake = 0.015
-        # 0.05% is 0.0005
-        # fee = 0.001
-
-        # we set stake amount to an arbitrary amount.
-        # as it doesn't change the calculation.
-        # all returned values are relative. they are percentages.
-        stake = 0.015
-        fee = self.fee
-        open_fee = fee / 2
-        close_fee = fee / 2
-
-        result['trade_duration'] = result['close_time'] - result['open_time']
-
-        result['trade_duration'] = result['trade_duration'].map(
-            lambda x: int(x.total_seconds() / 60))
-
-        # Spends, Takes, Profit, Absolute Profit
-
-        # Buy Price
-        result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
-        result['buy_fee'] = stake * open_fee
-        result['buy_spend'] = stake + result['buy_fee']  # How much we're spending
-
-        # Sell price
-        result['sell_sum'] = result['buy_vol'] * result['close_rate']
-        result['sell_fee'] = result['sell_sum'] * close_fee
-        result['sell_take'] = result['sell_sum'] - result['sell_fee']
-
-        # profit_percent
-        result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']
-
-        # Absolute profit
-        result['profit_abs'] = result['sell_take'] - result['buy_spend']
-
-        return result
-
-    def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
-        """
-        This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
-        The calulation will be done per pair and per strategy.
-        """
-        # Removing pairs having less than min_trades_number
-        min_trades_number = self.edge_config.get('min_trade_number', 10)
-        results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number)
-        ###################################
-
-        # Removing outliers (Only Pumps) from the dataset
-        # The method to detect outliers is to calculate standard deviation
-        # Then every value more than (standard deviation + 2*average) is out (pump)
-        #
-        # Removing Pumps
-        if self.edge_config.get('remove_pumps', False):
-            results = results.groupby(['pair', 'stoploss']).apply(
-                lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()])
-        ##########################################################################
-
-        # Removing trades having a duration more than X minutes (set in config)
-        max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440)
-        results = results[results.trade_duration < max_trade_duration]
-        #######################################################################
-
-        if results.empty:
-            return {}
-
-        groupby_aggregator = {
-            'profit_abs': [
-                ('nb_trades', 'count'),  # number of all trades
-                ('profit_sum', lambda x: x[x > 0].sum()),  # cumulative profit of all winning trades
-                ('loss_sum', lambda x: abs(x[x < 0].sum())),  # cumulative loss of all losing trades
-                ('nb_win_trades', lambda x: x[x > 0].count())  # number of winning trades
-            ],
-            'trade_duration': [('avg_trade_duration', 'mean')]
-        }
-
-        # Group by (pair and stoploss) by applying above aggregator
-        df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg(
-            groupby_aggregator).reset_index(col_level=1)
-
-        # Dropping level 0 as we don't need it
-        df.columns = df.columns.droplevel(0)
-
-        # Calculating number of losing trades, average win and average loss
-        df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']
-        df['average_win'] = df['profit_sum'] / df['nb_win_trades']
-        df['average_loss'] = df['loss_sum'] / df['nb_loss_trades']
-
-        # Win rate = number of profitable trades / number of trades
-        df['winrate'] = df['nb_win_trades'] / df['nb_trades']
-
-        # risk_reward_ratio = average win / average loss
-        df['risk_reward_ratio'] = df['average_win'] / df['average_loss']
-
-        # required_risk_reward = (1 / winrate) - 1
-        df['required_risk_reward'] = (1 / df['winrate']) - 1
-
-        # expectancy = (risk_reward_ratio * winrate) - (lossrate)
-        df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate'])
-
-        # sort by expectancy and stoploss
-        df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby(
-            'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index()
-
-        final = {}
-        for x in df.itertuples():
-            final[x.pair] = PairInfo(
-                x.stoploss,
-                x.winrate,
-                x.risk_reward_ratio,
-                x.required_risk_reward,
-                x.expectancy,
-                x.nb_trades,
-                x.avg_trade_duration
-            )
-
-        # Returning a list of pairs in order of "expectancy"
-        return final
-
-    def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range):
-        buy_column = ticker_data['buy'].values
-        sell_column = ticker_data['sell'].values
-        date_column = ticker_data['date'].values
-        ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values
-
-        result: list = []
-        for stoploss in stoploss_range:
-            result += self._detect_next_stop_or_sell_point(
-                buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair
-            )
-
-        return result
-
-    def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,
-                                        ohlc_columns, stoploss, pair):
-        """
-        Iterate through ohlc_columns in order to find the next trade
-        Next trade opens from the first buy signal noticed to
-        The sell or stoploss signal after it.
-        It then cuts OHLC, buy_column, sell_column and date_column.
-        Cut from (the exit trade index) + 1.
-
-        Author: https://github.com/mishaker
-        """
-
-        result: list = []
-        start_point = 0
-
-        while True:
-            open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal)
-
-            # Return empty if we don't find trade entry (i.e. buy==1) or
-            # we find a buy but at the end of array
-            if open_trade_index == -1 or open_trade_index == len(buy_column) - 1:
-                break
-            else:
-                # When a buy signal is seen,
-                # trade opens in reality on the next candle
-                open_trade_index += 1
-
-            stop_price_percentage = stoploss + 1
-            open_price = ohlc_columns[open_trade_index, 0]
-            stop_price = (open_price * stop_price_percentage)
-
-            # Searching for the index where stoploss is hit
-            stop_index = utf1st.find_1st(
-                ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller)
-
-            # If we don't find it then we assume stop_index will be far in future (infinite number)
-            if stop_index == -1:
-                stop_index = float('inf')
-
-            # Searching for the index where sell is hit
-            sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal)
-
-            # If we don't find it then we assume sell_index will be far in future (infinite number)
-            if sell_index == -1:
-                sell_index = float('inf')
-
-            # Check if we don't find any stop or sell point (in that case trade remains open)
-            # It is not interesting for Edge to consider it so we simply ignore the trade
-            # And stop iterating there is no more entry
-            if stop_index == sell_index == float('inf'):
-                break
-
-            if stop_index <= sell_index:
-                exit_index = open_trade_index + stop_index
-                exit_type = SellType.STOP_LOSS
-                exit_price = stop_price
-            elif stop_index > sell_index:
-                # If exit is SELL then we exit at the next candle
-                exit_index = open_trade_index + sell_index + 1
-
-                # Check if we have the next candle
-                if len(ohlc_columns) - 1 < exit_index:
-                    break
-
-                exit_type = SellType.SELL_SIGNAL
-                exit_price = ohlc_columns[exit_index, 0]
-
-            trade = {'pair': pair,
-                     'stoploss': stoploss,
-                     'profit_percent': '',
-                     'profit_abs': '',
-                     'open_time': date_column[open_trade_index],
-                     'close_time': date_column[exit_index],
-                     'open_index': start_point + open_trade_index,
-                     'close_index': start_point + exit_index,
-                     'trade_duration': '',
-                     'open_rate': round(open_price, 15),
-                     'close_rate': round(exit_price, 15),
-                     'exit_type': exit_type
-                     }
-
-            result.append(trade)
-
-            # Giving a view of exit_index till the end of array
-            buy_column = buy_column[exit_index:]
-            sell_column = sell_column[exit_index:]
-            date_column = date_column[exit_index:]
-            ohlc_columns = ohlc_columns[exit_index:]
-            start_point += exit_index
-
-        return result

freqtrade/edge/edge_positioning.py

@@ -0,0 +1,464 @@
+# pragma pylint: disable=W0603
+""" Edge positioning package """
+import logging
+from typing import Any, Dict, NamedTuple
+
+import arrow
+import numpy as np
+import utils_find_1st as utf1st
+from pandas import DataFrame
+
+from freqtrade import constants
+from freqtrade.configuration import TimeRange
+from freqtrade.data import history
+from freqtrade.exceptions import OperationalException
+from freqtrade.strategy.interface import SellType
+
+logger = logging.getLogger(__name__)
+
+
+class PairInfo(NamedTuple):
+    stoploss: float
+    winrate: float
+    risk_reward_ratio: float
+    required_risk_reward: float
+    expectancy: float
+    nb_trades: int
+    avg_trade_duration: float
+
+
+class Edge:
+    """
+    Calculates Win Rate, Risk Reward Ratio, Expectancy
+    against historical data for a give set of markets and a strategy
+    it then adjusts stoploss and position size accordingly
+    and force it into the strategy
+    Author: https://github.com/mishaker
+    """
+
+    config: Dict = {}
+    _cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
+
+    def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:
+
+        self.config = config
+        self.exchange = exchange
+        self.strategy = strategy
+
+        self.edge_config = self.config.get('edge', {})
+        self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
+        self._final_pairs: list = []
+
+        # checking max_open_trades. it should be -1 as with Edge
+        # the number of trades is determined by position size
+        if self.config['max_open_trades'] != float('inf'):
+            logger.critical('max_open_trades should be -1 in config !')
+
+        if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT:
+            raise OperationalException('Edge works only with unlimited stake amount')
+
+        # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future.
+        self._capital_percentage: float = self.edge_config.get(
+            'capital_available_percentage', self.config['tradable_balance_ratio'])
+        self._allowed_risk: float = self.edge_config.get('allowed_risk')
+        self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14)
+        self._last_updated: int = 0  # Timestamp of pairs last updated time
+        self._refresh_pairs = True
+
+        self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01))
+        self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05))
+        self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001))
+
+        # calculating stoploss range
+        self._stoploss_range = np.arange(
+            self._stoploss_range_min,
+            self._stoploss_range_max,
+            self._stoploss_range_step
+        )
+
+        self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
+            days=-1 * self._since_number_of_days).format('YYYYMMDD'))
+        if config.get('fee'):
+            self.fee = config['fee']
+        else:
+            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
+
+    def calculate(self) -> bool:
+        pairs = self.config['exchange']['pair_whitelist']
+        heartbeat = self.edge_config.get('process_throttle_secs')
+
+        if (self._last_updated > 0) and (
+                self._last_updated + heartbeat > arrow.utcnow().timestamp):
+            return False
+
+        data: Dict[str, Any] = {}
+        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
+        logger.info('Using local backtesting data (using whitelist in given config) ...')
+
+        if self._refresh_pairs:
+            history.refresh_data(
+                datadir=self.config['datadir'],
+                pairs=pairs,
+                exchange=self.exchange,
+                timeframe=self.strategy.ticker_interval,
+                timerange=self._timerange,
+            )
+
+        data = history.load_data(
+            datadir=self.config['datadir'],
+            pairs=pairs,
+            timeframe=self.strategy.ticker_interval,
+            timerange=self._timerange,
+            startup_candles=self.strategy.startup_candle_count,
+        )
+
+        if not data:
+            # Reinitializing cached pairs
+            self._cached_pairs = {}
+            logger.critical("No data found. Edge is stopped ...")
+            return False
+
+        preprocessed = self.strategy.tickerdata_to_dataframe(data)
+
+        # Print timeframe
+        min_date, max_date = history.get_timerange(preprocessed)
+        logger.info(
+            'Measuring data from %s up to %s (%s days) ...',
+            min_date.isoformat(),
+            max_date.isoformat(),
+            (max_date - min_date).days
+        )
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
+
+        trades: list = []
+        for pair, pair_data in preprocessed.items():
+            # Sorting dataframe by date and reset index
+            pair_data = pair_data.sort_values(by=['date'])
+            pair_data = pair_data.reset_index(drop=True)
+
+            ticker_data = self.strategy.advise_sell(
+                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
+
+            trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range)
+
+        # If no trade found then exit
+        if len(trades) == 0:
+            logger.info("No trades found.")
+            return False
+
+        # Fill missing, calculable columns, profit, duration , abs etc.
+        trades_df = self._fill_calculable_fields(DataFrame(trades))
+        self._cached_pairs = self._process_expectancy(trades_df)
+        self._last_updated = arrow.utcnow().timestamp
+
+        return True
+
+    def stake_amount(self, pair: str, free_capital: float,
+                     total_capital: float, capital_in_trade: float) -> float:
+        stoploss = self.stoploss(pair)
+        available_capital = (total_capital + capital_in_trade) * self._capital_percentage
+        allowed_capital_at_risk = available_capital * self._allowed_risk
+        max_position_size = abs(allowed_capital_at_risk / stoploss)
+        position_size = min(max_position_size, free_capital)
+        if pair in self._cached_pairs:
+            logger.info(
+                'winrate: %s, expectancy: %s, position size: %s, pair: %s,'
+                ' capital in trade: %s, free capital: %s, total capital: %s,'
+                ' stoploss: %s, available capital: %s.',
+                self._cached_pairs[pair].winrate,
+                self._cached_pairs[pair].expectancy,
+                position_size, pair,
+                capital_in_trade, free_capital, total_capital,
+                stoploss, available_capital
+            )
+        return round(position_size, 15)
+
+    def stoploss(self, pair: str) -> float:
+        if pair in self._cached_pairs:
+            return self._cached_pairs[pair].stoploss
+        else:
+            logger.warning('tried to access stoploss of a non-existing pair, '
+                           'strategy stoploss is returned instead.')
+            return self.strategy.stoploss
+
+    def adjust(self, pairs) -> list:
+        """
+        Filters out and sorts "pairs" according to Edge calculated pairs
+        """
+        final = []
+        for pair, info in self._cached_pairs.items():
+            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
+                info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
+                    pair in pairs:
+                final.append(pair)
+
+        if self._final_pairs != final:
+            self._final_pairs = final
+            if self._final_pairs:
+                logger.info(
+                    'Minimum expectancy and minimum winrate are met only for %s,'
+                    ' so other pairs are filtered out.',
+                    self._final_pairs
+                    )
+            else:
+                logger.info(
+                    'Edge removed all pairs as no pair with minimum expectancy '
+                    'and minimum winrate was found !'
+                    )
+
+        return self._final_pairs
+
+    def accepted_pairs(self) -> list:
+        """
+        return a list of accepted pairs along with their winrate, expectancy and stoploss
+        """
+        final = []
+        for pair, info in self._cached_pairs.items():
+            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
+                 info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
+                final.append({
+                    'Pair': pair,
+                    'Winrate': info.winrate,
+                    'Expectancy': info.expectancy,
+                    'Stoploss': info.stoploss,
+                })
+        return final
+
+    def _fill_calculable_fields(self, result: DataFrame) -> DataFrame:
+        """
+        The result frame contains a number of columns that are calculable
+        from other columns. These are left blank till all rows are added,
+        to be populated in single vector calls.
+
+        Columns to be populated are:
+        - Profit
+        - trade duration
+        - profit abs
+        :param result Dataframe
+        :return: result Dataframe
+        """
+
+        # stake and fees
+        # stake = 0.015
+        # 0.05% is 0.0005
+        # fee = 0.001
+
+        # we set stake amount to an arbitrary amount.
+        # as it doesn't change the calculation.
+        # all returned values are relative. they are percentages.
+        stake = 0.015
+        fee = self.fee
+        open_fee = fee / 2
+        close_fee = fee / 2
+
+        result['trade_duration'] = result['close_time'] - result['open_time']
+
+        result['trade_duration'] = result['trade_duration'].map(
+            lambda x: int(x.total_seconds() / 60))
+
+        # Spends, Takes, Profit, Absolute Profit
+
+        # Buy Price
+        result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
+        result['buy_fee'] = stake * open_fee
+        result['buy_spend'] = stake + result['buy_fee']  # How much we're spending
+
+        # Sell price
+        result['sell_sum'] = result['buy_vol'] * result['close_rate']
+        result['sell_fee'] = result['sell_sum'] * close_fee
+        result['sell_take'] = result['sell_sum'] - result['sell_fee']
+
+        # profit_percent
+        result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']
+
+        # Absolute profit
+        result['profit_abs'] = result['sell_take'] - result['buy_spend']
+
+        return result
+
+    def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
+        """
+        This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
+        The calulation will be done per pair and per strategy.
+        """
+        # Removing pairs having less than min_trades_number
+        min_trades_number = self.edge_config.get('min_trade_number', 10)
+        results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number)
+        ###################################
+
+        # Removing outliers (Only Pumps) from the dataset
+        # The method to detect outliers is to calculate standard deviation
+        # Then every value more than (standard deviation + 2*average) is out (pump)
+        #
+        # Removing Pumps
+        if self.edge_config.get('remove_pumps', False):
+            results = results.groupby(['pair', 'stoploss']).apply(
+                lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()])
+        ##########################################################################
+
+        # Removing trades having a duration more than X minutes (set in config)
+        max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440)
+        results = results[results.trade_duration < max_trade_duration]
+        #######################################################################
+
+        if results.empty:
+            return {}
+
+        groupby_aggregator = {
+            'profit_abs': [
+                ('nb_trades', 'count'),  # number of all trades
+                ('profit_sum', lambda x: x[x > 0].sum()),  # cumulative profit of all winning trades
+                ('loss_sum', lambda x: abs(x[x < 0].sum())),  # cumulative loss of all losing trades
+                ('nb_win_trades', lambda x: x[x > 0].count())  # number of winning trades
+            ],
+            'trade_duration': [('avg_trade_duration', 'mean')]
+        }
+
+        # Group by (pair and stoploss) by applying above aggregator
+        df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg(
+            groupby_aggregator).reset_index(col_level=1)
+
+        # Dropping level 0 as we don't need it
+        df.columns = df.columns.droplevel(0)
+
+        # Calculating number of losing trades, average win and average loss
+        df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']
+        df['average_win'] = df['profit_sum'] / df['nb_win_trades']
+        df['average_loss'] = df['loss_sum'] / df['nb_loss_trades']
+
+        # Win rate = number of profitable trades / number of trades
+        df['winrate'] = df['nb_win_trades'] / df['nb_trades']
+
+        # risk_reward_ratio = average win / average loss
+        df['risk_reward_ratio'] = df['average_win'] / df['average_loss']
+
+        # required_risk_reward = (1 / winrate) - 1
+        df['required_risk_reward'] = (1 / df['winrate']) - 1
+
+        # expectancy = (risk_reward_ratio * winrate) - (lossrate)
+        df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate'])
+
+        # sort by expectancy and stoploss
+        df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby(
+            'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index()
+
+        final = {}
+        for x in df.itertuples():
+            final[x.pair] = PairInfo(
+                x.stoploss,
+                x.winrate,
+                x.risk_reward_ratio,
+                x.required_risk_reward,
+                x.expectancy,
+                x.nb_trades,
+                x.avg_trade_duration
+            )
+
+        # Returning a list of pairs in order of "expectancy"
+        return final
+
+    def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range):
+        buy_column = ticker_data['buy'].values
+        sell_column = ticker_data['sell'].values
+        date_column = ticker_data['date'].values
+        ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values
+
+        result: list = []
+        for stoploss in stoploss_range:
+            result += self._detect_next_stop_or_sell_point(
+                buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair
+            )
+
+        return result
+
+    def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,
+                                        ohlc_columns, stoploss, pair):
+        """
+        Iterate through ohlc_columns in order to find the next trade
+        Next trade opens from the first buy signal noticed to
+        The sell or stoploss signal after it.
+        It then cuts OHLC, buy_column, sell_column and date_column.
+        Cut from (the exit trade index) + 1.
+
+        Author: https://github.com/mishaker
+        """
+
+        result: list = []
+        start_point = 0
+
+        while True:
+            open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal)
+
+            # Return empty if we don't find trade entry (i.e. buy==1) or
+            # we find a buy but at the end of array
+            if open_trade_index == -1 or open_trade_index == len(buy_column) - 1:
+                break
+            else:
+                # When a buy signal is seen,
+                # trade opens in reality on the next candle
+                open_trade_index += 1
+
+            stop_price_percentage = stoploss + 1
+            open_price = ohlc_columns[open_trade_index, 0]
+            stop_price = (open_price * stop_price_percentage)
+
+            # Searching for the index where stoploss is hit
+            stop_index = utf1st.find_1st(
+                ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller)
+
+            # If we don't find it then we assume stop_index will be far in future (infinite number)
+            if stop_index == -1:
+                stop_index = float('inf')
+
+            # Searching for the index where sell is hit
+            sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal)
+
+            # If we don't find it then we assume sell_index will be far in future (infinite number)
+            if sell_index == -1:
+                sell_index = float('inf')
+
+            # Check if we don't find any stop or sell point (in that case trade remains open)
+            # It is not interesting for Edge to consider it so we simply ignore the trade
+            # And stop iterating there is no more entry
+            if stop_index == sell_index == float('inf'):
+                break
+
+            if stop_index <= sell_index:
+                exit_index = open_trade_index + stop_index
+                exit_type = SellType.STOP_LOSS
+                exit_price = stop_price
+            elif stop_index > sell_index:
+                # If exit is SELL then we exit at the next candle
+                exit_index = open_trade_index + sell_index + 1
+
+                # Check if we have the next candle
+                if len(ohlc_columns) - 1 < exit_index:
+                    break
+
+                exit_type = SellType.SELL_SIGNAL
+                exit_price = ohlc_columns[exit_index, 0]
+
+            trade = {'pair': pair,
+                     'stoploss': stoploss,
+                     'profit_percent': '',
+                     'profit_abs': '',
+                     'open_time': date_column[open_trade_index],
+                     'close_time': date_column[exit_index],
+                     'open_index': start_point + open_trade_index,
+                     'close_index': start_point + exit_index,
+                     'trade_duration': '',
+                     'open_rate': round(open_price, 15),
+                     'close_rate': round(exit_price, 15),
+                     'exit_type': exit_type
+                     }
+
+            result.append(trade)
+
+            # Giving a view of exit_index till the end of array
+            buy_column = buy_column[exit_index:]
+            sell_column = sell_column[exit_index:]
+            date_column = date_column[exit_index:]
+            ohlc_columns = ohlc_columns[exit_index:]
+            start_point += exit_index
+
+        return result

freqtrade/main.py

@@ -14,7 +14,7 @@ if sys.version_info < (3, 6):
 import logging
 from typing import Any, List
 
-from freqtrade.configuration import Arguments
+from freqtrade.commands import Arguments
 
 
 logger = logging.getLogger('freqtrade')

freqtrade/optimize/__init__.py

@@ -1,102 +0,0 @@
-import logging
-from typing import Any, Dict
-
-from freqtrade import constants
-from freqtrade.exceptions import DependencyException, OperationalException
-from freqtrade.state import RunMode
-from freqtrade.utils import setup_utils_configuration
-
-logger = logging.getLogger(__name__)
-
-
-def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
-    """
-    Prepare the configuration for the Hyperopt module
-    :param args: Cli args from Arguments()
-    :return: Configuration
-    """
-    config = setup_utils_configuration(args, method)
-
-    if method == RunMode.BACKTEST:
-        if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
-            raise DependencyException('stake amount could not be "%s" for backtesting' %
-                                      constants.UNLIMITED_STAKE_AMOUNT)
-
-    return config
-
-
-def start_backtesting(args: Dict[str, Any]) -> None:
-    """
-    Start Backtesting script
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    # Import here to avoid loading backtesting module when it's not used
-    from freqtrade.optimize.backtesting import Backtesting
-
-    # Initialize configuration
-    config = setup_configuration(args, RunMode.BACKTEST)
-
-    logger.info('Starting freqtrade in Backtesting mode')
-
-    # Initialize backtesting object
-    backtesting = Backtesting(config)
-    backtesting.start()
-
-
-def start_hyperopt(args: Dict[str, Any]) -> None:
-    """
-    Start hyperopt script
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    # Import here to avoid loading hyperopt module when it's not used
-    try:
-        from filelock import FileLock, Timeout
-        from freqtrade.optimize.hyperopt import Hyperopt
-    except ImportError as e:
-        raise OperationalException(
-            f"{e}. Please ensure that the hyperopt dependencies are installed.") from e
-    # Initialize configuration
-    config = setup_configuration(args, RunMode.HYPEROPT)
-
-    logger.info('Starting freqtrade in Hyperopt mode')
-
-    lock = FileLock(Hyperopt.get_lock_filename(config))
-
-    try:
-        with lock.acquire(timeout=1):
-
-            # Remove noisy log messages
-            logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
-            logging.getLogger('filelock').setLevel(logging.WARNING)
-
-            # Initialize backtesting object
-            hyperopt = Hyperopt(config)
-            hyperopt.start()
-
-    except Timeout:
-        logger.info("Another running instance of freqtrade Hyperopt detected.")
-        logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. "
-                    "Hyperopt module is resource hungry. Please run your Hyperopt sequentially "
-                    "or on separate machines.")
-        logger.info("Quitting now.")
-        # TODO: return False here in order to help freqtrade to exit
-        # with non-zero exit code...
-        # Same in Edge and Backtesting start() functions.
-
-
-def start_edge(args: Dict[str, Any]) -> None:
-    """
-    Start Edge script
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    from freqtrade.optimize.edge_cli import EdgeCli
-    # Initialize configuration
-    config = setup_configuration(args, RunMode.EDGE)
-    logger.info('Starting freqtrade in Edge mode')
-
-    # Initialize Edge object
-    edge_cli = EdgeCli(config)
-    edge_cli.start()

freqtrade/optimize/backtesting.py

@@ -279,30 +279,28 @@ class Backtesting:
             return bt_res
         return None
 
-    def backtest(self, args: Dict) -> DataFrame:
+    def backtest(self, processed: Dict, stake_amount: float,
+                 start_date, end_date,
+                 max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame:
         """
-        Implements backtesting functionality
+        Implement backtesting functionality
 
         NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
         Of course try to not have ugly code. By some accessor are sometime slower than functions.
-        Avoid, logging on this method
+        Avoid extensive logging in this method and functions it calls.
 
-        :param args: a dict containing:
+        :param processed: a processed dictionary with format {pair, data}
-            stake_amount: btc amount to use for each trade
+        :param stake_amount: amount to use for each trade
-            processed: a processed dictionary with format {pair, data}
+        :param start_date: backtesting timerange start datetime
-            max_open_trades: maximum number of concurrent trades (default: 0, disabled)
+        :param end_date: backtesting timerange end datetime
-            position_stacking: do we allow position stacking? (default: False)
+        :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited
-        :return: DataFrame
+        :param position_stacking: do we allow position stacking?
+        :return: DataFrame with trades (results of backtesting)
         """
-        # Arguments are long and noisy, so this is commented out.
+        logger.debug(f"Run backtest, stake_amount: {stake_amount}, "
-        # Uncomment if you need to debug the backtest() method.
+                     f"start_date: {start_date}, end_date: {end_date}, "
-        # logger.debug(f"Start backtest, args: {args}")
+                     f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}"
-        processed = args['processed']
+                     )
-        stake_amount = args['stake_amount']
-        max_open_trades = args.get('max_open_trades', 0)
-        position_stacking = args.get('position_stacking', False)
-        start_date = args['start_date']
-        end_date = args['end_date']
         trades = []
         trade_count_lock: Dict = {}
 
@@ -369,18 +367,21 @@ class Backtesting:
 
     def start(self) -> None:
         """
-        Run a backtesting end-to-end
+        Run backtesting end-to-end
         :return: None
         """
         data: Dict[str, Any] = {}
+
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
+
         # Use max_open_trades in backtesting, except --disable-max-market-positions is set
         if self.config.get('use_max_market_positions', True):
             max_open_trades = self.config['max_open_trades']
         else:
             logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
             max_open_trades = 0
+        position_stacking = self.config.get('position_stacking', False)
 
         data, timerange = self.load_bt_data()
 
@@ -403,14 +404,12 @@ class Backtesting:
             )
             # Execute backtest and print results
             all_results[self.strategy.get_strategy_name()] = self.backtest(
-                {
+                    processed=preprocessed,
-                    'stake_amount': self.config.get('stake_amount'),
+                    stake_amount=self.config['stake_amount'],
-                    'processed': preprocessed,
+                    start_date=min_date,
-                    'max_open_trades': max_open_trades,
+                    end_date=max_date,
-                    'position_stacking': self.config.get('position_stacking', False),
+                    max_open_trades=max_open_trades,
-                    'start_date': min_date,
+                    position_stacking=position_stacking,
-                    'end_date': max_date,
-                }
             )
 
         for strategy, results in all_results.items():

freqtrade/optimize/hyperopt.py

@@ -372,14 +372,12 @@ class Hyperopt:
         min_date, max_date = get_timerange(processed)
 
         backtesting_results = self.backtesting.backtest(
-            {
+                processed=processed,
-                'stake_amount': self.config['stake_amount'],
+                stake_amount=self.config['stake_amount'],
-                'processed': processed,
+                start_date=min_date,
-                'max_open_trades': self.max_open_trades,
+                end_date=max_date,
-                'position_stacking': self.position_stacking,
+                max_open_trades=self.max_open_trades,
-                'start_date': min_date,
+                position_stacking=self.position_stacking,
-                'end_date': max_date,
-            }
         )
         return self._get_results_dict(backtesting_results, min_date, max_date,
                                       params_dict, params_details)

freqtrade/utils.py

@@ -1,482 +0,0 @@
-import csv
-import logging
-import sys
-from collections import OrderedDict
-from operator import itemgetter
-from pathlib import Path
-from typing import Any, Dict, List
-
-import arrow
-import rapidjson
-from colorama import init as colorama_init
-from tabulate import tabulate
-
-from freqtrade.configuration import (Configuration, TimeRange,
-                                     remove_credentials,
-                                     validate_config_consistency)
-from freqtrade.configuration.directory_operations import (copy_sample_files,
-                                                          create_userdata_dir)
-from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
-from freqtrade.data.history import (convert_trades_to_ohlcv,
-                                    refresh_backtest_ohlcv_data,
-                                    refresh_backtest_trades_data)
-from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
-                                market_is_active, symbol_is_pair)
-from freqtrade.misc import plural, render_template
-from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.state import RunMode
-
-logger = logging.getLogger(__name__)
-
-
-def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
-    """
-    Prepare the configuration for utils subcommands
-    :param args: Cli args from Arguments()
-    :return: Configuration
-    """
-    configuration = Configuration(args, method)
-    config = configuration.get_config()
-
-    # Ensure we do not use Exchange credentials
-    remove_credentials(config)
-    validate_config_consistency(config)
-
-    return config
-
-
-def start_trading(args: Dict[str, Any]) -> int:
-    """
-    Main entry point for trading mode
-    """
-    from freqtrade.worker import Worker
-    # Load and run worker
-    worker = None
-    try:
-        worker = Worker(args)
-        worker.run()
-    except KeyboardInterrupt:
-        logger.info('SIGINT received, aborting ...')
-    finally:
-        if worker:
-            logger.info("worker found ... calling exit")
-            worker.exit()
-    return 0
-
-
-def start_list_exchanges(args: Dict[str, Any]) -> None:
-    """
-    Print available exchanges
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
-    if args['print_one_column']:
-        print('\n'.join(exchanges))
-    else:
-        if args['list_exchanges_all']:
-            print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
-        else:
-            print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
-
-
-def start_create_userdir(args: Dict[str, Any]) -> None:
-    """
-    Create "user_data" directory to contain user data strategies, hyperopt, ...)
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    if "user_data_dir" in args and args["user_data_dir"]:
-        userdir = create_userdata_dir(args["user_data_dir"], create_dir=True)
-        copy_sample_files(userdir, overwrite=args["reset"])
-    else:
-        logger.warning("`create-userdir` requires --userdir to be set.")
-        sys.exit(1)
-
-
-def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str):
-    """
-    Deploy new strategy from template to strategy_path
-    """
-    indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",)
-    buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",)
-    sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",)
-    plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",)
-
-    strategy_text = render_template(templatefile='base_strategy.py.j2',
-                                    arguments={"strategy": strategy_name,
-                                               "indicators": indicators,
-                                               "buy_trend": buy_trend,
-                                               "sell_trend": sell_trend,
-                                               "plot_config": plot_config,
-                                               })
-
-    logger.info(f"Writing strategy to `{strategy_path}`.")
-    strategy_path.write_text(strategy_text)
-
-
-def start_new_strategy(args: Dict[str, Any]) -> None:
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    if "strategy" in args and args["strategy"]:
-        if args["strategy"] == "DefaultStrategy":
-            raise OperationalException("DefaultStrategy is not allowed as name.")
-
-        new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py")
-
-        if new_path.exists():
-            raise OperationalException(f"`{new_path}` already exists. "
-                                       "Please choose another Strategy Name.")
-
-        deploy_new_strategy(args['strategy'], new_path, args['template'])
-
-    else:
-        raise OperationalException("`new-strategy` requires --strategy to be set.")
-
-
-def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str):
-    """
-    Deploys a new hyperopt template to hyperopt_path
-    """
-    buy_guards = render_template(
-        templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",)
-    sell_guards = render_template(
-        templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",)
-    buy_space = render_template(
-        templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",)
-    sell_space = render_template(
-        templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",)
-
-    strategy_text = render_template(templatefile='base_hyperopt.py.j2',
-                                    arguments={"hyperopt": hyperopt_name,
-                                               "buy_guards": buy_guards,
-                                               "sell_guards": sell_guards,
-                                               "buy_space": buy_space,
-                                               "sell_space": sell_space,
-                                               })
-
-    logger.info(f"Writing hyperopt to `{hyperopt_path}`.")
-    hyperopt_path.write_text(strategy_text)
-
-
-def start_new_hyperopt(args: Dict[str, Any]) -> None:
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    if "hyperopt" in args and args["hyperopt"]:
-        if args["hyperopt"] == "DefaultHyperopt":
-            raise OperationalException("DefaultHyperopt is not allowed as name.")
-
-        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py")
-
-        if new_path.exists():
-            raise OperationalException(f"`{new_path}` already exists. "
-                                       "Please choose another Strategy Name.")
-        deploy_new_hyperopt(args['hyperopt'], new_path, args['template'])
-    else:
-        raise OperationalException("`new-hyperopt` requires --hyperopt to be set.")
-
-
-def start_download_data(args: Dict[str, Any]) -> None:
-    """
-    Download data (former download_backtest_data.py script)
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    timerange = TimeRange()
-    if 'days' in config:
-        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
-        timerange = TimeRange.parse_timerange(f'{time_since}-')
-
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
-
-    logger.info(f'About to download pairs: {config["pairs"]}, '
-                f'intervals: {config["timeframes"]} to {config["datadir"]}')
-
-    pairs_not_available: List[str] = []
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
-    try:
-
-        if config.get('download_trades'):
-            pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config["pairs"], datadir=config['datadir'],
-                timerange=timerange, erase=config.get("erase"))
-
-            # Convert downloaded trade data to different timeframes
-            convert_trades_to_ohlcv(
-                pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
-        else:
-            pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"))
-
-    except KeyboardInterrupt:
-        sys.exit("SIGINT received, aborting ...")
-
-    finally:
-        if pairs_not_available:
-            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
-                        f"on exchange {exchange.name}.")
-
-
-def start_list_strategies(args: Dict[str, Any]) -> None:
-    """
-    Print Strategies available in a directory
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY))
-    strategies = StrategyResolver.search_all_objects(directory)
-    # Sort alphabetically
-    strategies = sorted(strategies, key=lambda x: x['name'])
-    strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies]
-
-    if args['print_one_column']:
-        print('\n'.join([s['name'] for s in strategies]))
-    else:
-        print(tabulate(strats_to_print, headers='keys', tablefmt='pipe'))
-
-
-def start_list_timeframes(args: Dict[str, Any]) -> None:
-    """
-    Print ticker intervals (timeframes) available on Exchange
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-    # Do not use ticker_interval set in the config
-    config['ticker_interval'] = None
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
-
-    if args['print_one_column']:
-        print('\n'.join(exchange.timeframes))
-    else:
-        print(f"Timeframes available for the exchange `{exchange.name}`: "
-              f"{', '.join(exchange.timeframes)}")
-
-
-def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
-    """
-    Print pairs/markets on the exchange
-    :param args: Cli args from Arguments()
-    :param pairs_only: if True print only pairs, otherwise print all instruments (markets)
-    :return: None
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
-
-    # By default only active pairs/markets are to be shown
-    active_only = not args.get('list_pairs_all', False)
-
-    base_currencies = args.get('base_currencies', [])
-    quote_currencies = args.get('quote_currencies', [])
-
-    try:
-        pairs = exchange.get_markets(base_currencies=base_currencies,
-                                     quote_currencies=quote_currencies,
-                                     pairs_only=pairs_only,
-                                     active_only=active_only)
-        # Sort the pairs/markets by symbol
-        pairs = OrderedDict(sorted(pairs.items()))
-    except Exception as e:
-        raise OperationalException(f"Cannot get markets. Reason: {e}") from e
-
-    else:
-        summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") +
-                       ("active " if active_only else "") +
-                       (plural(len(pairs), "pair" if pairs_only else "market")) +
-                       (f" with {', '.join(base_currencies)} as base "
-                        f"{plural(len(base_currencies), 'currency', 'currencies')}"
-                        if base_currencies else "") +
-                       (" and" if base_currencies and quote_currencies else "") +
-                       (f" with {', '.join(quote_currencies)} as quote "
-                        f"{plural(len(quote_currencies), 'currency', 'currencies')}"
-                        if quote_currencies else ""))
-
-        headers = ["Id", "Symbol", "Base", "Quote", "Active",
-                   *(['Is pair'] if not pairs_only else [])]
-
-        tabular_data = []
-        for _, v in pairs.items():
-            tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
-                                 'Base': v['base'], 'Quote': v['quote'],
-                                 'Active': market_is_active(v),
-                                 **({'Is pair': symbol_is_pair(v['symbol'])}
-                                    if not pairs_only else {})})
-
-        if (args.get('print_one_column', False) or
-                args.get('list_pairs_print_json', False) or
-                args.get('print_csv', False)):
-            # Print summary string in the log in case of machine-readable
-            # regular formats.
-            logger.info(f"{summary_str}.")
-        else:
-            # Print empty string separating leading logs and output in case of
-            # human-readable formats.
-            print()
-
-        if len(pairs):
-            if args.get('print_list', False):
-                # print data as a list, with human-readable summary
-                print(f"{summary_str}: {', '.join(pairs.keys())}.")
-            elif args.get('print_one_column', False):
-                print('\n'.join(pairs.keys()))
-            elif args.get('list_pairs_print_json', False):
-                print(rapidjson.dumps(list(pairs.keys()), default=str))
-            elif args.get('print_csv', False):
-                writer = csv.DictWriter(sys.stdout, fieldnames=headers)
-                writer.writeheader()
-                writer.writerows(tabular_data)
-            else:
-                # print data as a table, with the human-readable summary
-                print(f"{summary_str}:")
-                print(tabulate(tabular_data, headers='keys', tablefmt='pipe'))
-        elif not (args.get('print_one_column', False) or
-                  args.get('list_pairs_print_json', False) or
-                  args.get('print_csv', False)):
-            print(f"{summary_str}.")
-
-
-def start_test_pairlist(args: Dict[str, Any]) -> None:
-    """
-    Test Pairlist configuration
-    """
-    from freqtrade.pairlist.pairlistmanager import PairListManager
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
-
-    quote_currencies = args.get('quote_currencies')
-    if not quote_currencies:
-        quote_currencies = [config.get('stake_currency')]
-    results = {}
-    for curr in quote_currencies:
-        config['stake_currency'] = curr
-        # Do not use ticker_interval set in the config
-        pairlists = PairListManager(exchange, config)
-        pairlists.refresh_pairlist()
-        results[curr] = pairlists.whitelist
-
-    for curr, pairlist in results.items():
-        if not args.get('print_one_column', False):
-            print(f"Pairs for {curr}: ")
-
-        if args.get('print_one_column', False):
-            print('\n'.join(pairlist))
-        elif args.get('list_pairs_print_json', False):
-            print(rapidjson.dumps(list(pairlist), default=str))
-        else:
-            print(pairlist)
-
-
-def start_hyperopt_list(args: Dict[str, Any]) -> None:
-    """
-    List hyperopt epochs previously evaluated
-    """
-    from freqtrade.optimize.hyperopt import Hyperopt
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    only_best = config.get('hyperopt_list_best', False)
-    only_profitable = config.get('hyperopt_list_profitable', False)
-    print_colorized = config.get('print_colorized', False)
-    print_json = config.get('print_json', False)
-    no_details = config.get('hyperopt_list_no_details', False)
-    no_header = False
-
-    trials_file = (config['user_data_dir'] /
-                   'hyperopt_results' / 'hyperopt_results.pickle')
-
-    # Previous evaluations
-    trials = Hyperopt.load_previous_results(trials_file)
-    total_epochs = len(trials)
-
-    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
-
-    # TODO: fetch the interval for epochs to print from the cli option
-    epoch_start, epoch_stop = 0, None
-
-    if print_colorized:
-        colorama_init(autoreset=True)
-
-    try:
-        # Human-friendly indexes used here (starting from 1)
-        for val in trials[epoch_start:epoch_stop]:
-            Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized)
-
-    except KeyboardInterrupt:
-        print('User interrupted..')
-
-    if trials and not no_details:
-        sorted_trials = sorted(trials, key=itemgetter('loss'))
-        results = sorted_trials[0]
-        Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
-
-
-def start_hyperopt_show(args: Dict[str, Any]) -> None:
-    """
-    Show details of a hyperopt epoch previously evaluated
-    """
-    from freqtrade.optimize.hyperopt import Hyperopt
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    only_best = config.get('hyperopt_list_best', False)
-    only_profitable = config.get('hyperopt_list_profitable', False)
-    no_header = config.get('hyperopt_show_no_header', False)
-
-    trials_file = (config['user_data_dir'] /
-                   'hyperopt_results' / 'hyperopt_results.pickle')
-
-    # Previous evaluations
-    trials = Hyperopt.load_previous_results(trials_file)
-    total_epochs = len(trials)
-
-    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
-    trials_epochs = len(trials)
-
-    n = config.get('hyperopt_show_index', -1)
-    if n > trials_epochs:
-        raise OperationalException(
-                f"The index of the epoch to show should be less than {trials_epochs + 1}.")
-    if n < -trials_epochs:
-        raise OperationalException(
-                f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
-
-    # Translate epoch index from human-readable format to pythonic
-    if n > 0:
-        n -= 1
-
-    print_json = config.get('print_json', False)
-
-    if trials:
-        val = trials[n]
-        Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
-                                     header_str="Epoch details")
-
-
-def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List:
-    """
-    Filter our items from the list of hyperopt results
-    """
-    if only_best:
-        trials = [x for x in trials if x['is_best']]
-    if only_profitable:
-        trials = [x for x in trials if x['results_metrics']['profit'] > 0]
-
-    logger.info(f"{len(trials)} " +
-                ("best " if only_best else "") +
-                ("profitable " if only_profitable else "") +
-                "epochs found.")
-
-    return trials

tests/commands/test_commands.py

@@ -4,15 +4,15 @@ from unittest.mock import MagicMock, PropertyMock
 
 import pytest
 
+from freqtrade.commands import (start_create_userdir, start_download_data,
+                                start_hyperopt_list, start_hyperopt_show,
+                                start_list_exchanges, start_list_markets,
+                                start_list_strategies, start_list_timeframes,
+                                start_new_hyperopt, start_new_strategy,
+                                start_test_pairlist, start_trading)
+from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
-from freqtrade.utils import (setup_utils_configuration, start_create_userdir,
-                             start_download_data, start_hyperopt_list,
-                             start_hyperopt_show, start_list_exchanges,
-                             start_list_markets, start_list_strategies,
-                             start_list_timeframes, start_new_hyperopt,
-                             start_new_strategy, start_test_pairlist,
-                             start_trading)
 from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
@@ -451,8 +451,8 @@ def test_create_datadir(caplog, mocker):
     # Added assert here to analyze random test-failures ...
     assert len(caplog.record_tuples) == 0
 
-    cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock())
+    cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock())
-    csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock())
+    csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock())
     args = [
         "create-userdir",
         "--userdir",
@@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog):
 
 
 def test_download_data_keyboardInterrupt(mocker, caplog, markets):
-    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+    dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(side_effect=KeyboardInterrupt))
     patch_exchange(mocker)
     mocker.patch(
@@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets):
 
 
 def test_download_data_no_markets(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+    dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                            MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker, id='binance')
     mocker.patch(
@@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog):
 
 
 def test_download_data_no_exchange(mocker, caplog):
-    mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+    mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                  MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
     mocker.patch(
@@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog):
 
     mocker.patch.object(Path, "exists", MagicMock(return_value=False))
 
-    mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data',
+    mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data',
                  MagicMock(return_value=["ETH/BTC", "XRP/BTC"]))
     patch_exchange(mocker)
     mocker.patch(
@@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog):
 
 
 def test_download_data_trades(mocker, caplog):
-    dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data',
+    dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data',
                            MagicMock(return_value=[]))
-    convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv',
+    convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
                                 MagicMock(return_value=[]))
     patch_exchange(mocker)
     mocker.patch(
@@ -639,7 +639,7 @@ def test_start_list_strategies(mocker, caplog, capsys):
     args = [
         "list-strategies",
         "--strategy-path",
-        str(Path(__file__).parent / "strategy"),
+        str(Path(__file__).parent.parent / "strategy"),
         "-1"
     ]
     pargs = get_args(args)
@@ -654,7 +654,7 @@ def test_start_list_strategies(mocker, caplog, capsys):
     args = [
         "list-strategies",
         "--strategy-path",
-        str(Path(__file__).parent / "strategy"),
+        str(Path(__file__).parent.parent / "strategy"),
     ]
     pargs = get_args(args)
     # pargs['config'] = None
@@ -665,11 +665,11 @@ def test_start_list_strategies(mocker, caplog, capsys):
     assert "DefaultStrategy" in captured.out
 
 
-def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys):
+def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys):
+    patch_exchange(mocker, mock_markets=True)
     mocker.patch.multiple('freqtrade.exchange.Exchange',
-                          markets=PropertyMock(return_value=markets),
                           exchange_has=MagicMock(return_value=True),
-                          get_tickers=tickers
+                          get_tickers=tickers,
                           )
 
     default_conf['pairlists'] = [

tests/conftest.py

@@ -14,7 +14,7 @@ import pytest
 from telegram import Chat, Message, Update
 
 from freqtrade import constants, persistence
-from freqtrade.configuration import Arguments
+from freqtrade.commands import Arguments
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.edge import Edge, PairInfo
 from freqtrade.exchange import Exchange

tests/optimize/test_backtest_detail.py

@@ -382,13 +382,11 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     data_processed = {pair: frame.copy()}
     min_date, max_date = get_timerange({pair: frame})
     results = backtesting.backtest(
-        {
+        processed=data_processed,
-            'stake_amount': default_conf['stake_amount'],
+        stake_amount=default_conf['stake_amount'],
-            'processed': data_processed,
+        start_date=min_date,
-            'max_open_trades': 10,
+        end_date=max_date,
-            'start_date': min_date,
+        max_open_trades=10,
-            'end_date': max_date,
-        }
     )
 
     assert len(results) == len(data.trades)

tests/optimize/test_backtesting.py

@@ -12,13 +12,13 @@ from arrow import Arrow
 
 from freqtrade import constants
 from freqtrade.configuration import TimeRange
+from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting
 from freqtrade.data import history
 from freqtrade.data.btanalysis import evaluate_result_multi
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import get_timerange
 from freqtrade.exceptions import DependencyException, OperationalException
-from freqtrade.optimize import setup_configuration, start_backtesting
 from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.state import RunMode
 from freqtrade.strategy.default_strategy import DefaultStrategy
@@ -103,14 +103,12 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None:
     min_date, max_date = get_timerange(processed)
     assert isinstance(processed, dict)
     results = backtesting.backtest(
-        {
+        processed=processed,
-            'stake_amount': config['stake_amount'],
+        stake_amount=config['stake_amount'],
-            'processed': processed,
+        start_date=min_date,
-            'max_open_trades': 1,
+        end_date=max_date,
-            'position_stacking': False,
+        max_open_trades=1,
-            'start_date': min_date,
+        position_stacking=False,
-            'end_date': max_date,
-        }
     )
     # results :: <class 'pandas.core.frame.DataFrame'>
     assert len(results) == num_results
@@ -132,7 +130,7 @@ def _load_pair_as_ticks(pair, tickfreq):
 
 
 # FIX: fixturize this?
-def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=None):
+def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'):
     data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair])
     data = trim_dictlist(data, -201)
     patch_exchange(mocker)
@@ -140,13 +138,12 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
     min_date, max_date = get_timerange(processed)
     return {
-        'stake_amount': conf['stake_amount'],
         'processed': processed,
-        'max_open_trades': 10,
+        'stake_amount': conf['stake_amount'],
-        'position_stacking': False,
-        'record': record,
         'start_date': min_date,
         'end_date': max_date,
+        'max_open_trades': 10,
+        'position_stacking': False,
     }
 
 
@@ -180,7 +177,7 @@ def _trend_alternate(dataframe=None, metadata=None):
 
 
 # Unit tests
-def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None:
+def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
 
     args = [
@@ -189,7 +186,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
         '--strategy', 'DefaultStrategy',
     ]
 
-    config = setup_configuration(get_args(args), RunMode.BACKTEST)
+    config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST)
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -230,7 +227,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
         '--fee', '0',
     ]
 
-    config = setup_configuration(get_args(args), RunMode.BACKTEST)
+    config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST)
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -263,7 +260,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) ->
     assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog)
 
 
-def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None:
+def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None:
     default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
 
     patched_configuration_load_config_file(mocker, default_conf)
@@ -275,7 +272,7 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog
     ]
 
     with pytest.raises(DependencyException, match=r'.*stake amount.*'):
-        setup_configuration(get_args(args), RunMode.BACKTEST)
+        setup_optimize_configuration(get_args(args), RunMode.BACKTEST)
 
 
 def test_start(mocker, fee, default_conf, caplog) -> None:
@@ -422,14 +419,12 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None:
     data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
     min_date, max_date = get_timerange(data_processed)
     results = backtesting.backtest(
-        {
+        processed=data_processed,
-            'stake_amount': default_conf['stake_amount'],
+        stake_amount=default_conf['stake_amount'],
-            'processed': data_processed,
+        start_date=min_date,
-            'max_open_trades': 10,
+        end_date=max_date,
-            'position_stacking': False,
+        max_open_trades=10,
-            'start_date': min_date,
+        position_stacking=False,
-            'end_date': max_date,
-        }
     )
     assert not results.empty
     assert len(results) == 2
@@ -478,14 +473,12 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) -
     processed = backtesting.strategy.tickerdata_to_dataframe(data)
     min_date, max_date = get_timerange(processed)
     results = backtesting.backtest(
-        {
+        processed=processed,
-            'stake_amount': default_conf['stake_amount'],
+        stake_amount=default_conf['stake_amount'],
-            'processed': processed,
+        start_date=min_date,
-            'max_open_trades': 1,
+        end_date=max_date,
-            'position_stacking': False,
+        max_open_trades=1,
-            'start_date': min_date,
+        position_stacking=False,
-            'end_date': max_date,
-        }
     )
     assert not results.empty
     assert len(results) == 1
@@ -525,7 +518,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir):
     backtesting = Backtesting(default_conf)
     backtesting.strategy.advise_buy = fun  # Override
     backtesting.strategy.advise_sell = fun  # Override
-    results = backtesting.backtest(backtest_conf)
+    results = backtesting.backtest(**backtest_conf)
     assert results.empty
 
 
@@ -540,7 +533,7 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir):
     backtesting = Backtesting(default_conf)
     backtesting.strategy.advise_buy = fun  # Override
     backtesting.strategy.advise_sell = fun  # Override
-    results = backtesting.backtest(backtest_conf)
+    results = backtesting.backtest(**backtest_conf)
     assert results.empty
 
 
@@ -553,7 +546,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
     backtesting = Backtesting(default_conf)
     backtesting.strategy.advise_buy = _trend_alternate  # Override
     backtesting.strategy.advise_sell = _trend_alternate  # Override
-    results = backtesting.backtest(backtest_conf)
+    results = backtesting.backtest(**backtest_conf)
     backtesting._store_backtest_result("test_.json", results)
     # 200 candles in backtest data
     # won't buy on first (shifted by 1)
@@ -598,15 +591,15 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
     data_processed = backtesting.strategy.tickerdata_to_dataframe(data)
     min_date, max_date = get_timerange(data_processed)
     backtest_conf = {
-        'stake_amount': default_conf['stake_amount'],
         'processed': data_processed,
-        'max_open_trades': 3,
+        'stake_amount': default_conf['stake_amount'],
-        'position_stacking': False,
         'start_date': min_date,
         'end_date': max_date,
+        'max_open_trades': 3,
+        'position_stacking': False,
     }
 
-    results = backtesting.backtest(backtest_conf)
+    results = backtesting.backtest(**backtest_conf)
 
     # Make sure we have parallel trades
     assert len(evaluate_result_multi(results, '5m', 2)) > 0
@@ -614,14 +607,14 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
     assert len(evaluate_result_multi(results, '5m', 3)) == 0
 
     backtest_conf = {
-        'stake_amount': default_conf['stake_amount'],
         'processed': data_processed,
-        'max_open_trades': 1,
+        'stake_amount': default_conf['stake_amount'],
-        'position_stacking': False,
         'start_date': min_date,
         'end_date': max_date,
+        'max_open_trades': 1,
+        'position_stacking': False,
     }
-    results = backtesting.backtest(backtest_conf)
+    results = backtesting.backtest(**backtest_conf)
     assert len(evaluate_result_multi(results, '5m', 1)) == 0
 
 

tests/optimize/test_edge_cli.py

@@ -3,14 +3,14 @@
 
 from unittest.mock import MagicMock
 
-from freqtrade.optimize import setup_configuration, start_edge
+from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
 from freqtrade.optimize.edge_cli import EdgeCli
 from freqtrade.state import RunMode
 from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
 
 
-def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None:
+def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
 
     args = [
@@ -19,7 +19,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
         '--strategy', 'DefaultStrategy',
     ]
 
-    config = setup_configuration(get_args(args), RunMode.EDGE)
+    config = setup_optimize_configuration(get_args(args), RunMode.EDGE)
     assert config['runmode'] == RunMode.EDGE
 
     assert 'max_open_trades' in config
@@ -53,7 +53,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N
         '--stoplosses=-0.01,-0.10,-0.001'
     ]
 
-    config = setup_configuration(get_args(args), RunMode.EDGE)
+    config = setup_optimize_configuration(get_args(args), RunMode.EDGE)
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config

tests/optimize/test_hyperopt.py

@@ -9,10 +9,10 @@ import pytest
 from arrow import Arrow
 from filelock import Timeout
 
-from freqtrade.exceptions import OperationalException
+from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.data.history import load_tickerdata_file
-from freqtrade.optimize import setup_configuration, start_hyperopt
+from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.default_hyperopt import DefaultHyperOpt
 from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss
 from freqtrade.optimize.hyperopt import Hyperopt
@@ -77,7 +77,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca
         '--hyperopt', 'DefaultHyperOpt',
     ]
 
-    config = setup_configuration(get_args(args), RunMode.HYPEROPT)
+    config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT)
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config
@@ -117,7 +117,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo
         '--print-all'
     ]
 
-    config = setup_configuration(get_args(args), RunMode.HYPEROPT)
+    config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT)
     assert 'max_open_trades' in config
     assert 'stake_currency' in config
     assert 'stake_amount' in config

tests/test_arguments.py

@@ -5,8 +5,8 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.configuration import Arguments
+from freqtrade.commands import Arguments
-from freqtrade.configuration.cli_options import check_int_positive
+from freqtrade.commands.cli_options import check_int_positive
 
 
 # Parse common command-line-arguments. Used for all tools

tests/test_configuration.py

@@ -10,7 +10,8 @@ from unittest.mock import MagicMock
 import pytest
 from jsonschema import ValidationError
 
-from freqtrade.configuration import (Arguments, Configuration, check_exchange,
+from freqtrade.commands import Arguments
+from freqtrade.configuration import (Configuration, check_exchange,
                                      remove_credentials,
                                      validate_config_consistency)
 from freqtrade.configuration.config_validation import validate_config_schema

tests/test_main.py

@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock
 
 import pytest
 
-from freqtrade.configuration import Arguments
+from freqtrade.commands import Arguments
 from freqtrade.exceptions import OperationalException, FreqtradeException
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.main import main
@@ -26,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None:
     Test that main() can start backtesting and also ensure we can pass some specific arguments
     further argument parsing is done in test_arguments.py
     """
-    backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock())
+    backtesting_mock = mocker.patch('freqtrade.commands.start_backtesting')
     backtesting_mock.__name__ = PropertyMock("start_backtesting")
     # it's sys.exit(0) at the end of backtesting
     with pytest.raises(SystemExit):
@@ -42,7 +42,7 @@ def test_parse_args_backtesting(mocker) -> None:
 
 
 def test_main_start_hyperopt(mocker) -> None:
-    hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock())
+    hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock())
     hyperopt_mock.__name__ = PropertyMock("start_hyperopt")
     # it's sys.exit(0) at the end of hyperopt
     with pytest.raises(SystemExit):

tests/test_plotting.py

@@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data import history
 from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
 from freqtrade.exceptions import OperationalException
-from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit
+from freqtrade.commands import start_plot_dataframe, start_plot_profit
 from freqtrade.plot.plotting import (add_indicators, add_profit,
                                      create_plotconfig,
                                      generate_candlestick_graph,