Merge branch 'develop' into data_handler

freqtrade/commands/__init__.py

@@ -0,0 +1,26 @@
+# flake8: noqa: F401
+"""
+Commands module.
+Contains all start-commands, subcommands and CLI Interface creation.
+
+Note: Be careful with file-scoped imports in these subfiles.
+    as they are parsed on startup, nothing containing optional modules should be loaded.
+"""
+from freqtrade.commands.arguments import Arguments
+from freqtrade.commands.data_commands import (start_convert_data,
+                                              start_download_data)
+from freqtrade.commands.deploy_commands import (start_create_userdir,
+                                                start_new_hyperopt,
+                                                start_new_strategy)
+from freqtrade.commands.hyperopt_commands import (start_hyperopt_list,
+                                                  start_hyperopt_show)
+from freqtrade.commands.list_commands import (start_list_exchanges,
+                                              start_list_markets,
+                                              start_list_strategies,
+                                              start_list_timeframes)
+from freqtrade.commands.optimize_commands import (start_backtesting,
+                                                  start_edge, start_hyperopt)
+from freqtrade.commands.pairlist_commands import start_test_pairlist
+from freqtrade.commands.plot_commands import (start_plot_dataframe,
+                                              start_plot_profit)
+from freqtrade.commands.trade_commands import start_trading

freqtrade/commands/arguments.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from freqtrade import constants
-from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS
+from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS
 
 ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"]
 
@@ -134,14 +134,15 @@ class Arguments:
         self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
         self._build_args(optionlist=['version'], parser=self.parser)
 
-        from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge
-        from freqtrade.utils import (start_create_userdir, start_convert_data, start_download_data,
-                                     start_hyperopt_list, start_hyperopt_show,
-                                     start_list_exchanges, start_list_markets,
-                                     start_list_strategies, start_new_hyperopt,
-                                     start_new_strategy, start_list_timeframes,
-                                     start_test_pairlist, start_trading)
-        from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit
+        from freqtrade.commands import (start_create_userdir, start_convert_data,
+                                        start_download_data,
+                                        start_hyperopt_list, start_hyperopt_show,
+                                        start_list_exchanges, start_list_markets,
+                                        start_list_strategies, start_new_hyperopt,
+                                        start_new_strategy, start_list_timeframes,
+                                        start_plot_dataframe, start_plot_profit,
+                                        start_backtesting, start_hyperopt, start_edge,
+                                        start_test_pairlist, start_trading)
 
         subparsers = self.parser.add_subparsers(dest='command',
                                                 # Use custom message when no subhandler is added

freqtrade/commands/cli_options.py

@@ -1,7 +1,7 @@
 """
 Definition of cli arguments used in arguments.py
 """
-import argparse
+from argparse import ArgumentTypeError
 
 from freqtrade import __version__, constants
 
@@ -12,7 +12,7 @@ def check_int_positive(value: str) -> int:
         if uint <= 0:
             raise ValueError
     except ValueError:
-        raise argparse.ArgumentTypeError(
+        raise ArgumentTypeError(
             f"{value} is invalid for this parameter, should be a positive integer value"
         )
     return uint
@@ -24,7 +24,7 @@ def check_int_nonzero(value: str) -> int:
         if uint == 0:
             raise ValueError
     except ValueError:
-        raise argparse.ArgumentTypeError(
+        raise ArgumentTypeError(
             f"{value} is invalid for this parameter, should be a non-zero integer value"
         )
     return uint

freqtrade/commands/data_commands.py

@@ -0,0 +1,85 @@
+import logging
+import sys
+from typing import Any, Dict, List
+
+import arrow
+
+from freqtrade.configuration import TimeRange, setup_utils_configuration
+from freqtrade.data.converter import (convert_ohlcv_format,
+                                      convert_trades_format)
+from freqtrade.data.history import (convert_trades_to_ohlcv,
+                                    refresh_backtest_ohlcv_data,
+                                    refresh_backtest_trades_data)
+from freqtrade.exceptions import OperationalException
+from freqtrade.resolvers import ExchangeResolver
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_download_data(args: Dict[str, Any]) -> None:
+    """
+    Download data (former download_backtest_data.py script)
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+    logger.info(f'About to download pairs: {config["pairs"]}, '
+                f'intervals: {config["timeframes"]} to {config["datadir"]}')
+
+    pairs_not_available: List[str] = []
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
+    try:
+
+        if config.get('download_trades'):
+            pairs_not_available = refresh_backtest_trades_data(
+                exchange, pairs=config["pairs"], datadir=config['datadir'],
+                timerange=timerange, erase=config.get("erase"),
+                data_format=config['dataformat_trades'])
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"),
+                data_format_ohlcv=config['dataformat_ohlcv'],
+                data_format_trades=config['dataformat_trades'],
+                )
+        else:
+            pairs_not_available = refresh_backtest_ohlcv_data(
+                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
+                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"),
+                data_format=config['dataformat_ohlcv'])
+
+    except KeyboardInterrupt:
+        sys.exit("SIGINT received, aborting ...")
+
+    finally:
+        if pairs_not_available:
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {exchange.name}.")
+
+
+def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
+    """
+    Convert data from one format to another
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+    if ohlcv:
+        convert_ohlcv_format(config,
+                             convert_from=args['format_from'], convert_to=args['format_to'],
+                             erase=args['erase'])
+    else:
+        convert_trades_format(config,
+                              convert_from=args['format_from'], convert_to=args['format_to'],
+                              erase=args['erase'])

freqtrade/commands/deploy_commands.py

@@ -0,0 +1,112 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.configuration.directory_operations import (copy_sample_files,
+                                                          create_userdata_dir)
+from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
+from freqtrade.exceptions import OperationalException
+from freqtrade.misc import render_template
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_create_userdir(args: Dict[str, Any]) -> None:
+    """
+    Create "user_data" directory to contain user data strategies, hyperopt, ...)
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    if "user_data_dir" in args and args["user_data_dir"]:
+        userdir = create_userdata_dir(args["user_data_dir"], create_dir=True)
+        copy_sample_files(userdir, overwrite=args["reset"])
+    else:
+        logger.warning("`create-userdir` requires --userdir to be set.")
+        sys.exit(1)
+
+
+def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str):
+    """
+    Deploy new strategy from template to strategy_path
+    """
+    indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",)
+    buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",)
+    sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",)
+    plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",)
+
+    strategy_text = render_template(templatefile='base_strategy.py.j2',
+                                    arguments={"strategy": strategy_name,
+                                               "indicators": indicators,
+                                               "buy_trend": buy_trend,
+                                               "sell_trend": sell_trend,
+                                               "plot_config": plot_config,
+                                               })
+
+    logger.info(f"Writing strategy to `{strategy_path}`.")
+    strategy_path.write_text(strategy_text)
+
+
+def start_new_strategy(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    if "strategy" in args and args["strategy"]:
+        if args["strategy"] == "DefaultStrategy":
+            raise OperationalException("DefaultStrategy is not allowed as name.")
+
+        new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py")
+
+        if new_path.exists():
+            raise OperationalException(f"`{new_path}` already exists. "
+                                       "Please choose another Strategy Name.")
+
+        deploy_new_strategy(args['strategy'], new_path, args['template'])
+
+    else:
+        raise OperationalException("`new-strategy` requires --strategy to be set.")
+
+
+def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str):
+    """
+    Deploys a new hyperopt template to hyperopt_path
+    """
+    buy_guards = render_template(
+        templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",)
+    sell_guards = render_template(
+        templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",)
+    buy_space = render_template(
+        templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",)
+    sell_space = render_template(
+        templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",)
+
+    strategy_text = render_template(templatefile='base_hyperopt.py.j2',
+                                    arguments={"hyperopt": hyperopt_name,
+                                               "buy_guards": buy_guards,
+                                               "sell_guards": sell_guards,
+                                               "buy_space": buy_space,
+                                               "sell_space": sell_space,
+                                               })
+
+    logger.info(f"Writing hyperopt to `{hyperopt_path}`.")
+    hyperopt_path.write_text(strategy_text)
+
+
+def start_new_hyperopt(args: Dict[str, Any]) -> None:
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    if "hyperopt" in args and args["hyperopt"]:
+        if args["hyperopt"] == "DefaultHyperopt":
+            raise OperationalException("DefaultHyperopt is not allowed as name.")
+
+        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py")
+
+        if new_path.exists():
+            raise OperationalException(f"`{new_path}` already exists. "
+                                       "Please choose another Strategy Name.")
+        deploy_new_hyperopt(args['hyperopt'], new_path, args['template'])
+    else:
+        raise OperationalException("`new-hyperopt` requires --hyperopt to be set.")

freqtrade/commands/hyperopt_commands.py

@@ -0,0 +1,114 @@
+import logging
+from operator import itemgetter
+from typing import Any, Dict, List
+
+from colorama import init as colorama_init
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.exceptions import OperationalException
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_hyperopt_list(args: Dict[str, Any]) -> None:
+    """
+    List hyperopt epochs previously evaluated
+    """
+    from freqtrade.optimize.hyperopt import Hyperopt
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    only_best = config.get('hyperopt_list_best', False)
+    only_profitable = config.get('hyperopt_list_profitable', False)
+    print_colorized = config.get('print_colorized', False)
+    print_json = config.get('print_json', False)
+    no_details = config.get('hyperopt_list_no_details', False)
+    no_header = False
+
+    trials_file = (config['user_data_dir'] /
+                   'hyperopt_results' / 'hyperopt_results.pickle')
+
+    # Previous evaluations
+    trials = Hyperopt.load_previous_results(trials_file)
+    total_epochs = len(trials)
+
+    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
+
+    # TODO: fetch the interval for epochs to print from the cli option
+    epoch_start, epoch_stop = 0, None
+
+    if print_colorized:
+        colorama_init(autoreset=True)
+
+    try:
+        # Human-friendly indexes used here (starting from 1)
+        for val in trials[epoch_start:epoch_stop]:
+            Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized)
+
+    except KeyboardInterrupt:
+        print('User interrupted..')
+
+    if trials and not no_details:
+        sorted_trials = sorted(trials, key=itemgetter('loss'))
+        results = sorted_trials[0]
+        Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
+
+
+def start_hyperopt_show(args: Dict[str, Any]) -> None:
+    """
+    Show details of a hyperopt epoch previously evaluated
+    """
+    from freqtrade.optimize.hyperopt import Hyperopt
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    only_best = config.get('hyperopt_list_best', False)
+    only_profitable = config.get('hyperopt_list_profitable', False)
+    no_header = config.get('hyperopt_show_no_header', False)
+
+    trials_file = (config['user_data_dir'] /
+                   'hyperopt_results' / 'hyperopt_results.pickle')
+
+    # Previous evaluations
+    trials = Hyperopt.load_previous_results(trials_file)
+    total_epochs = len(trials)
+
+    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
+    trials_epochs = len(trials)
+
+    n = config.get('hyperopt_show_index', -1)
+    if n > trials_epochs:
+        raise OperationalException(
+                f"The index of the epoch to show should be less than {trials_epochs + 1}.")
+    if n < -trials_epochs:
+        raise OperationalException(
+                f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
+
+    # Translate epoch index from human-readable format to pythonic
+    if n > 0:
+        n -= 1
+
+    print_json = config.get('print_json', False)
+
+    if trials:
+        val = trials[n]
+        Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
+                                     header_str="Epoch details")
+
+
+def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List:
+    """
+    Filter our items from the list of hyperopt results
+    """
+    if only_best:
+        trials = [x for x in trials if x['is_best']]
+    if only_profitable:
+        trials = [x for x in trials if x['results_metrics']['profit'] > 0]
+
+    logger.info(f"{len(trials)} " +
+                ("best " if only_best else "") +
+                ("profitable " if only_profitable else "") +
+                "epochs found.")
+
+    return trials

freqtrade/commands/list_commands.py

@@ -0,0 +1,156 @@
+import csv
+import logging
+import sys
+from collections import OrderedDict
+from pathlib import Path
+from typing import Any, Dict
+
+import rapidjson
+from tabulate import tabulate
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.constants import USERPATH_STRATEGY
+from freqtrade.exceptions import OperationalException
+from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
+                                market_is_active, symbol_is_pair)
+from freqtrade.misc import plural
+from freqtrade.resolvers import ExchangeResolver, StrategyResolver
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_list_exchanges(args: Dict[str, Any]) -> None:
+    """
+    Print available exchanges
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
+    if args['print_one_column']:
+        print('\n'.join(exchanges))
+    else:
+        if args['list_exchanges_all']:
+            print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
+        else:
+            print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
+
+
+def start_list_strategies(args: Dict[str, Any]) -> None:
+    """
+    Print Strategies available in a directory
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY))
+    strategies = StrategyResolver.search_all_objects(directory)
+    # Sort alphabetically
+    strategies = sorted(strategies, key=lambda x: x['name'])
+    strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies]
+
+    if args['print_one_column']:
+        print('\n'.join([s['name'] for s in strategies]))
+    else:
+        print(tabulate(strats_to_print, headers='keys', tablefmt='pipe'))
+
+
+def start_list_timeframes(args: Dict[str, Any]) -> None:
+    """
+    Print ticker intervals (timeframes) available on Exchange
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+    # Do not use ticker_interval set in the config
+    config['ticker_interval'] = None
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    if args['print_one_column']:
+        print('\n'.join(exchange.timeframes))
+    else:
+        print(f"Timeframes available for the exchange `{exchange.name}`: "
+              f"{', '.join(exchange.timeframes)}")
+
+
+def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
+    """
+    Print pairs/markets on the exchange
+    :param args: Cli args from Arguments()
+    :param pairs_only: if True print only pairs, otherwise print all instruments (markets)
+    :return: None
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    # By default only active pairs/markets are to be shown
+    active_only = not args.get('list_pairs_all', False)
+
+    base_currencies = args.get('base_currencies', [])
+    quote_currencies = args.get('quote_currencies', [])
+
+    try:
+        pairs = exchange.get_markets(base_currencies=base_currencies,
+                                     quote_currencies=quote_currencies,
+                                     pairs_only=pairs_only,
+                                     active_only=active_only)
+        # Sort the pairs/markets by symbol
+        pairs = OrderedDict(sorted(pairs.items()))
+    except Exception as e:
+        raise OperationalException(f"Cannot get markets. Reason: {e}") from e
+
+    else:
+        summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") +
+                       ("active " if active_only else "") +
+                       (plural(len(pairs), "pair" if pairs_only else "market")) +
+                       (f" with {', '.join(base_currencies)} as base "
+                        f"{plural(len(base_currencies), 'currency', 'currencies')}"
+                        if base_currencies else "") +
+                       (" and" if base_currencies and quote_currencies else "") +
+                       (f" with {', '.join(quote_currencies)} as quote "
+                        f"{plural(len(quote_currencies), 'currency', 'currencies')}"
+                        if quote_currencies else ""))
+
+        headers = ["Id", "Symbol", "Base", "Quote", "Active",
+                   *(['Is pair'] if not pairs_only else [])]
+
+        tabular_data = []
+        for _, v in pairs.items():
+            tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
+                                 'Base': v['base'], 'Quote': v['quote'],
+                                 'Active': market_is_active(v),
+                                 **({'Is pair': symbol_is_pair(v['symbol'])}
+                                    if not pairs_only else {})})
+
+        if (args.get('print_one_column', False) or
+                args.get('list_pairs_print_json', False) or
+                args.get('print_csv', False)):
+            # Print summary string in the log in case of machine-readable
+            # regular formats.
+            logger.info(f"{summary_str}.")
+        else:
+            # Print empty string separating leading logs and output in case of
+            # human-readable formats.
+            print()
+
+        if len(pairs):
+            if args.get('print_list', False):
+                # print data as a list, with human-readable summary
+                print(f"{summary_str}: {', '.join(pairs.keys())}.")
+            elif args.get('print_one_column', False):
+                print('\n'.join(pairs.keys()))
+            elif args.get('list_pairs_print_json', False):
+                print(rapidjson.dumps(list(pairs.keys()), default=str))
+            elif args.get('print_csv', False):
+                writer = csv.DictWriter(sys.stdout, fieldnames=headers)
+                writer.writeheader()
+                writer.writerows(tabular_data)
+            else:
+                # print data as a table, with the human-readable summary
+                print(f"{summary_str}:")
+                print(tabulate(tabular_data, headers='keys', tablefmt='pipe'))
+        elif not (args.get('print_one_column', False) or
+                  args.get('list_pairs_print_json', False) or
+                  args.get('print_csv', False)):
+            print(f"{summary_str}.")

freqtrade/commands/optimize_commands.py

@@ -0,0 +1,102 @@
+import logging
+from typing import Any, Dict
+
+from freqtrade import constants
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.exceptions import DependencyException, OperationalException
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
+    """
+    Prepare the configuration for the Hyperopt module
+    :param args: Cli args from Arguments()
+    :return: Configuration
+    """
+    config = setup_utils_configuration(args, method)
+
+    if method == RunMode.BACKTEST:
+        if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
+            raise DependencyException('stake amount could not be "%s" for backtesting' %
+                                      constants.UNLIMITED_STAKE_AMOUNT)
+
+    return config
+
+
+def start_backtesting(args: Dict[str, Any]) -> None:
+    """
+    Start Backtesting script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    # Import here to avoid loading backtesting module when it's not used
+    from freqtrade.optimize.backtesting import Backtesting
+
+    # Initialize configuration
+    config = setup_optimize_configuration(args, RunMode.BACKTEST)
+
+    logger.info('Starting freqtrade in Backtesting mode')
+
+    # Initialize backtesting object
+    backtesting = Backtesting(config)
+    backtesting.start()
+
+
+def start_hyperopt(args: Dict[str, Any]) -> None:
+    """
+    Start hyperopt script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    # Import here to avoid loading hyperopt module when it's not used
+    try:
+        from filelock import FileLock, Timeout
+        from freqtrade.optimize.hyperopt import Hyperopt
+    except ImportError as e:
+        raise OperationalException(
+            f"{e}. Please ensure that the hyperopt dependencies are installed.") from e
+    # Initialize configuration
+    config = setup_optimize_configuration(args, RunMode.HYPEROPT)
+
+    logger.info('Starting freqtrade in Hyperopt mode')
+
+    lock = FileLock(Hyperopt.get_lock_filename(config))
+
+    try:
+        with lock.acquire(timeout=1):
+
+            # Remove noisy log messages
+            logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
+            logging.getLogger('filelock').setLevel(logging.WARNING)
+
+            # Initialize backtesting object
+            hyperopt = Hyperopt(config)
+            hyperopt.start()
+
+    except Timeout:
+        logger.info("Another running instance of freqtrade Hyperopt detected.")
+        logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. "
+                    "Hyperopt module is resource hungry. Please run your Hyperopt sequentially "
+                    "or on separate machines.")
+        logger.info("Quitting now.")
+        # TODO: return False here in order to help freqtrade to exit
+        # with non-zero exit code...
+        # Same in Edge and Backtesting start() functions.
+
+
+def start_edge(args: Dict[str, Any]) -> None:
+    """
+    Start Edge script
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    from freqtrade.optimize.edge_cli import EdgeCli
+    # Initialize configuration
+    config = setup_optimize_configuration(args, RunMode.EDGE)
+    logger.info('Starting freqtrade in Edge mode')
+
+    # Initialize Edge object
+    edge_cli = EdgeCli(config)
+    edge_cli.start()

freqtrade/commands/pairlist_commands.py

@@ -0,0 +1,42 @@
+import logging
+from typing import Any, Dict
+
+import rapidjson
+
+from freqtrade.configuration import setup_utils_configuration
+from freqtrade.resolvers import ExchangeResolver
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def start_test_pairlist(args: Dict[str, Any]) -> None:
+    """
+    Test Pairlist configuration
+    """
+    from freqtrade.pairlist.pairlistmanager import PairListManager
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+
+    quote_currencies = args.get('quote_currencies')
+    if not quote_currencies:
+        quote_currencies = [config.get('stake_currency')]
+    results = {}
+    for curr in quote_currencies:
+        config['stake_currency'] = curr
+        # Do not use ticker_interval set in the config
+        pairlists = PairListManager(exchange, config)
+        pairlists.refresh_pairlist()
+        results[curr] = pairlists.whitelist
+
+    for curr, pairlist in results.items():
+        if not args.get('print_one_column', False):
+            print(f"Pairs for {curr}: ")
+
+        if args.get('print_one_column', False):
+            print('\n'.join(pairlist))
+        elif args.get('list_pairs_print_json', False):
+            print(rapidjson.dumps(list(pairlist), default=str))
+        else:
+            print(pairlist)

freqtrade/commands/plot_commands.py

@@ -1,8 +1,8 @@
 from typing import Any, Dict
 
+from freqtrade.configuration import setup_utils_configuration
 from freqtrade.exceptions import OperationalException
 from freqtrade.state import RunMode
-from freqtrade.utils import setup_utils_configuration
 
 
 def validate_plot_args(args: Dict[str, Any]):

freqtrade/commands/trade_commands.py

@@ -0,0 +1,25 @@
+import logging
+
+from typing import Any, Dict
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_trading(args: Dict[str, Any]) -> int:
+    """
+    Main entry point for trading mode
+    """
+    from freqtrade.worker import Worker
+    # Load and run worker
+    worker = None
+    try:
+        worker = Worker(args)
+        worker.run()
+    except KeyboardInterrupt:
+        logger.info('SIGINT received, aborting ...')
+    finally:
+        if worker:
+            logger.info("worker found ... calling exit")
+            worker.exit()
+    return 0

freqtrade/configuration/__init__.py

@@ -1,5 +1,7 @@
-from freqtrade.configuration.arguments import Arguments  # noqa: F401
-from freqtrade.configuration.check_exchange import check_exchange, remove_credentials  # noqa: F401
-from freqtrade.configuration.timerange import TimeRange  # noqa: F401
-from freqtrade.configuration.configuration import Configuration  # noqa: F401
-from freqtrade.configuration.config_validation import validate_config_consistency  # noqa: F401
+# flake8: noqa: F401
+
+from freqtrade.configuration.config_setup import setup_utils_configuration
+from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
+from freqtrade.configuration.timerange import TimeRange
+from freqtrade.configuration.configuration import Configuration
+from freqtrade.configuration.config_validation import validate_config_consistency

freqtrade/configuration/config_setup.py

@@ -0,0 +1,25 @@
+import logging
+from typing import Any, Dict
+
+from .config_validation import validate_config_consistency
+from .configuration import Configuration
+from .check_exchange import remove_credentials
+from freqtrade.state import RunMode
+
+logger = logging.getLogger(__name__)
+
+
+def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
+    """
+    Prepare the configuration for utils subcommands
+    :param args: Cli args from Arguments()
+    :return: Configuration
+    """
+    configuration = Configuration(args, method)
+    config = configuration.get_config()
+
+    # Ensure we do not use Exchange credentials
+    remove_credentials(config)
+    validate_config_consistency(config)
+
+    return config

freqtrade/edge/__init__.py

@@ -1,465 +1 @@
-# pragma pylint: disable=W0603
-""" Edge positioning package """
-import logging
-from typing import Any, Dict, NamedTuple
-
-import arrow
-import numpy as np
-import utils_find_1st as utf1st
-from pandas import DataFrame
-
-from freqtrade import constants
-from freqtrade.configuration import TimeRange
-from freqtrade.data import history
-from freqtrade.exceptions import OperationalException
-from freqtrade.strategy.interface import SellType
-
-logger = logging.getLogger(__name__)
-
-
-class PairInfo(NamedTuple):
-    stoploss: float
-    winrate: float
-    risk_reward_ratio: float
-    required_risk_reward: float
-    expectancy: float
-    nb_trades: int
-    avg_trade_duration: float
-
-
-class Edge:
-    """
-    Calculates Win Rate, Risk Reward Ratio, Expectancy
-    against historical data for a give set of markets and a strategy
-    it then adjusts stoploss and position size accordingly
-    and force it into the strategy
-    Author: https://github.com/mishaker
-    """
-
-    config: Dict = {}
-    _cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
-
-    def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:
-
-        self.config = config
-        self.exchange = exchange
-        self.strategy = strategy
-
-        self.edge_config = self.config.get('edge', {})
-        self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
-        self._final_pairs: list = []
-
-        # checking max_open_trades. it should be -1 as with Edge
-        # the number of trades is determined by position size
-        if self.config['max_open_trades'] != float('inf'):
-            logger.critical('max_open_trades should be -1 in config !')
-
-        if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT:
-            raise OperationalException('Edge works only with unlimited stake amount')
-
-        # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future.
-        self._capital_percentage: float = self.edge_config.get(
-            'capital_available_percentage', self.config['tradable_balance_ratio'])
-        self._allowed_risk: float = self.edge_config.get('allowed_risk')
-        self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14)
-        self._last_updated: int = 0  # Timestamp of pairs last updated time
-        self._refresh_pairs = True
-
-        self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01))
-        self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05))
-        self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001))
-
-        # calculating stoploss range
-        self._stoploss_range = np.arange(
-            self._stoploss_range_min,
-            self._stoploss_range_max,
-            self._stoploss_range_step
-        )
-
-        self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
-            days=-1 * self._since_number_of_days).format('YYYYMMDD'))
-        if config.get('fee'):
-            self.fee = config['fee']
-        else:
-            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
-
-    def calculate(self) -> bool:
-        pairs = self.config['exchange']['pair_whitelist']
-        heartbeat = self.edge_config.get('process_throttle_secs')
-
-        if (self._last_updated > 0) and (
-                self._last_updated + heartbeat > arrow.utcnow().timestamp):
-            return False
-
-        data: Dict[str, Any] = {}
-        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
-        logger.info('Using local backtesting data (using whitelist in given config) ...')
-
-        if self._refresh_pairs:
-            history.refresh_data(
-                datadir=self.config['datadir'],
-                pairs=pairs,
-                exchange=self.exchange,
-                timeframe=self.strategy.ticker_interval,
-                timerange=self._timerange,
-            )
-
-        data = history.load_data(
-            datadir=self.config['datadir'],
-            pairs=pairs,
-            timeframe=self.strategy.ticker_interval,
-            timerange=self._timerange,
-            startup_candles=self.strategy.startup_candle_count,
-            data_format=self.config.get('dataformat_ohlcv', 'json'),
-        )
-
-        if not data:
-            # Reinitializing cached pairs
-            self._cached_pairs = {}
-            logger.critical("No data found. Edge is stopped ...")
-            return False
-
-        preprocessed = self.strategy.tickerdata_to_dataframe(data)
-
-        # Print timeframe
-        min_date, max_date = history.get_timerange(preprocessed)
-        logger.info(
-            'Measuring data from %s up to %s (%s days) ...',
-            min_date.isoformat(),
-            max_date.isoformat(),
-            (max_date - min_date).days
-        )
-        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
-
-        trades: list = []
-        for pair, pair_data in preprocessed.items():
-            # Sorting dataframe by date and reset index
-            pair_data = pair_data.sort_values(by=['date'])
-            pair_data = pair_data.reset_index(drop=True)
-
-            ticker_data = self.strategy.advise_sell(
-                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
-
-            trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range)
-
-        # If no trade found then exit
-        if len(trades) == 0:
-            logger.info("No trades found.")
-            return False
-
-        # Fill missing, calculable columns, profit, duration , abs etc.
-        trades_df = self._fill_calculable_fields(DataFrame(trades))
-        self._cached_pairs = self._process_expectancy(trades_df)
-        self._last_updated = arrow.utcnow().timestamp
-
-        return True
-
-    def stake_amount(self, pair: str, free_capital: float,
-                     total_capital: float, capital_in_trade: float) -> float:
-        stoploss = self.stoploss(pair)
-        available_capital = (total_capital + capital_in_trade) * self._capital_percentage
-        allowed_capital_at_risk = available_capital * self._allowed_risk
-        max_position_size = abs(allowed_capital_at_risk / stoploss)
-        position_size = min(max_position_size, free_capital)
-        if pair in self._cached_pairs:
-            logger.info(
-                'winrate: %s, expectancy: %s, position size: %s, pair: %s,'
-                ' capital in trade: %s, free capital: %s, total capital: %s,'
-                ' stoploss: %s, available capital: %s.',
-                self._cached_pairs[pair].winrate,
-                self._cached_pairs[pair].expectancy,
-                position_size, pair,
-                capital_in_trade, free_capital, total_capital,
-                stoploss, available_capital
-            )
-        return round(position_size, 15)
-
-    def stoploss(self, pair: str) -> float:
-        if pair in self._cached_pairs:
-            return self._cached_pairs[pair].stoploss
-        else:
-            logger.warning('tried to access stoploss of a non-existing pair, '
-                           'strategy stoploss is returned instead.')
-            return self.strategy.stoploss
-
-    def adjust(self, pairs) -> list:
-        """
-        Filters out and sorts "pairs" according to Edge calculated pairs
-        """
-        final = []
-        for pair, info in self._cached_pairs.items():
-            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
-                info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
-                    pair in pairs:
-                final.append(pair)
-
-        if self._final_pairs != final:
-            self._final_pairs = final
-            if self._final_pairs:
-                logger.info(
-                    'Minimum expectancy and minimum winrate are met only for %s,'
-                    ' so other pairs are filtered out.',
-                    self._final_pairs
-                    )
-            else:
-                logger.info(
-                    'Edge removed all pairs as no pair with minimum expectancy '
-                    'and minimum winrate was found !'
-                    )
-
-        return self._final_pairs
-
-    def accepted_pairs(self) -> list:
-        """
-        return a list of accepted pairs along with their winrate, expectancy and stoploss
-        """
-        final = []
-        for pair, info in self._cached_pairs.items():
-            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
-                 info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
-                final.append({
-                    'Pair': pair,
-                    'Winrate': info.winrate,
-                    'Expectancy': info.expectancy,
-                    'Stoploss': info.stoploss,
-                })
-        return final
-
-    def _fill_calculable_fields(self, result: DataFrame) -> DataFrame:
-        """
-        The result frame contains a number of columns that are calculable
-        from other columns. These are left blank till all rows are added,
-        to be populated in single vector calls.
-
-        Columns to be populated are:
-        - Profit
-        - trade duration
-        - profit abs
-        :param result Dataframe
-        :return: result Dataframe
-        """
-
-        # stake and fees
-        # stake = 0.015
-        # 0.05% is 0.0005
-        # fee = 0.001
-
-        # we set stake amount to an arbitrary amount.
-        # as it doesn't change the calculation.
-        # all returned values are relative. they are percentages.
-        stake = 0.015
-        fee = self.fee
-        open_fee = fee / 2
-        close_fee = fee / 2
-
-        result['trade_duration'] = result['close_time'] - result['open_time']
-
-        result['trade_duration'] = result['trade_duration'].map(
-            lambda x: int(x.total_seconds() / 60))
-
-        # Spends, Takes, Profit, Absolute Profit
-
-        # Buy Price
-        result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
-        result['buy_fee'] = stake * open_fee
-        result['buy_spend'] = stake + result['buy_fee']  # How much we're spending
-
-        # Sell price
-        result['sell_sum'] = result['buy_vol'] * result['close_rate']
-        result['sell_fee'] = result['sell_sum'] * close_fee
-        result['sell_take'] = result['sell_sum'] - result['sell_fee']
-
-        # profit_percent
-        result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']
-
-        # Absolute profit
-        result['profit_abs'] = result['sell_take'] - result['buy_spend']
-
-        return result
-
-    def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
-        """
-        This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
-        The calulation will be done per pair and per strategy.
-        """
-        # Removing pairs having less than min_trades_number
-        min_trades_number = self.edge_config.get('min_trade_number', 10)
-        results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number)
-        ###################################
-
-        # Removing outliers (Only Pumps) from the dataset
-        # The method to detect outliers is to calculate standard deviation
-        # Then every value more than (standard deviation + 2*average) is out (pump)
-        #
-        # Removing Pumps
-        if self.edge_config.get('remove_pumps', False):
-            results = results.groupby(['pair', 'stoploss']).apply(
-                lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()])
-        ##########################################################################
-
-        # Removing trades having a duration more than X minutes (set in config)
-        max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440)
-        results = results[results.trade_duration < max_trade_duration]
-        #######################################################################
-
-        if results.empty:
-            return {}
-
-        groupby_aggregator = {
-            'profit_abs': [
-                ('nb_trades', 'count'),  # number of all trades
-                ('profit_sum', lambda x: x[x > 0].sum()),  # cumulative profit of all winning trades
-                ('loss_sum', lambda x: abs(x[x < 0].sum())),  # cumulative loss of all losing trades
-                ('nb_win_trades', lambda x: x[x > 0].count())  # number of winning trades
-            ],
-            'trade_duration': [('avg_trade_duration', 'mean')]
-        }
-
-        # Group by (pair and stoploss) by applying above aggregator
-        df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg(
-            groupby_aggregator).reset_index(col_level=1)
-
-        # Dropping level 0 as we don't need it
-        df.columns = df.columns.droplevel(0)
-
-        # Calculating number of losing trades, average win and average loss
-        df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']
-        df['average_win'] = df['profit_sum'] / df['nb_win_trades']
-        df['average_loss'] = df['loss_sum'] / df['nb_loss_trades']
-
-        # Win rate = number of profitable trades / number of trades
-        df['winrate'] = df['nb_win_trades'] / df['nb_trades']
-
-        # risk_reward_ratio = average win / average loss
-        df['risk_reward_ratio'] = df['average_win'] / df['average_loss']
-
-        # required_risk_reward = (1 / winrate) - 1
-        df['required_risk_reward'] = (1 / df['winrate']) - 1
-
-        # expectancy = (risk_reward_ratio * winrate) - (lossrate)
-        df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate'])
-
-        # sort by expectancy and stoploss
-        df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby(
-            'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index()
-
-        final = {}
-        for x in df.itertuples():
-            final[x.pair] = PairInfo(
-                x.stoploss,
-                x.winrate,
-                x.risk_reward_ratio,
-                x.required_risk_reward,
-                x.expectancy,
-                x.nb_trades,
-                x.avg_trade_duration
-            )
-
-        # Returning a list of pairs in order of "expectancy"
-        return final
-
-    def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range):
-        buy_column = ticker_data['buy'].values
-        sell_column = ticker_data['sell'].values
-        date_column = ticker_data['date'].values
-        ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values
-
-        result: list = []
-        for stoploss in stoploss_range:
-            result += self._detect_next_stop_or_sell_point(
-                buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair
-            )
-
-        return result
-
-    def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,
-                                        ohlc_columns, stoploss, pair):
-        """
-        Iterate through ohlc_columns in order to find the next trade
-        Next trade opens from the first buy signal noticed to
-        The sell or stoploss signal after it.
-        It then cuts OHLC, buy_column, sell_column and date_column.
-        Cut from (the exit trade index) + 1.
-
-        Author: https://github.com/mishaker
-        """
-
-        result: list = []
-        start_point = 0
-
-        while True:
-            open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal)
-
-            # Return empty if we don't find trade entry (i.e. buy==1) or
-            # we find a buy but at the end of array
-            if open_trade_index == -1 or open_trade_index == len(buy_column) - 1:
-                break
-            else:
-                # When a buy signal is seen,
-                # trade opens in reality on the next candle
-                open_trade_index += 1
-
-            stop_price_percentage = stoploss + 1
-            open_price = ohlc_columns[open_trade_index, 0]
-            stop_price = (open_price * stop_price_percentage)
-
-            # Searching for the index where stoploss is hit
-            stop_index = utf1st.find_1st(
-                ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller)
-
-            # If we don't find it then we assume stop_index will be far in future (infinite number)
-            if stop_index == -1:
-                stop_index = float('inf')
-
-            # Searching for the index where sell is hit
-            sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal)
-
-            # If we don't find it then we assume sell_index will be far in future (infinite number)
-            if sell_index == -1:
-                sell_index = float('inf')
-
-            # Check if we don't find any stop or sell point (in that case trade remains open)
-            # It is not interesting for Edge to consider it so we simply ignore the trade
-            # And stop iterating there is no more entry
-            if stop_index == sell_index == float('inf'):
-                break
-
-            if stop_index <= sell_index:
-                exit_index = open_trade_index + stop_index
-                exit_type = SellType.STOP_LOSS
-                exit_price = stop_price
-            elif stop_index > sell_index:
-                # If exit is SELL then we exit at the next candle
-                exit_index = open_trade_index + sell_index + 1
-
-                # Check if we have the next candle
-                if len(ohlc_columns) - 1 < exit_index:
-                    break
-
-                exit_type = SellType.SELL_SIGNAL
-                exit_price = ohlc_columns[exit_index, 0]
-
-            trade = {'pair': pair,
-                     'stoploss': stoploss,
-                     'profit_percent': '',
-                     'profit_abs': '',
-                     'open_time': date_column[open_trade_index],
-                     'close_time': date_column[exit_index],
-                     'open_index': start_point + open_trade_index,
-                     'close_index': start_point + exit_index,
-                     'trade_duration': '',
-                     'open_rate': round(open_price, 15),
-                     'close_rate': round(exit_price, 15),
-                     'exit_type': exit_type
-                     }
-
-            result.append(trade)
-
-            # Giving a view of exit_index till the end of array
-            buy_column = buy_column[exit_index:]
-            sell_column = sell_column[exit_index:]
-            date_column = date_column[exit_index:]
-            ohlc_columns = ohlc_columns[exit_index:]
-            start_point += exit_index
-
-        return result
+from .edge_positioning import Edge, PairInfo  # noqa: F401

freqtrade/edge/edge_positioning.py

@@ -0,0 +1,465 @@
+# pragma pylint: disable=W0603
+""" Edge positioning package """
+import logging
+from typing import Any, Dict, NamedTuple
+
+import arrow
+import numpy as np
+import utils_find_1st as utf1st
+from pandas import DataFrame
+
+from freqtrade import constants
+from freqtrade.configuration import TimeRange
+from freqtrade.data import history
+from freqtrade.exceptions import OperationalException
+from freqtrade.strategy.interface import SellType
+
+logger = logging.getLogger(__name__)
+
+
+class PairInfo(NamedTuple):
+    stoploss: float
+    winrate: float
+    risk_reward_ratio: float
+    required_risk_reward: float
+    expectancy: float
+    nb_trades: int
+    avg_trade_duration: float
+
+
+class Edge:
+    """
+    Calculates Win Rate, Risk Reward Ratio, Expectancy
+    against historical data for a give set of markets and a strategy
+    it then adjusts stoploss and position size accordingly
+    and force it into the strategy
+    Author: https://github.com/mishaker
+    """
+
+    config: Dict = {}
+    _cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
+
+    def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:
+
+        self.config = config
+        self.exchange = exchange
+        self.strategy = strategy
+
+        self.edge_config = self.config.get('edge', {})
+        self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
+        self._final_pairs: list = []
+
+        # checking max_open_trades. it should be -1 as with Edge
+        # the number of trades is determined by position size
+        if self.config['max_open_trades'] != float('inf'):
+            logger.critical('max_open_trades should be -1 in config !')
+
+        if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT:
+            raise OperationalException('Edge works only with unlimited stake amount')
+
+        # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future.
+        self._capital_percentage: float = self.edge_config.get(
+            'capital_available_percentage', self.config['tradable_balance_ratio'])
+        self._allowed_risk: float = self.edge_config.get('allowed_risk')
+        self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14)
+        self._last_updated: int = 0  # Timestamp of pairs last updated time
+        self._refresh_pairs = True
+
+        self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01))
+        self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05))
+        self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001))
+
+        # calculating stoploss range
+        self._stoploss_range = np.arange(
+            self._stoploss_range_min,
+            self._stoploss_range_max,
+            self._stoploss_range_step
+        )
+
+        self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
+            days=-1 * self._since_number_of_days).format('YYYYMMDD'))
+        if config.get('fee'):
+            self.fee = config['fee']
+        else:
+            self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
+
+    def calculate(self) -> bool:
+        pairs = self.config['exchange']['pair_whitelist']
+        heartbeat = self.edge_config.get('process_throttle_secs')
+
+        if (self._last_updated > 0) and (
+                self._last_updated + heartbeat > arrow.utcnow().timestamp):
+            return False
+
+        data: Dict[str, Any] = {}
+        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
+        logger.info('Using local backtesting data (using whitelist in given config) ...')
+
+        if self._refresh_pairs:
+            history.refresh_data(
+                datadir=self.config['datadir'],
+                pairs=pairs,
+                exchange=self.exchange,
+                timeframe=self.strategy.ticker_interval,
+                timerange=self._timerange,
+            )
+
+        data = history.load_data(
+            datadir=self.config['datadir'],
+            pairs=pairs,
+            timeframe=self.strategy.ticker_interval,
+            timerange=self._timerange,
+            startup_candles=self.strategy.startup_candle_count,
+            data_format=self.config.get('dataformat_ohlcv', 'json'),
+        )
+
+        if not data:
+            # Reinitializing cached pairs
+            self._cached_pairs = {}
+            logger.critical("No data found. Edge is stopped ...")
+            return False
+
+        preprocessed = self.strategy.tickerdata_to_dataframe(data)
+
+        # Print timeframe
+        min_date, max_date = history.get_timerange(preprocessed)
+        logger.info(
+            'Measuring data from %s up to %s (%s days) ...',
+            min_date.isoformat(),
+            max_date.isoformat(),
+            (max_date - min_date).days
+        )
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
+
+        trades: list = []
+        for pair, pair_data in preprocessed.items():
+            # Sorting dataframe by date and reset index
+            pair_data = pair_data.sort_values(by=['date'])
+            pair_data = pair_data.reset_index(drop=True)
+
+            ticker_data = self.strategy.advise_sell(
+                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
+
+            trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range)
+
+        # If no trade found then exit
+        if len(trades) == 0:
+            logger.info("No trades found.")
+            return False
+
+        # Fill missing, calculable columns, profit, duration , abs etc.
+        trades_df = self._fill_calculable_fields(DataFrame(trades))
+        self._cached_pairs = self._process_expectancy(trades_df)
+        self._last_updated = arrow.utcnow().timestamp
+
+        return True
+
+    def stake_amount(self, pair: str, free_capital: float,
+                     total_capital: float, capital_in_trade: float) -> float:
+        stoploss = self.stoploss(pair)
+        available_capital = (total_capital + capital_in_trade) * self._capital_percentage
+        allowed_capital_at_risk = available_capital * self._allowed_risk
+        max_position_size = abs(allowed_capital_at_risk / stoploss)
+        position_size = min(max_position_size, free_capital)
+        if pair in self._cached_pairs:
+            logger.info(
+                'winrate: %s, expectancy: %s, position size: %s, pair: %s,'
+                ' capital in trade: %s, free capital: %s, total capital: %s,'
+                ' stoploss: %s, available capital: %s.',
+                self._cached_pairs[pair].winrate,
+                self._cached_pairs[pair].expectancy,
+                position_size, pair,
+                capital_in_trade, free_capital, total_capital,
+                stoploss, available_capital
+            )
+        return round(position_size, 15)
+
+    def stoploss(self, pair: str) -> float:
+        if pair in self._cached_pairs:
+            return self._cached_pairs[pair].stoploss
+        else:
+            logger.warning('tried to access stoploss of a non-existing pair, '
+                           'strategy stoploss is returned instead.')
+            return self.strategy.stoploss
+
+    def adjust(self, pairs) -> list:
+        """
+        Filters out and sorts "pairs" according to Edge calculated pairs
+        """
+        final = []
+        for pair, info in self._cached_pairs.items():
+            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
+                info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
+                    pair in pairs:
+                final.append(pair)
+
+        if self._final_pairs != final:
+            self._final_pairs = final
+            if self._final_pairs:
+                logger.info(
+                    'Minimum expectancy and minimum winrate are met only for %s,'
+                    ' so other pairs are filtered out.',
+                    self._final_pairs
+                    )
+            else:
+                logger.info(
+                    'Edge removed all pairs as no pair with minimum expectancy '
+                    'and minimum winrate was found !'
+                    )
+
+        return self._final_pairs
+
+    def accepted_pairs(self) -> list:
+        """
+        return a list of accepted pairs along with their winrate, expectancy and stoploss
+        """
+        final = []
+        for pair, info in self._cached_pairs.items():
+            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
+                 info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
+                final.append({
+                    'Pair': pair,
+                    'Winrate': info.winrate,
+                    'Expectancy': info.expectancy,
+                    'Stoploss': info.stoploss,
+                })
+        return final
+
+    def _fill_calculable_fields(self, result: DataFrame) -> DataFrame:
+        """
+        The result frame contains a number of columns that are calculable
+        from other columns. These are left blank till all rows are added,
+        to be populated in single vector calls.
+
+        Columns to be populated are:
+        - Profit
+        - trade duration
+        - profit abs
+        :param result Dataframe
+        :return: result Dataframe
+        """
+
+        # stake and fees
+        # stake = 0.015
+        # 0.05% is 0.0005
+        # fee = 0.001
+
+        # we set stake amount to an arbitrary amount.
+        # as it doesn't change the calculation.
+        # all returned values are relative. they are percentages.
+        stake = 0.015
+        fee = self.fee
+        open_fee = fee / 2
+        close_fee = fee / 2
+
+        result['trade_duration'] = result['close_time'] - result['open_time']
+
+        result['trade_duration'] = result['trade_duration'].map(
+            lambda x: int(x.total_seconds() / 60))
+
+        # Spends, Takes, Profit, Absolute Profit
+
+        # Buy Price
+        result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
+        result['buy_fee'] = stake * open_fee
+        result['buy_spend'] = stake + result['buy_fee']  # How much we're spending
+
+        # Sell price
+        result['sell_sum'] = result['buy_vol'] * result['close_rate']
+        result['sell_fee'] = result['sell_sum'] * close_fee
+        result['sell_take'] = result['sell_sum'] - result['sell_fee']
+
+        # profit_percent
+        result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']
+
+        # Absolute profit
+        result['profit_abs'] = result['sell_take'] - result['buy_spend']
+
+        return result
+
+    def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
+        """
+        This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
+        The calulation will be done per pair and per strategy.
+        """
+        # Removing pairs having less than min_trades_number
+        min_trades_number = self.edge_config.get('min_trade_number', 10)
+        results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number)
+        ###################################
+
+        # Removing outliers (Only Pumps) from the dataset
+        # The method to detect outliers is to calculate standard deviation
+        # Then every value more than (standard deviation + 2*average) is out (pump)
+        #
+        # Removing Pumps
+        if self.edge_config.get('remove_pumps', False):
+            results = results.groupby(['pair', 'stoploss']).apply(
+                lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()])
+        ##########################################################################
+
+        # Removing trades having a duration more than X minutes (set in config)
+        max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440)
+        results = results[results.trade_duration < max_trade_duration]
+        #######################################################################
+
+        if results.empty:
+            return {}
+
+        groupby_aggregator = {
+            'profit_abs': [
+                ('nb_trades', 'count'),  # number of all trades
+                ('profit_sum', lambda x: x[x > 0].sum()),  # cumulative profit of all winning trades
+                ('loss_sum', lambda x: abs(x[x < 0].sum())),  # cumulative loss of all losing trades
+                ('nb_win_trades', lambda x: x[x > 0].count())  # number of winning trades
+            ],
+            'trade_duration': [('avg_trade_duration', 'mean')]
+        }
+
+        # Group by (pair and stoploss) by applying above aggregator
+        df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg(
+            groupby_aggregator).reset_index(col_level=1)
+
+        # Dropping level 0 as we don't need it
+        df.columns = df.columns.droplevel(0)
+
+        # Calculating number of losing trades, average win and average loss
+        df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']
+        df['average_win'] = df['profit_sum'] / df['nb_win_trades']
+        df['average_loss'] = df['loss_sum'] / df['nb_loss_trades']
+
+        # Win rate = number of profitable trades / number of trades
+        df['winrate'] = df['nb_win_trades'] / df['nb_trades']
+
+        # risk_reward_ratio = average win / average loss
+        df['risk_reward_ratio'] = df['average_win'] / df['average_loss']
+
+        # required_risk_reward = (1 / winrate) - 1
+        df['required_risk_reward'] = (1 / df['winrate']) - 1
+
+        # expectancy = (risk_reward_ratio * winrate) - (lossrate)
+        df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate'])
+
+        # sort by expectancy and stoploss
+        df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby(
+            'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index()
+
+        final = {}
+        for x in df.itertuples():
+            final[x.pair] = PairInfo(
+                x.stoploss,
+                x.winrate,
+                x.risk_reward_ratio,
+                x.required_risk_reward,
+                x.expectancy,
+                x.nb_trades,
+                x.avg_trade_duration
+            )
+
+        # Returning a list of pairs in order of "expectancy"
+        return final
+
+    def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range):
+        buy_column = ticker_data['buy'].values
+        sell_column = ticker_data['sell'].values
+        date_column = ticker_data['date'].values
+        ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values
+
+        result: list = []
+        for stoploss in stoploss_range:
+            result += self._detect_next_stop_or_sell_point(
+                buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair
+            )
+
+        return result
+
+    def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,
+                                        ohlc_columns, stoploss, pair):
+        """
+        Iterate through ohlc_columns in order to find the next trade
+        Next trade opens from the first buy signal noticed to
+        The sell or stoploss signal after it.
+        It then cuts OHLC, buy_column, sell_column and date_column.
+        Cut from (the exit trade index) + 1.
+
+        Author: https://github.com/mishaker
+        """
+
+        result: list = []
+        start_point = 0
+
+        while True:
+            open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal)
+
+            # Return empty if we don't find trade entry (i.e. buy==1) or
+            # we find a buy but at the end of array
+            if open_trade_index == -1 or open_trade_index == len(buy_column) - 1:
+                break
+            else:
+                # When a buy signal is seen,
+                # trade opens in reality on the next candle
+                open_trade_index += 1
+
+            stop_price_percentage = stoploss + 1
+            open_price = ohlc_columns[open_trade_index, 0]
+            stop_price = (open_price * stop_price_percentage)
+
+            # Searching for the index where stoploss is hit
+            stop_index = utf1st.find_1st(
+                ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller)
+
+            # If we don't find it then we assume stop_index will be far in future (infinite number)
+            if stop_index == -1:
+                stop_index = float('inf')
+
+            # Searching for the index where sell is hit
+            sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal)
+
+            # If we don't find it then we assume sell_index will be far in future (infinite number)
+            if sell_index == -1:
+                sell_index = float('inf')
+
+            # Check if we don't find any stop or sell point (in that case trade remains open)
+            # It is not interesting for Edge to consider it so we simply ignore the trade
+            # And stop iterating there is no more entry
+            if stop_index == sell_index == float('inf'):
+                break
+
+            if stop_index <= sell_index:
+                exit_index = open_trade_index + stop_index
+                exit_type = SellType.STOP_LOSS
+                exit_price = stop_price
+            elif stop_index > sell_index:
+                # If exit is SELL then we exit at the next candle
+                exit_index = open_trade_index + sell_index + 1
+
+                # Check if we have the next candle
+                if len(ohlc_columns) - 1 < exit_index:
+                    break
+
+                exit_type = SellType.SELL_SIGNAL
+                exit_price = ohlc_columns[exit_index, 0]
+
+            trade = {'pair': pair,
+                     'stoploss': stoploss,
+                     'profit_percent': '',
+                     'profit_abs': '',
+                     'open_time': date_column[open_trade_index],
+                     'close_time': date_column[exit_index],
+                     'open_index': start_point + open_trade_index,
+                     'close_index': start_point + exit_index,
+                     'trade_duration': '',
+                     'open_rate': round(open_price, 15),
+                     'close_rate': round(exit_price, 15),
+                     'exit_type': exit_type
+                     }
+
+            result.append(trade)
+
+            # Giving a view of exit_index till the end of array
+            buy_column = buy_column[exit_index:]
+            sell_column = sell_column[exit_index:]
+            date_column = date_column[exit_index:]
+            ohlc_columns = ohlc_columns[exit_index:]
+            start_point += exit_index
+
+        return result

freqtrade/freqtradebot.py

@@ -7,6 +7,7 @@ import traceback
 from datetime import datetime
 from math import isclose
 from os import getpid
+from threading import Lock
 from typing import Any, Dict, List, Optional, Tuple
 
 import arrow
@@ -27,7 +28,6 @@ from freqtrade.state import State
 from freqtrade.strategy.interface import IStrategy, SellType
 from freqtrade.wallets import Wallets
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -92,6 +92,8 @@ class FreqtradeBot:
         # the initial state of the bot.
         # Keep this at the end of this initialization method.
         self.rpc: RPCManager = RPCManager(self)
+        # Protect sell-logic from forcesell and viceversa
+        self._sell_lock = Lock()
 
     def cleanup(self) -> None:
         """
@@ -132,8 +134,12 @@ class FreqtradeBot:
         self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist),
                                   self.strategy.informative_pairs())
 
-        # First process current opened trades (positions)
-        self.exit_positions(trades)
+        # Protect from collisions with forcesell.
+        # Without this, freqtrade my try to recreate stoploss_on_exchange orders
+        # while selling is in process, since telegram messages arrive in an different thread.
+        with self._sell_lock:
+            # First process current opened trades (positions)
+            self.exit_positions(trades)
 
         # Then looking for buy opportunities
         if self.get_free_open_trades():
@@ -218,7 +224,7 @@ class FreqtradeBot:
 
         return trades_created
 
-    def get_target_bid(self, pair: str, tick: Dict = None) -> float:
+    def get_buy_rate(self, pair: str, tick: Dict = None) -> float:
         """
         Calculates bid target between current ask price and last price
         :return: float: Price
@@ -435,7 +441,7 @@ class FreqtradeBot:
             buy_limit_requested = price
         else:
             # Calculate price
-            buy_limit_requested = self.get_target_bid(pair)
+            buy_limit_requested = self.get_buy_rate(pair)
 
         min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested)
         if min_stake_amount is not None and min_stake_amount > stake_amount:
@@ -748,8 +754,8 @@ class FreqtradeBot:
         Check and execute sell
         """
         should_sell = self.strategy.should_sell(
-                trade, sell_rate, datetime.utcnow(), buy, sell,
-                force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0
+            trade, sell_rate, datetime.utcnow(), buy, sell,
+            force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0
         )
 
         if should_sell.sell_flag:

freqtrade/main.py

@@ -14,7 +14,7 @@ if sys.version_info < (3, 6):
 import logging
 from typing import Any, List
 
-from freqtrade.configuration import Arguments
+from freqtrade.commands import Arguments
 
 
 logger = logging.getLogger('freqtrade')

freqtrade/optimize/__init__.py

@@ -1,102 +0,0 @@
-import logging
-from typing import Any, Dict
-
-from freqtrade import constants
-from freqtrade.exceptions import DependencyException, OperationalException
-from freqtrade.state import RunMode
-from freqtrade.utils import setup_utils_configuration
-
-logger = logging.getLogger(__name__)
-
-
-def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
-    """
-    Prepare the configuration for the Hyperopt module
-    :param args: Cli args from Arguments()
-    :return: Configuration
-    """
-    config = setup_utils_configuration(args, method)
-
-    if method == RunMode.BACKTEST:
-        if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
-            raise DependencyException('stake amount could not be "%s" for backtesting' %
-                                      constants.UNLIMITED_STAKE_AMOUNT)
-
-    return config
-
-
-def start_backtesting(args: Dict[str, Any]) -> None:
-    """
-    Start Backtesting script
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    # Import here to avoid loading backtesting module when it's not used
-    from freqtrade.optimize.backtesting import Backtesting
-
-    # Initialize configuration
-    config = setup_configuration(args, RunMode.BACKTEST)
-
-    logger.info('Starting freqtrade in Backtesting mode')
-
-    # Initialize backtesting object
-    backtesting = Backtesting(config)
-    backtesting.start()
-
-
-def start_hyperopt(args: Dict[str, Any]) -> None:
-    """
-    Start hyperopt script
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    # Import here to avoid loading hyperopt module when it's not used
-    try:
-        from filelock import FileLock, Timeout
-        from freqtrade.optimize.hyperopt import Hyperopt
-    except ImportError as e:
-        raise OperationalException(
-            f"{e}. Please ensure that the hyperopt dependencies are installed.") from e
-    # Initialize configuration
-    config = setup_configuration(args, RunMode.HYPEROPT)
-
-    logger.info('Starting freqtrade in Hyperopt mode')
-
-    lock = FileLock(Hyperopt.get_lock_filename(config))
-
-    try:
-        with lock.acquire(timeout=1):
-
-            # Remove noisy log messages
-            logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
-            logging.getLogger('filelock').setLevel(logging.WARNING)
-
-            # Initialize backtesting object
-            hyperopt = Hyperopt(config)
-            hyperopt.start()
-
-    except Timeout:
-        logger.info("Another running instance of freqtrade Hyperopt detected.")
-        logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. "
-                    "Hyperopt module is resource hungry. Please run your Hyperopt sequentially "
-                    "or on separate machines.")
-        logger.info("Quitting now.")
-        # TODO: return False here in order to help freqtrade to exit
-        # with non-zero exit code...
-        # Same in Edge and Backtesting start() functions.
-
-
-def start_edge(args: Dict[str, Any]) -> None:
-    """
-    Start Edge script
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    from freqtrade.optimize.edge_cli import EdgeCli
-    # Initialize configuration
-    config = setup_configuration(args, RunMode.EDGE)
-    logger.info('Starting freqtrade in Edge mode')
-
-    # Initialize Edge object
-    edge_cli = EdgeCli(config)
-    edge_cli.start()

freqtrade/optimize/backtesting.py

@@ -281,30 +281,28 @@ class Backtesting:
             return bt_res
         return None
 
-    def backtest(self, args: Dict) -> DataFrame:
+    def backtest(self, processed: Dict, stake_amount: float,
+                 start_date, end_date,
+                 max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame:
         """
-        Implements backtesting functionality
+        Implement backtesting functionality
 
         NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
         Of course try to not have ugly code. By some accessor are sometime slower than functions.
-        Avoid, logging on this method
+        Avoid extensive logging in this method and functions it calls.
 
-        :param args: a dict containing:
-            stake_amount: btc amount to use for each trade
-            processed: a processed dictionary with format {pair, data}
-            max_open_trades: maximum number of concurrent trades (default: 0, disabled)
-            position_stacking: do we allow position stacking? (default: False)
-        :return: DataFrame
+        :param processed: a processed dictionary with format {pair, data}
+        :param stake_amount: amount to use for each trade
+        :param start_date: backtesting timerange start datetime
+        :param end_date: backtesting timerange end datetime
+        :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited
+        :param position_stacking: do we allow position stacking?
+        :return: DataFrame with trades (results of backtesting)
         """
-        # Arguments are long and noisy, so this is commented out.
-        # Uncomment if you need to debug the backtest() method.
-        # logger.debug(f"Start backtest, args: {args}")
-        processed = args['processed']
-        stake_amount = args['stake_amount']
-        max_open_trades = args.get('max_open_trades', 0)
-        position_stacking = args.get('position_stacking', False)
-        start_date = args['start_date']
-        end_date = args['end_date']
+        logger.debug(f"Run backtest, stake_amount: {stake_amount}, "
+                     f"start_date: {start_date}, end_date: {end_date}, "
+                     f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}"
+                     )
         trades = []
         trade_count_lock: Dict = {}
 
@@ -371,18 +369,21 @@ class Backtesting:
 
     def start(self) -> None:
         """
-        Run a backtesting end-to-end
+        Run backtesting end-to-end
         :return: None
         """
         data: Dict[str, Any] = {}
+
         logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
         logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
+
         # Use max_open_trades in backtesting, except --disable-max-market-positions is set
         if self.config.get('use_max_market_positions', True):
             max_open_trades = self.config['max_open_trades']
         else:
             logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
             max_open_trades = 0
+        position_stacking = self.config.get('position_stacking', False)
 
         data, timerange = self.load_bt_data()
 
@@ -405,14 +406,12 @@ class Backtesting:
             )
             # Execute backtest and print results
             all_results[self.strategy.get_strategy_name()] = self.backtest(
-                {
-                    'stake_amount': self.config.get('stake_amount'),
-                    'processed': preprocessed,
-                    'max_open_trades': max_open_trades,
-                    'position_stacking': self.config.get('position_stacking', False),
-                    'start_date': min_date,
-                    'end_date': max_date,
-                }
+                    processed=preprocessed,
+                    stake_amount=self.config['stake_amount'],
+                    start_date=min_date,
+                    end_date=max_date,
+                    max_open_trades=max_open_trades,
+                    position_stacking=position_stacking,
             )
 
         for strategy, results in all_results.items():

freqtrade/optimize/hyperopt.py

@@ -373,14 +373,12 @@ class Hyperopt:
         min_date, max_date = get_timerange(processed)
 
         backtesting_results = self.backtesting.backtest(
-            {
-                'stake_amount': self.config['stake_amount'],
-                'processed': processed,
-                'max_open_trades': self.max_open_trades,
-                'position_stacking': self.position_stacking,
-                'start_date': min_date,
-                'end_date': max_date,
-            }
+                processed=processed,
+                stake_amount=self.config['stake_amount'],
+                start_date=min_date,
+                end_date=max_date,
+                max_open_trades=self.max_open_trades,
+                position_stacking=self.position_stacking,
         )
         return self._get_results_dict(backtesting_results, min_date, max_date,
                                       params_dict, params_details)

freqtrade/optimize/optimize_reports.py

@@ -70,7 +70,7 @@ def generate_text_table_sell_reason(data: Dict[str, Dict], results: DataFrame) -
     for reason, count in results['sell_reason'].value_counts().iteritems():
         result = results.loc[results['sell_reason'] == reason]
         profit = len(result[result['profit_abs'] >= 0])
-        loss = len(result[results['profit_abs'] < 0])
+        loss = len(result[result['profit_abs'] < 0])
         profit_mean = round(result['profit_percent'].mean() * 100.0, 2)
         tabular_data.append([reason.value, count, profit, loss, profit_mean])
     return tabulate(tabular_data, headers=headers, tablefmt="pipe")

freqtrade/rpc/rpc.py

@@ -420,24 +420,27 @@ class RPC:
         if self._freqtrade.state != State.RUNNING:
             raise RPCException('trader is not running')
 
-        if trade_id == 'all':
-            # Execute sell for all open orders
-            for trade in Trade.get_open_trades():
-                _exec_forcesell(trade)
+        with self._freqtrade._sell_lock:
+            if trade_id == 'all':
+                # Execute sell for all open orders
+                for trade in Trade.get_open_trades():
+                    _exec_forcesell(trade)
+                Trade.session.flush()
+                self._freqtrade.wallets.update()
+                return {'result': 'Created sell orders for all open trades.'}
+
+            # Query for trade
+            trade = Trade.get_trades(
+                trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ]
+            ).first()
+            if not trade:
+                logger.warning('forcesell: Invalid argument received')
+                raise RPCException('invalid argument')
+
+            _exec_forcesell(trade)
             Trade.session.flush()
-            return {'result': 'Created sell orders for all open trades.'}
-
-        # Query for trade
-        trade = Trade.get_trades(
-            trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ]
-        ).first()
-        if not trade:
-            logger.warning('forcesell: Invalid argument received')
-            raise RPCException('invalid argument')
-
-        _exec_forcesell(trade)
-        Trade.session.flush()
-        return {'result': f'Created sell order for trade {trade_id}.'}
+            self._freqtrade.wallets.update()
+            return {'result': f'Created sell order for trade {trade_id}.'}
 
     def _rpc_forcebuy(self, pair: str, price: Optional[float]) -> Optional[Trade]:
         """

freqtrade/templates/sample_hyperopt_loss.py

@@ -27,7 +27,8 @@ class SampleHyperOptLoss(IHyperOptLoss):
     Defines the default loss function for hyperopt
     This is intended to give you some inspiration for your own loss function.
 
-    The Function needs to return a number (float) - which becomes for better backtest results.
+    The Function needs to return a number (float) - which becomes smaller for better backtest
+    results.
     """
 
     @staticmethod

freqtrade/utils.py

@@ -1,505 +0,0 @@
-import csv
-import logging
-import sys
-from collections import OrderedDict
-from operator import itemgetter
-from pathlib import Path
-from typing import Any, Dict, List
-
-import arrow
-import rapidjson
-from colorama import init as colorama_init
-from tabulate import tabulate
-
-from freqtrade.configuration import (Configuration, TimeRange,
-                                     remove_credentials,
-                                     validate_config_consistency)
-from freqtrade.configuration.directory_operations import (copy_sample_files,
-                                                          create_userdata_dir)
-from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY
-from freqtrade.data.converter import (convert_ohlcv_format,
-                                      convert_trades_format)
-from freqtrade.data.history import (convert_trades_to_ohlcv,
-                                    refresh_backtest_ohlcv_data,
-                                    refresh_backtest_trades_data)
-from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import (available_exchanges, ccxt_exchanges,
-                                market_is_active, symbol_is_pair)
-from freqtrade.misc import plural, render_template
-from freqtrade.resolvers import ExchangeResolver, StrategyResolver
-from freqtrade.state import RunMode
-
-logger = logging.getLogger(__name__)
-
-
-def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
-    """
-    Prepare the configuration for utils subcommands
-    :param args: Cli args from Arguments()
-    :return: Configuration
-    """
-    configuration = Configuration(args, method)
-    config = configuration.get_config()
-
-    # Ensure we do not use Exchange credentials
-    remove_credentials(config)
-    validate_config_consistency(config)
-
-    return config
-
-
-def start_trading(args: Dict[str, Any]) -> int:
-    """
-    Main entry point for trading mode
-    """
-    from freqtrade.worker import Worker
-    # Load and run worker
-    worker = None
-    try:
-        worker = Worker(args)
-        worker.run()
-    except KeyboardInterrupt:
-        logger.info('SIGINT received, aborting ...')
-    finally:
-        if worker:
-            logger.info("worker found ... calling exit")
-            worker.exit()
-    return 0
-
-
-def start_list_exchanges(args: Dict[str, Any]) -> None:
-    """
-    Print available exchanges
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
-    if args['print_one_column']:
-        print('\n'.join(exchanges))
-    else:
-        if args['list_exchanges_all']:
-            print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
-        else:
-            print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
-
-
-def start_create_userdir(args: Dict[str, Any]) -> None:
-    """
-    Create "user_data" directory to contain user data strategies, hyperopt, ...)
-    :param args: Cli args from Arguments()
-    :return: None
-    """
-    if "user_data_dir" in args and args["user_data_dir"]:
-        userdir = create_userdata_dir(args["user_data_dir"], create_dir=True)
-        copy_sample_files(userdir, overwrite=args["reset"])
-    else:
-        logger.warning("`create-userdir` requires --userdir to be set.")
-        sys.exit(1)
-
-
-def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str):
-    """
-    Deploy new strategy from template to strategy_path
-    """
-    indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",)
-    buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",)
-    sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",)
-    plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",)
-
-    strategy_text = render_template(templatefile='base_strategy.py.j2',
-                                    arguments={"strategy": strategy_name,
-                                               "indicators": indicators,
-                                               "buy_trend": buy_trend,
-                                               "sell_trend": sell_trend,
-                                               "plot_config": plot_config,
-                                               })
-
-    logger.info(f"Writing strategy to `{strategy_path}`.")
-    strategy_path.write_text(strategy_text)
-
-
-def start_new_strategy(args: Dict[str, Any]) -> None:
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    if "strategy" in args and args["strategy"]:
-        if args["strategy"] == "DefaultStrategy":
-            raise OperationalException("DefaultStrategy is not allowed as name.")
-
-        new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py")
-
-        if new_path.exists():
-            raise OperationalException(f"`{new_path}` already exists. "
-                                       "Please choose another Strategy Name.")
-
-        deploy_new_strategy(args['strategy'], new_path, args['template'])
-
-    else:
-        raise OperationalException("`new-strategy` requires --strategy to be set.")
-
-
-def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str):
-    """
-    Deploys a new hyperopt template to hyperopt_path
-    """
-    buy_guards = render_template(
-        templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",)
-    sell_guards = render_template(
-        templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",)
-    buy_space = render_template(
-        templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",)
-    sell_space = render_template(
-        templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",)
-
-    strategy_text = render_template(templatefile='base_hyperopt.py.j2',
-                                    arguments={"hyperopt": hyperopt_name,
-                                               "buy_guards": buy_guards,
-                                               "sell_guards": sell_guards,
-                                               "buy_space": buy_space,
-                                               "sell_space": sell_space,
-                                               })
-
-    logger.info(f"Writing hyperopt to `{hyperopt_path}`.")
-    hyperopt_path.write_text(strategy_text)
-
-
-def start_new_hyperopt(args: Dict[str, Any]) -> None:
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    if "hyperopt" in args and args["hyperopt"]:
-        if args["hyperopt"] == "DefaultHyperopt":
-            raise OperationalException("DefaultHyperopt is not allowed as name.")
-
-        new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py")
-
-        if new_path.exists():
-            raise OperationalException(f"`{new_path}` already exists. "
-                                       "Please choose another Strategy Name.")
-        deploy_new_hyperopt(args['hyperopt'], new_path, args['template'])
-    else:
-        raise OperationalException("`new-hyperopt` requires --hyperopt to be set.")
-
-
-def start_download_data(args: Dict[str, Any]) -> None:
-    """
-    Download data (former download_backtest_data.py script)
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    timerange = TimeRange()
-    if 'days' in config:
-        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
-        timerange = TimeRange.parse_timerange(f'{time_since}-')
-
-    if 'pairs' not in config:
-        raise OperationalException(
-            "Downloading data requires a list of pairs. "
-            "Please check the documentation on how to configure this.")
-
-    logger.info(f'About to download pairs: {config["pairs"]}, '
-                f'intervals: {config["timeframes"]} to {config["datadir"]}')
-
-    pairs_not_available: List[str] = []
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
-    try:
-
-        if config.get('download_trades'):
-            pairs_not_available = refresh_backtest_trades_data(
-                exchange, pairs=config["pairs"], datadir=config['datadir'],
-                timerange=timerange, erase=config.get("erase"),
-                data_format=config['dataformat_trades'])
-
-            # Convert downloaded trade data to different timeframes
-            convert_trades_to_ohlcv(
-                pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"),
-                data_format_ohlcv=config['dataformat_ohlcv'],
-                data_format_trades=config['dataformat_trades'],
-                )
-
-        else:
-            pairs_not_available = refresh_backtest_ohlcv_data(
-                exchange, pairs=config["pairs"], timeframes=config["timeframes"],
-                datadir=config['datadir'], timerange=timerange, erase=config.get("erase"),
-                data_format=config['dataformat_ohlcv'])
-
-    except KeyboardInterrupt:
-        sys.exit("SIGINT received, aborting ...")
-
-    finally:
-        if pairs_not_available:
-            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
-                        f"on exchange {exchange.name}.")
-
-
-def start_list_strategies(args: Dict[str, Any]) -> None:
-    """
-    Print Strategies available in a directory
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY))
-    strategies = StrategyResolver.search_all_objects(directory)
-    # Sort alphabetically
-    strategies = sorted(strategies, key=lambda x: x['name'])
-    strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies]
-
-    if args['print_one_column']:
-        print('\n'.join([s['name'] for s in strategies]))
-    else:
-        print(tabulate(strats_to_print, headers='keys', tablefmt='pipe'))
-
-
-def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
-    """
-    Convert data from one format to another
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-    if ohlcv:
-        convert_ohlcv_format(config,
-                             convert_from=args['format_from'], convert_to=args['format_to'],
-                             erase=args['erase'])
-    else:
-        convert_trades_format(config,
-                              convert_from=args['format_from'], convert_to=args['format_to'],
-                              erase=args['erase'])
-
-
-def start_list_timeframes(args: Dict[str, Any]) -> None:
-    """
-    Print ticker intervals (timeframes) available on Exchange
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-    # Do not use ticker_interval set in the config
-    config['ticker_interval'] = None
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
-
-    if args['print_one_column']:
-        print('\n'.join(exchange.timeframes))
-    else:
-        print(f"Timeframes available for the exchange `{exchange.name}`: "
-              f"{', '.join(exchange.timeframes)}")
-
-
-def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
-    """
-    Print pairs/markets on the exchange
-    :param args: Cli args from Arguments()
-    :param pairs_only: if True print only pairs, otherwise print all instruments (markets)
-    :return: None
-    """
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    # Init exchange
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
-
-    # By default only active pairs/markets are to be shown
-    active_only = not args.get('list_pairs_all', False)
-
-    base_currencies = args.get('base_currencies', [])
-    quote_currencies = args.get('quote_currencies', [])
-
-    try:
-        pairs = exchange.get_markets(base_currencies=base_currencies,
-                                     quote_currencies=quote_currencies,
-                                     pairs_only=pairs_only,
-                                     active_only=active_only)
-        # Sort the pairs/markets by symbol
-        pairs = OrderedDict(sorted(pairs.items()))
-    except Exception as e:
-        raise OperationalException(f"Cannot get markets. Reason: {e}") from e
-
-    else:
-        summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") +
-                       ("active " if active_only else "") +
-                       (plural(len(pairs), "pair" if pairs_only else "market")) +
-                       (f" with {', '.join(base_currencies)} as base "
-                        f"{plural(len(base_currencies), 'currency', 'currencies')}"
-                        if base_currencies else "") +
-                       (" and" if base_currencies and quote_currencies else "") +
-                       (f" with {', '.join(quote_currencies)} as quote "
-                        f"{plural(len(quote_currencies), 'currency', 'currencies')}"
-                        if quote_currencies else ""))
-
-        headers = ["Id", "Symbol", "Base", "Quote", "Active",
-                   *(['Is pair'] if not pairs_only else [])]
-
-        tabular_data = []
-        for _, v in pairs.items():
-            tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
-                                 'Base': v['base'], 'Quote': v['quote'],
-                                 'Active': market_is_active(v),
-                                 **({'Is pair': symbol_is_pair(v['symbol'])}
-                                    if not pairs_only else {})})
-
-        if (args.get('print_one_column', False) or
-                args.get('list_pairs_print_json', False) or
-                args.get('print_csv', False)):
-            # Print summary string in the log in case of machine-readable
-            # regular formats.
-            logger.info(f"{summary_str}.")
-        else:
-            # Print empty string separating leading logs and output in case of
-            # human-readable formats.
-            print()
-
-        if len(pairs):
-            if args.get('print_list', False):
-                # print data as a list, with human-readable summary
-                print(f"{summary_str}: {', '.join(pairs.keys())}.")
-            elif args.get('print_one_column', False):
-                print('\n'.join(pairs.keys()))
-            elif args.get('list_pairs_print_json', False):
-                print(rapidjson.dumps(list(pairs.keys()), default=str))
-            elif args.get('print_csv', False):
-                writer = csv.DictWriter(sys.stdout, fieldnames=headers)
-                writer.writeheader()
-                writer.writerows(tabular_data)
-            else:
-                # print data as a table, with the human-readable summary
-                print(f"{summary_str}:")
-                print(tabulate(tabular_data, headers='keys', tablefmt='pipe'))
-        elif not (args.get('print_one_column', False) or
-                  args.get('list_pairs_print_json', False) or
-                  args.get('print_csv', False)):
-            print(f"{summary_str}.")
-
-
-def start_test_pairlist(args: Dict[str, Any]) -> None:
-    """
-    Test Pairlist configuration
-    """
-    from freqtrade.pairlist.pairlistmanager import PairListManager
-    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
-
-    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
-
-    quote_currencies = args.get('quote_currencies')
-    if not quote_currencies:
-        quote_currencies = [config.get('stake_currency')]
-    results = {}
-    for curr in quote_currencies:
-        config['stake_currency'] = curr
-        # Do not use ticker_interval set in the config
-        pairlists = PairListManager(exchange, config)
-        pairlists.refresh_pairlist()
-        results[curr] = pairlists.whitelist
-
-    for curr, pairlist in results.items():
-        if not args.get('print_one_column', False):
-            print(f"Pairs for {curr}: ")
-
-        if args.get('print_one_column', False):
-            print('\n'.join(pairlist))
-        elif args.get('list_pairs_print_json', False):
-            print(rapidjson.dumps(list(pairlist), default=str))
-        else:
-            print(pairlist)
-
-
-def start_hyperopt_list(args: Dict[str, Any]) -> None:
-    """
-    List hyperopt epochs previously evaluated
-    """
-    from freqtrade.optimize.hyperopt import Hyperopt
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    only_best = config.get('hyperopt_list_best', False)
-    only_profitable = config.get('hyperopt_list_profitable', False)
-    print_colorized = config.get('print_colorized', False)
-    print_json = config.get('print_json', False)
-    no_details = config.get('hyperopt_list_no_details', False)
-    no_header = False
-
-    trials_file = (config['user_data_dir'] /
-                   'hyperopt_results' / 'hyperopt_results.pickle')
-
-    # Previous evaluations
-    trials = Hyperopt.load_previous_results(trials_file)
-    total_epochs = len(trials)
-
-    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
-
-    # TODO: fetch the interval for epochs to print from the cli option
-    epoch_start, epoch_stop = 0, None
-
-    if print_colorized:
-        colorama_init(autoreset=True)
-
-    try:
-        # Human-friendly indexes used here (starting from 1)
-        for val in trials[epoch_start:epoch_stop]:
-            Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized)
-
-    except KeyboardInterrupt:
-        print('User interrupted..')
-
-    if trials and not no_details:
-        sorted_trials = sorted(trials, key=itemgetter('loss'))
-        results = sorted_trials[0]
-        Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
-
-
-def start_hyperopt_show(args: Dict[str, Any]) -> None:
-    """
-    Show details of a hyperopt epoch previously evaluated
-    """
-    from freqtrade.optimize.hyperopt import Hyperopt
-
-    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
-
-    only_best = config.get('hyperopt_list_best', False)
-    only_profitable = config.get('hyperopt_list_profitable', False)
-    no_header = config.get('hyperopt_show_no_header', False)
-
-    trials_file = (config['user_data_dir'] /
-                   'hyperopt_results' / 'hyperopt_results.pickle')
-
-    # Previous evaluations
-    trials = Hyperopt.load_previous_results(trials_file)
-    total_epochs = len(trials)
-
-    trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
-    trials_epochs = len(trials)
-
-    n = config.get('hyperopt_show_index', -1)
-    if n > trials_epochs:
-        raise OperationalException(
-            f"The index of the epoch to show should be less than {trials_epochs + 1}.")
-    if n < -trials_epochs:
-        raise OperationalException(
-            f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
-
-    # Translate epoch index from human-readable format to pythonic
-    if n > 0:
-        n -= 1
-
-    print_json = config.get('print_json', False)
-
-    if trials:
-        val = trials[n]
-        Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
-                                     header_str="Epoch details")
-
-
-def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List:
-    """
-    Filter our items from the list of hyperopt results
-    """
-    if only_best:
-        trials = [x for x in trials if x['is_best']]
-    if only_profitable:
-        trials = [x for x in trials if x['results_metrics']['profit'] > 0]
-
-    logger.info(f"{len(trials)} " +
-                ("best " if only_best else "") +
-                ("profitable " if only_profitable else "") +
-                "epochs found.")
-
-    return trials