diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 30427c3cc..138fba208 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -2,6 +2,7 @@ # Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib # Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" +python -m pip install --upgrade pip pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl pip install -r requirements-dev.txt diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py new file mode 100644 index 000000000..24197519a --- /dev/null +++ b/freqtrade/commands/__init__.py @@ -0,0 +1,26 @@ +# flake8: noqa: F401 +""" +Commands module. +Contains all start-commands, subcommands and CLI Interface creation. + +Note: Be careful with file-scoped imports in these subfiles. + as they are parsed on startup, nothing containing optional modules should be loaded. +""" +from freqtrade.commands.arguments import Arguments +from freqtrade.commands.data_commands import (start_convert_data, + start_download_data) +from freqtrade.commands.deploy_commands import (start_create_userdir, + start_new_hyperopt, + start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, + start_list_markets, + start_list_strategies, + start_list_timeframes) +from freqtrade.commands.optimize_commands import (start_backtesting, + start_edge, start_hyperopt) +from freqtrade.commands.pairlist_commands import start_test_pairlist +from freqtrade.commands.plot_commands import (start_plot_dataframe, + start_plot_profit) +from freqtrade.commands.trade_commands import start_trading diff --git a/freqtrade/configuration/arguments.py b/freqtrade/commands/arguments.py similarity index 94% rename from freqtrade/configuration/arguments.py rename to freqtrade/commands/arguments.py index 718b6dedc..7e68b00cd 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/commands/arguments.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional from freqtrade import constants -from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS +from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] @@ -134,14 +134,15 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import (start_create_userdir, start_convert_data, start_download_data, - start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_markets, - start_list_strategies, start_new_hyperopt, - start_new_strategy, start_list_timeframes, - start_test_pairlist, start_trading) - from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit + from freqtrade.commands import (start_create_userdir, start_convert_data, + start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_markets, + start_list_strategies, start_new_hyperopt, + start_new_strategy, start_list_timeframes, + start_plot_dataframe, start_plot_profit, + start_backtesting, start_hyperopt, start_edge, + start_test_pairlist, start_trading) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/commands/cli_options.py similarity index 99% rename from freqtrade/configuration/cli_options.py rename to freqtrade/commands/cli_options.py index 9f178cdbd..39cb5bf93 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -import argparse +from argparse import ArgumentTypeError from freqtrade import __version__, constants @@ -12,7 +12,7 @@ def check_int_positive(value: str) -> int: if uint <= 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a positive integer value" ) return uint @@ -24,7 +24,7 @@ def check_int_nonzero(value: str) -> int: if uint == 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a non-zero integer value" ) return uint diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py new file mode 100644 index 000000000..7c57c5f2b --- /dev/null +++ b/freqtrade/commands/data_commands.py @@ -0,0 +1,85 @@ +import logging +import sys +from typing import Any, Dict, List + +import arrow + +from freqtrade.configuration import TimeRange, setup_utils_configuration +from freqtrade.data.converter import (convert_ohlcv_format, + convert_trades_format) +from freqtrade.data.history import (convert_trades_to_ohlcv, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data) +from freqtrade.exceptions import OperationalException +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_download_data(args: Dict[str, Any]) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {config["datadir"]}') + + pairs_not_available: List[str] = [] + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + try: + + if config.get('download_trades'): + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=config["pairs"], datadir=config['datadir'], + timerange=timerange, erase=config.get("erase"), + data_format=config['dataformat_trades']) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase"), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + else: + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase"), + data_format=config['dataformat_ohlcv']) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.") + + +def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: + """ + Convert data from one format to another + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + if ohlcv: + convert_ohlcv_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) + else: + convert_trades_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py new file mode 100644 index 000000000..99ae63244 --- /dev/null +++ b/freqtrade/commands/deploy_commands.py @@ -0,0 +1,112 @@ +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.configuration.directory_operations import (copy_sample_files, + create_userdata_dir) +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.exceptions import OperationalException +from freqtrade.misc import render_template +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_create_userdir(args: Dict[str, Any]) -> None: + """ + Create "user_data" directory to contain user data strategies, hyperopt, ...) + :param args: Cli args from Arguments() + :return: None + """ + if "user_data_dir" in args and args["user_data_dir"]: + userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) + copy_sample_files(userdir, overwrite=args["reset"]) + else: + logger.warning("`create-userdir` requires --userdir to be set.") + sys.exit(1) + + +def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): + """ + Deploy new strategy from template to strategy_path + """ + indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) + buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) + sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) + plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",) + + strategy_text = render_template(templatefile='base_strategy.py.j2', + arguments={"strategy": strategy_name, + "indicators": indicators, + "buy_trend": buy_trend, + "sell_trend": sell_trend, + "plot_config": plot_config, + }) + + logger.info(f"Writing strategy to `{strategy_path}`.") + strategy_path.write_text(strategy_text) + + +def start_new_strategy(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "strategy" in args and args["strategy"]: + if args["strategy"] == "DefaultStrategy": + raise OperationalException("DefaultStrategy is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + + deploy_new_strategy(args['strategy'], new_path, args['template']) + + else: + raise OperationalException("`new-strategy` requires --strategy to be set.") + + +def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): + """ + Deploys a new hyperopt template to hyperopt_path + """ + buy_guards = render_template( + templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) + sell_guards = render_template( + templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) + buy_space = render_template( + templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) + sell_space = render_template( + templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) + + strategy_text = render_template(templatefile='base_hyperopt.py.j2', + arguments={"hyperopt": hyperopt_name, + "buy_guards": buy_guards, + "sell_guards": sell_guards, + "buy_space": buy_space, + "sell_space": sell_space, + }) + + logger.info(f"Writing hyperopt to `{hyperopt_path}`.") + hyperopt_path.write_text(strategy_text) + + +def start_new_hyperopt(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "hyperopt" in args and args["hyperopt"]: + if args["hyperopt"] == "DefaultHyperopt": + raise OperationalException("DefaultHyperopt is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) + else: + raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py new file mode 100644 index 000000000..5c6f25848 --- /dev/null +++ b/freqtrade/commands/hyperopt_commands.py @@ -0,0 +1,114 @@ +import logging +from operator import itemgetter +from typing import Any, Dict, List + +from colorama import init as colorama_init + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.exceptions import OperationalException +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_hyperopt_list(args: Dict[str, Any]) -> None: + """ + List hyperopt epochs previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + only_best = config.get('hyperopt_list_best', False) + only_profitable = config.get('hyperopt_list_profitable', False) + print_colorized = config.get('print_colorized', False) + print_json = config.get('print_json', False) + no_details = config.get('hyperopt_list_no_details', False) + no_header = False + + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + trials = Hyperopt.load_previous_results(trials_file) + total_epochs = len(trials) + + trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + + # TODO: fetch the interval for epochs to print from the cli option + epoch_start, epoch_stop = 0, None + + if print_colorized: + colorama_init(autoreset=True) + + try: + # Human-friendly indexes used here (starting from 1) + for val in trials[epoch_start:epoch_stop]: + Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) + + except KeyboardInterrupt: + print('User interrupted..') + + if trials and not no_details: + sorted_trials = sorted(trials, key=itemgetter('loss')) + results = sorted_trials[0] + Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) + + +def start_hyperopt_show(args: Dict[str, Any]) -> None: + """ + Show details of a hyperopt epoch previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + only_best = config.get('hyperopt_list_best', False) + only_profitable = config.get('hyperopt_list_profitable', False) + no_header = config.get('hyperopt_show_no_header', False) + + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + trials = Hyperopt.load_previous_results(trials_file) + total_epochs = len(trials) + + trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + trials_epochs = len(trials) + + n = config.get('hyperopt_show_index', -1) + if n > trials_epochs: + raise OperationalException( + f"The index of the epoch to show should be less than {trials_epochs + 1}.") + if n < -trials_epochs: + raise OperationalException( + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + + # Translate epoch index from human-readable format to pythonic + if n > 0: + n -= 1 + + print_json = config.get('print_json', False) + + if trials: + val = trials[n] + Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") + + +def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: + """ + Filter our items from the list of hyperopt results + """ + if only_best: + trials = [x for x in trials if x['is_best']] + if only_profitable: + trials = [x for x in trials if x['results_metrics']['profit'] > 0] + + logger.info(f"{len(trials)} " + + ("best " if only_best else "") + + ("profitable " if only_profitable else "") + + "epochs found.") + + return trials diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py new file mode 100644 index 000000000..022822782 --- /dev/null +++ b/freqtrade/commands/list_commands.py @@ -0,0 +1,156 @@ +import csv +import logging +import sys +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict + +import rapidjson +from tabulate import tabulate + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.constants import USERPATH_STRATEGY +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import (available_exchanges, ccxt_exchanges, + market_is_active, symbol_is_pair) +from freqtrade.misc import plural +from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_list_exchanges(args: Dict[str, Any]) -> None: + """ + Print available exchanges + :param args: Cli args from Arguments() + :return: None + """ + exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() + if args['print_one_column']: + print('\n'.join(exchanges)) + else: + if args['list_exchanges_all']: + print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") + else: + print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") + + +def start_list_strategies(args: Dict[str, Any]) -> None: + """ + Print Strategies available in a directory + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) + strategies = StrategyResolver.search_all_objects(directory) + # Sort alphabetically + strategies = sorted(strategies, key=lambda x: x['name']) + strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] + + if args['print_one_column']: + print('\n'.join([s['name'] for s in strategies])) + else: + print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) + + +def start_list_timeframes(args: Dict[str, Any]) -> None: + """ + Print ticker intervals (timeframes) available on Exchange + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + # Do not use ticker_interval set in the config + config['ticker_interval'] = None + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + if args['print_one_column']: + print('\n'.join(exchange.timeframes)) + else: + print(f"Timeframes available for the exchange `{exchange.name}`: " + f"{', '.join(exchange.timeframes)}") + + +def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: + """ + Print pairs/markets on the exchange + :param args: Cli args from Arguments() + :param pairs_only: if True print only pairs, otherwise print all instruments (markets) + :return: None + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + # By default only active pairs/markets are to be shown + active_only = not args.get('list_pairs_all', False) + + base_currencies = args.get('base_currencies', []) + quote_currencies = args.get('quote_currencies', []) + + try: + pairs = exchange.get_markets(base_currencies=base_currencies, + quote_currencies=quote_currencies, + pairs_only=pairs_only, + active_only=active_only) + # Sort the pairs/markets by symbol + pairs = OrderedDict(sorted(pairs.items())) + except Exception as e: + raise OperationalException(f"Cannot get markets. Reason: {e}") from e + + else: + summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + + ("active " if active_only else "") + + (plural(len(pairs), "pair" if pairs_only else "market")) + + (f" with {', '.join(base_currencies)} as base " + f"{plural(len(base_currencies), 'currency', 'currencies')}" + if base_currencies else "") + + (" and" if base_currencies and quote_currencies else "") + + (f" with {', '.join(quote_currencies)} as quote " + f"{plural(len(quote_currencies), 'currency', 'currencies')}" + if quote_currencies else "")) + + headers = ["Id", "Symbol", "Base", "Quote", "Active", + *(['Is pair'] if not pairs_only else [])] + + tabular_data = [] + for _, v in pairs.items(): + tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], + 'Base': v['base'], 'Quote': v['quote'], + 'Active': market_is_active(v), + **({'Is pair': symbol_is_pair(v['symbol'])} + if not pairs_only else {})}) + + if (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + # Print summary string in the log in case of machine-readable + # regular formats. + logger.info(f"{summary_str}.") + else: + # Print empty string separating leading logs and output in case of + # human-readable formats. + print() + + if len(pairs): + if args.get('print_list', False): + # print data as a list, with human-readable summary + print(f"{summary_str}: {', '.join(pairs.keys())}.") + elif args.get('print_one_column', False): + print('\n'.join(pairs.keys())) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairs.keys()), default=str)) + elif args.get('print_csv', False): + writer = csv.DictWriter(sys.stdout, fieldnames=headers) + writer.writeheader() + writer.writerows(tabular_data) + else: + # print data as a table, with the human-readable summary + print(f"{summary_str}:") + print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) + elif not (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + print(f"{summary_str}.") diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py new file mode 100644 index 000000000..a2d1b4601 --- /dev/null +++ b/freqtrade/commands/optimize_commands.py @@ -0,0 +1,102 @@ +import logging +from typing import Any, Dict + +from freqtrade import constants +from freqtrade.configuration import setup_utils_configuration +from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for the Hyperopt module + :param args: Cli args from Arguments() + :return: Configuration + """ + config = setup_utils_configuration(args, method) + + if method == RunMode.BACKTEST: + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: + raise DependencyException('stake amount could not be "%s" for backtesting' % + constants.UNLIMITED_STAKE_AMOUNT) + + return config + + +def start_backtesting(args: Dict[str, Any]) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Import here to avoid loading backtesting module when it's not used + from freqtrade.optimize.backtesting import Backtesting + + # Initialize configuration + config = setup_optimize_configuration(args, RunMode.BACKTEST) + + logger.info('Starting freqtrade in Backtesting mode') + + # Initialize backtesting object + backtesting = Backtesting(config) + backtesting.start() + + +def start_hyperopt(args: Dict[str, Any]) -> None: + """ + Start hyperopt script + :param args: Cli args from Arguments() + :return: None + """ + # Import here to avoid loading hyperopt module when it's not used + try: + from filelock import FileLock, Timeout + from freqtrade.optimize.hyperopt import Hyperopt + except ImportError as e: + raise OperationalException( + f"{e}. Please ensure that the hyperopt dependencies are installed.") from e + # Initialize configuration + config = setup_optimize_configuration(args, RunMode.HYPEROPT) + + logger.info('Starting freqtrade in Hyperopt mode') + + lock = FileLock(Hyperopt.get_lock_filename(config)) + + try: + with lock.acquire(timeout=1): + + # Remove noisy log messages + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) + logging.getLogger('filelock').setLevel(logging.WARNING) + + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() + + except Timeout: + logger.info("Another running instance of freqtrade Hyperopt detected.") + logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. " + "Hyperopt module is resource hungry. Please run your Hyperopt sequentially " + "or on separate machines.") + logger.info("Quitting now.") + # TODO: return False here in order to help freqtrade to exit + # with non-zero exit code... + # Same in Edge and Backtesting start() functions. + + +def start_edge(args: Dict[str, Any]) -> None: + """ + Start Edge script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.optimize.edge_cli import EdgeCli + # Initialize configuration + config = setup_optimize_configuration(args, RunMode.EDGE) + logger.info('Starting freqtrade in Edge mode') + + # Initialize Edge object + edge_cli = EdgeCli(config) + edge_cli.start() diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py new file mode 100644 index 000000000..bf0b217a5 --- /dev/null +++ b/freqtrade/commands/pairlist_commands.py @@ -0,0 +1,42 @@ +import logging +from typing import Any, Dict + +import rapidjson + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_test_pairlist(args: Dict[str, Any]) -> None: + """ + Test Pairlist configuration + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + quote_currencies = args.get('quote_currencies') + if not quote_currencies: + quote_currencies = [config.get('stake_currency')] + results = {} + for curr in quote_currencies: + config['stake_currency'] = curr + # Do not use ticker_interval set in the config + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist + + for curr, pairlist in results.items(): + if not args.get('print_one_column', False): + print(f"Pairs for {curr}: ") + + if args.get('print_one_column', False): + print('\n'.join(pairlist)) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairlist), default=str)) + else: + print(pairlist) diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/commands/plot_commands.py similarity index 94% rename from freqtrade/plot/plot_utils.py rename to freqtrade/commands/plot_commands.py index 9eff08396..028933ba7 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/commands/plot_commands.py @@ -1,8 +1,8 @@ from typing import Any, Dict +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/freqtrade/commands/trade_commands.py b/freqtrade/commands/trade_commands.py new file mode 100644 index 000000000..2c0c4c9c1 --- /dev/null +++ b/freqtrade/commands/trade_commands.py @@ -0,0 +1,25 @@ +import logging + +from typing import Any, Dict + + +logger = logging.getLogger(__name__) + + +def start_trading(args: Dict[str, Any]) -> int: + """ + Main entry point for trading mode + """ + from freqtrade.worker import Worker + # Load and run worker + worker = None + try: + worker = Worker(args) + worker.run() + except KeyboardInterrupt: + logger.info('SIGINT received, aborting ...') + finally: + if worker: + logger.info("worker found ... calling exit") + worker.exit() + return 0 diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 63c38d8c5..d41ac97ec 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,7 @@ -from freqtrade.configuration.arguments import Arguments # noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials # noqa: F401 -from freqtrade.configuration.timerange import TimeRange # noqa: F401 -from freqtrade.configuration.configuration import Configuration # noqa: F401 -from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 +# flake8: noqa: F401 + +from freqtrade.configuration.config_setup import setup_utils_configuration +from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.timerange import TimeRange +from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.config_validation import validate_config_consistency diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py new file mode 100644 index 000000000..64f283e42 --- /dev/null +++ b/freqtrade/configuration/config_setup.py @@ -0,0 +1,25 @@ +import logging +from typing import Any, Dict + +from .config_validation import validate_config_consistency +from .configuration import Configuration +from .check_exchange import remove_credentials +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for utils subcommands + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args, method) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + remove_credentials(config) + validate_config_consistency(config) + + return config diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 43c1522e7..d275a80e3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,465 +1 @@ -# pragma pylint: disable=W0603 -""" Edge positioning package """ -import logging -from typing import Any, Dict, NamedTuple - -import arrow -import numpy as np -import utils_find_1st as utf1st -from pandas import DataFrame - -from freqtrade import constants -from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.exceptions import OperationalException -from freqtrade.strategy.interface import SellType - -logger = logging.getLogger(__name__) - - -class PairInfo(NamedTuple): - stoploss: float - winrate: float - risk_reward_ratio: float - required_risk_reward: float - expectancy: float - nb_trades: int - avg_trade_duration: float - - -class Edge: - """ - Calculates Win Rate, Risk Reward Ratio, Expectancy - against historical data for a give set of markets and a strategy - it then adjusts stoploss and position size accordingly - and force it into the strategy - Author: https://github.com/mishaker - """ - - config: Dict = {} - _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: - - self.config = config - self.exchange = exchange - self.strategy = strategy - - self.edge_config = self.config.get('edge', {}) - self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._final_pairs: list = [] - - # checking max_open_trades. it should be -1 as with Edge - # the number of trades is determined by position size - if self.config['max_open_trades'] != float('inf'): - logger.critical('max_open_trades should be -1 in config !') - - if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: - raise OperationalException('Edge works only with unlimited stake amount') - - # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future. - self._capital_percentage: float = self.edge_config.get( - 'capital_available_percentage', self.config['tradable_balance_ratio']) - self._allowed_risk: float = self.edge_config.get('allowed_risk') - self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) - self._last_updated: int = 0 # Timestamp of pairs last updated time - self._refresh_pairs = True - - self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) - self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) - self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - - # calculating stoploss range - self._stoploss_range = np.arange( - self._stoploss_range_min, - self._stoploss_range_max, - self._stoploss_range_step - ) - - self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) - if config.get('fee'): - self.fee = config['fee'] - else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) - - def calculate(self) -> bool: - pairs = self.config['exchange']['pair_whitelist'] - heartbeat = self.edge_config.get('process_throttle_secs') - - if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().timestamp): - return False - - data: Dict[str, Any] = {} - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using local backtesting data (using whitelist in given config) ...') - - if self._refresh_pairs: - history.refresh_data( - datadir=self.config['datadir'], - pairs=pairs, - exchange=self.exchange, - timeframe=self.strategy.ticker_interval, - timerange=self._timerange, - ) - - data = history.load_data( - datadir=self.config['datadir'], - pairs=pairs, - timeframe=self.strategy.ticker_interval, - timerange=self._timerange, - startup_candles=self.strategy.startup_candle_count, - data_format=self.config.get('dataformat_ohlcv', 'json'), - ) - - if not data: - # Reinitializing cached pairs - self._cached_pairs = {} - logger.critical("No data found. Edge is stopped ...") - return False - - preprocessed = self.strategy.tickerdata_to_dataframe(data) - - # Print timeframe - min_date, max_date = history.get_timerange(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days) ...', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - - trades: list = [] - for pair, pair_data in preprocessed.items(): - # Sorting dataframe by date and reset index - pair_data = pair_data.sort_values(by=['date']) - pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.strategy.advise_sell( - self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - - trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) - - # If no trade found then exit - if len(trades) == 0: - logger.info("No trades found.") - return False - - # Fill missing, calculable columns, profit, duration , abs etc. - trades_df = self._fill_calculable_fields(DataFrame(trades)) - self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().timestamp - - return True - - def stake_amount(self, pair: str, free_capital: float, - total_capital: float, capital_in_trade: float) -> float: - stoploss = self.stoploss(pair) - available_capital = (total_capital + capital_in_trade) * self._capital_percentage - allowed_capital_at_risk = available_capital * self._allowed_risk - max_position_size = abs(allowed_capital_at_risk / stoploss) - position_size = min(max_position_size, free_capital) - if pair in self._cached_pairs: - logger.info( - 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' - ' capital in trade: %s, free capital: %s, total capital: %s,' - ' stoploss: %s, available capital: %s.', - self._cached_pairs[pair].winrate, - self._cached_pairs[pair].expectancy, - position_size, pair, - capital_in_trade, free_capital, total_capital, - stoploss, available_capital - ) - return round(position_size, 15) - - def stoploss(self, pair: str) -> float: - if pair in self._cached_pairs: - return self._cached_pairs[pair].stoploss - else: - logger.warning('tried to access stoploss of a non-existing pair, ' - 'strategy stoploss is returned instead.') - return self.strategy.stoploss - - def adjust(self, pairs) -> list: - """ - Filters out and sorts "pairs" according to Edge calculated pairs - """ - final = [] - for pair, info in self._cached_pairs.items(): - if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ - pair in pairs: - final.append(pair) - - if self._final_pairs != final: - self._final_pairs = final - if self._final_pairs: - logger.info( - 'Minimum expectancy and minimum winrate are met only for %s,' - ' so other pairs are filtered out.', - self._final_pairs - ) - else: - logger.info( - 'Edge removed all pairs as no pair with minimum expectancy ' - 'and minimum winrate was found !' - ) - - return self._final_pairs - - def accepted_pairs(self) -> list: - """ - return a list of accepted pairs along with their winrate, expectancy and stoploss - """ - final = [] - for pair, info in self._cached_pairs.items(): - if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): - final.append({ - 'Pair': pair, - 'Winrate': info.winrate, - 'Expectancy': info.expectancy, - 'Stoploss': info.stoploss, - }) - return final - - def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: - """ - The result frame contains a number of columns that are calculable - from other columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param result Dataframe - :return: result Dataframe - """ - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - # we set stake amount to an arbitrary amount. - # as it doesn't change the calculation. - # all returned values are relative. they are percentages. - stake = 0.015 - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - result['trade_duration'] = result['close_time'] - result['open_time'] - - result['trade_duration'] = result['trade_duration'].map( - lambda x: int(x.total_seconds() / 60)) - - # Spends, Takes, Profit, Absolute Profit - - # Buy Price - result['buy_vol'] = stake / result['open_rate'] # How many target are we buying - result['buy_fee'] = stake * open_fee - result['buy_spend'] = stake + result['buy_fee'] # How much we're spending - - # Sell price - result['sell_sum'] = result['buy_vol'] * result['close_rate'] - result['sell_fee'] = result['sell_sum'] * close_fee - result['sell_take'] = result['sell_sum'] - result['sell_fee'] - - # profit_percent - result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] - - # Absolute profit - result['profit_abs'] = result['sell_take'] - result['buy_spend'] - - return result - - def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: - """ - This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs - The calulation will be done per pair and per strategy. - """ - # Removing pairs having less than min_trades_number - min_trades_number = self.edge_config.get('min_trade_number', 10) - results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) - ################################### - - # Removing outliers (Only Pumps) from the dataset - # The method to detect outliers is to calculate standard deviation - # Then every value more than (standard deviation + 2*average) is out (pump) - # - # Removing Pumps - if self.edge_config.get('remove_pumps', False): - results = results.groupby(['pair', 'stoploss']).apply( - lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) - ########################################################################## - - # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) - results = results[results.trade_duration < max_trade_duration] - ####################################################################### - - if results.empty: - return {} - - groupby_aggregator = { - 'profit_abs': [ - ('nb_trades', 'count'), # number of all trades - ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades - ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades - ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades - ], - 'trade_duration': [('avg_trade_duration', 'mean')] - } - - # Group by (pair and stoploss) by applying above aggregator - df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( - groupby_aggregator).reset_index(col_level=1) - - # Dropping level 0 as we don't need it - df.columns = df.columns.droplevel(0) - - # Calculating number of losing trades, average win and average loss - df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] - df['average_win'] = df['profit_sum'] / df['nb_win_trades'] - df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] - - # Win rate = number of profitable trades / number of trades - df['winrate'] = df['nb_win_trades'] / df['nb_trades'] - - # risk_reward_ratio = average win / average loss - df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] - - # required_risk_reward = (1 / winrate) - 1 - df['required_risk_reward'] = (1 / df['winrate']) - 1 - - # expectancy = (risk_reward_ratio * winrate) - (lossrate) - df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) - - # sort by expectancy and stoploss - df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( - 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - - final = {} - for x in df.itertuples(): - final[x.pair] = PairInfo( - x.stoploss, - x.winrate, - x.risk_reward_ratio, - x.required_risk_reward, - x.expectancy, - x.nb_trades, - x.avg_trade_duration - ) - - # Returning a list of pairs in order of "expectancy" - return final - - def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): - buy_column = ticker_data['buy'].values - sell_column = ticker_data['sell'].values - date_column = ticker_data['date'].values - ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values - - result: list = [] - for stoploss in stoploss_range: - result += self._detect_next_stop_or_sell_point( - buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair - ) - - return result - - def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, - ohlc_columns, stoploss, pair): - """ - Iterate through ohlc_columns in order to find the next trade - Next trade opens from the first buy signal noticed to - The sell or stoploss signal after it. - It then cuts OHLC, buy_column, sell_column and date_column. - Cut from (the exit trade index) + 1. - - Author: https://github.com/mishaker - """ - - result: list = [] - start_point = 0 - - while True: - open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - - # Return empty if we don't find trade entry (i.e. buy==1) or - # we find a buy but at the end of array - if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: - break - else: - # When a buy signal is seen, - # trade opens in reality on the next candle - open_trade_index += 1 - - stop_price_percentage = stoploss + 1 - open_price = ohlc_columns[open_trade_index, 0] - stop_price = (open_price * stop_price_percentage) - - # Searching for the index where stoploss is hit - stop_index = utf1st.find_1st( - ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) - - # If we don't find it then we assume stop_index will be far in future (infinite number) - if stop_index == -1: - stop_index = float('inf') - - # Searching for the index where sell is hit - sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) - - # If we don't find it then we assume sell_index will be far in future (infinite number) - if sell_index == -1: - sell_index = float('inf') - - # Check if we don't find any stop or sell point (in that case trade remains open) - # It is not interesting for Edge to consider it so we simply ignore the trade - # And stop iterating there is no more entry - if stop_index == sell_index == float('inf'): - break - - if stop_index <= sell_index: - exit_index = open_trade_index + stop_index - exit_type = SellType.STOP_LOSS - exit_price = stop_price - elif stop_index > sell_index: - # If exit is SELL then we exit at the next candle - exit_index = open_trade_index + sell_index + 1 - - # Check if we have the next candle - if len(ohlc_columns) - 1 < exit_index: - break - - exit_type = SellType.SELL_SIGNAL - exit_price = ohlc_columns[exit_index, 0] - - trade = {'pair': pair, - 'stoploss': stoploss, - 'profit_percent': '', - 'profit_abs': '', - 'open_time': date_column[open_trade_index], - 'close_time': date_column[exit_index], - 'open_index': start_point + open_trade_index, - 'close_index': start_point + exit_index, - 'trade_duration': '', - 'open_rate': round(open_price, 15), - 'close_rate': round(exit_price, 15), - 'exit_type': exit_type - } - - result.append(trade) - - # Giving a view of exit_index till the end of array - buy_column = buy_column[exit_index:] - sell_column = sell_column[exit_index:] - date_column = date_column[exit_index:] - ohlc_columns = ohlc_columns[exit_index:] - start_point += exit_index - - return result +from .edge_positioning import Edge, PairInfo # noqa: F401 diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py new file mode 100644 index 000000000..43c1522e7 --- /dev/null +++ b/freqtrade/edge/edge_positioning.py @@ -0,0 +1,465 @@ +# pragma pylint: disable=W0603 +""" Edge positioning package """ +import logging +from typing import Any, Dict, NamedTuple + +import arrow +import numpy as np +import utils_find_1st as utf1st +from pandas import DataFrame + +from freqtrade import constants +from freqtrade.configuration import TimeRange +from freqtrade.data import history +from freqtrade.exceptions import OperationalException +from freqtrade.strategy.interface import SellType + +logger = logging.getLogger(__name__) + + +class PairInfo(NamedTuple): + stoploss: float + winrate: float + risk_reward_ratio: float + required_risk_reward: float + expectancy: float + nb_trades: int + avg_trade_duration: float + + +class Edge: + """ + Calculates Win Rate, Risk Reward Ratio, Expectancy + against historical data for a give set of markets and a strategy + it then adjusts stoploss and position size accordingly + and force it into the strategy + Author: https://github.com/mishaker + """ + + config: Dict = {} + _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: + + self.config = config + self.exchange = exchange + self.strategy = strategy + + self.edge_config = self.config.get('edge', {}) + self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + self._final_pairs: list = [] + + # checking max_open_trades. it should be -1 as with Edge + # the number of trades is determined by position size + if self.config['max_open_trades'] != float('inf'): + logger.critical('max_open_trades should be -1 in config !') + + if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: + raise OperationalException('Edge works only with unlimited stake amount') + + # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future. + self._capital_percentage: float = self.edge_config.get( + 'capital_available_percentage', self.config['tradable_balance_ratio']) + self._allowed_risk: float = self.edge_config.get('allowed_risk') + self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) + self._last_updated: int = 0 # Timestamp of pairs last updated time + self._refresh_pairs = True + + self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + # calculating stoploss range + self._stoploss_range = np.arange( + self._stoploss_range_min, + self._stoploss_range_max, + self._stoploss_range_step + ) + + self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( + days=-1 * self._since_number_of_days).format('YYYYMMDD')) + if config.get('fee'): + self.fee = config['fee'] + else: + self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + + def calculate(self) -> bool: + pairs = self.config['exchange']['pair_whitelist'] + heartbeat = self.edge_config.get('process_throttle_secs') + + if (self._last_updated > 0) and ( + self._last_updated + heartbeat > arrow.utcnow().timestamp): + return False + + data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using local backtesting data (using whitelist in given config) ...') + + if self._refresh_pairs: + history.refresh_data( + datadir=self.config['datadir'], + pairs=pairs, + exchange=self.exchange, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + ) + + data = history.load_data( + datadir=self.config['datadir'], + pairs=pairs, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + startup_candles=self.strategy.startup_candle_count, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) + + if not data: + # Reinitializing cached pairs + self._cached_pairs = {} + logger.critical("No data found. Edge is stopped ...") + return False + + preprocessed = self.strategy.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = history.get_timerange(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days) ...', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + + trades: list = [] + for pair, pair_data in preprocessed.items(): + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + + ticker_data = self.strategy.advise_sell( + self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + + trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) + + # If no trade found then exit + if len(trades) == 0: + logger.info("No trades found.") + return False + + # Fill missing, calculable columns, profit, duration , abs etc. + trades_df = self._fill_calculable_fields(DataFrame(trades)) + self._cached_pairs = self._process_expectancy(trades_df) + self._last_updated = arrow.utcnow().timestamp + + return True + + def stake_amount(self, pair: str, free_capital: float, + total_capital: float, capital_in_trade: float) -> float: + stoploss = self.stoploss(pair) + available_capital = (total_capital + capital_in_trade) * self._capital_percentage + allowed_capital_at_risk = available_capital * self._allowed_risk + max_position_size = abs(allowed_capital_at_risk / stoploss) + position_size = min(max_position_size, free_capital) + if pair in self._cached_pairs: + logger.info( + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' capital in trade: %s, free capital: %s, total capital: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, + position_size, pair, + capital_in_trade, free_capital, total_capital, + stoploss, available_capital + ) + return round(position_size, 15) + + def stoploss(self, pair: str) -> float: + if pair in self._cached_pairs: + return self._cached_pairs[pair].stoploss + else: + logger.warning('tried to access stoploss of a non-existing pair, ' + 'strategy stoploss is returned instead.') + return self.strategy.stoploss + + def adjust(self, pairs) -> list: + """ + Filters out and sorts "pairs" according to Edge calculated pairs + """ + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ + pair in pairs: + final.append(pair) + + if self._final_pairs != final: + self._final_pairs = final + if self._final_pairs: + logger.info( + 'Minimum expectancy and minimum winrate are met only for %s,' + ' so other pairs are filtered out.', + self._final_pairs + ) + else: + logger.info( + 'Edge removed all pairs as no pair with minimum expectancy ' + 'and minimum winrate was found !' + ) + + return self._final_pairs + + def accepted_pairs(self) -> list: + """ + return a list of accepted pairs along with their winrate, expectancy and stoploss + """ + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): + final.append({ + 'Pair': pair, + 'Winrate': info.winrate, + 'Expectancy': info.expectancy, + 'Stoploss': info.stoploss, + }) + return final + + def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: + """ + The result frame contains a number of columns that are calculable + from other columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param result Dataframe + :return: result Dataframe + """ + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + # we set stake amount to an arbitrary amount. + # as it doesn't change the calculation. + # all returned values are relative. they are percentages. + stake = 0.015 + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + result['trade_duration'] = result['close_time'] - result['open_time'] + + result['trade_duration'] = result['trade_duration'].map( + lambda x: int(x.total_seconds() / 60)) + + # Spends, Takes, Profit, Absolute Profit + + # Buy Price + result['buy_vol'] = stake / result['open_rate'] # How many target are we buying + result['buy_fee'] = stake * open_fee + result['buy_spend'] = stake + result['buy_fee'] # How much we're spending + + # Sell price + result['sell_sum'] = result['buy_vol'] * result['close_rate'] + result['sell_fee'] = result['sell_sum'] * close_fee + result['sell_take'] = result['sell_sum'] - result['sell_fee'] + + # profit_percent + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + + # Absolute profit + result['profit_abs'] = result['sell_take'] - result['buy_spend'] + + return result + + def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: + """ + This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs + The calulation will be done per pair and per strategy. + """ + # Removing pairs having less than min_trades_number + min_trades_number = self.edge_config.get('min_trade_number', 10) + results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) + ################################### + + # Removing outliers (Only Pumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # + # Removing Pumps + if self.edge_config.get('remove_pumps', False): + results = results.groupby(['pair', 'stoploss']).apply( + lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + if results.empty: + return {} + + groupby_aggregator = { + 'profit_abs': [ + ('nb_trades', 'count'), # number of all trades + ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades + ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades + ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades + ], + 'trade_duration': [('avg_trade_duration', 'mean')] + } + + # Group by (pair and stoploss) by applying above aggregator + df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + groupby_aggregator).reset_index(col_level=1) + + # Dropping level 0 as we don't need it + df.columns = df.columns.droplevel(0) + + # Calculating number of losing trades, average win and average loss + df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] + df['average_win'] = df['profit_sum'] / df['nb_win_trades'] + df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + + # Win rate = number of profitable trades / number of trades + df['winrate'] = df['nb_win_trades'] / df['nb_trades'] + + # risk_reward_ratio = average win / average loss + df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] + + # required_risk_reward = (1 / winrate) - 1 + df['required_risk_reward'] = (1 / df['winrate']) - 1 + + # expectancy = (risk_reward_ratio * winrate) - (lossrate) + df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) + + # sort by expectancy and stoploss + df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() + + final = {} + for x in df.itertuples(): + final[x.pair] = PairInfo( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy, + x.nb_trades, + x.avg_trade_duration + ) + + # Returning a list of pairs in order of "expectancy" + return final + + def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): + buy_column = ticker_data['buy'].values + sell_column = ticker_data['sell'].values + date_column = ticker_data['date'].values + ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values + + result: list = [] + for stoploss in stoploss_range: + result += self._detect_next_stop_or_sell_point( + buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair + ) + + return result + + def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, + ohlc_columns, stoploss, pair): + """ + Iterate through ohlc_columns in order to find the next trade + Next trade opens from the first buy signal noticed to + The sell or stoploss signal after it. + It then cuts OHLC, buy_column, sell_column and date_column. + Cut from (the exit trade index) + 1. + + Author: https://github.com/mishaker + """ + + result: list = [] + start_point = 0 + + while True: + open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) + + # Return empty if we don't find trade entry (i.e. buy==1) or + # we find a buy but at the end of array + if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: + break + else: + # When a buy signal is seen, + # trade opens in reality on the next candle + open_trade_index += 1 + + stop_price_percentage = stoploss + 1 + open_price = ohlc_columns[open_trade_index, 0] + stop_price = (open_price * stop_price_percentage) + + # Searching for the index where stoploss is hit + stop_index = utf1st.find_1st( + ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) + + # If we don't find it then we assume stop_index will be far in future (infinite number) + if stop_index == -1: + stop_index = float('inf') + + # Searching for the index where sell is hit + sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) + + # If we don't find it then we assume sell_index will be far in future (infinite number) + if sell_index == -1: + sell_index = float('inf') + + # Check if we don't find any stop or sell point (in that case trade remains open) + # It is not interesting for Edge to consider it so we simply ignore the trade + # And stop iterating there is no more entry + if stop_index == sell_index == float('inf'): + break + + if stop_index <= sell_index: + exit_index = open_trade_index + stop_index + exit_type = SellType.STOP_LOSS + exit_price = stop_price + elif stop_index > sell_index: + # If exit is SELL then we exit at the next candle + exit_index = open_trade_index + sell_index + 1 + + # Check if we have the next candle + if len(ohlc_columns) - 1 < exit_index: + break + + exit_type = SellType.SELL_SIGNAL + exit_price = ohlc_columns[exit_index, 0] + + trade = {'pair': pair, + 'stoploss': stoploss, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': date_column[open_trade_index], + 'close_time': date_column[exit_index], + 'open_index': start_point + open_trade_index, + 'close_index': start_point + exit_index, + 'trade_duration': '', + 'open_rate': round(open_price, 15), + 'close_rate': round(exit_price, 15), + 'exit_type': exit_type + } + + result.append(trade) + + # Giving a view of exit_index till the end of array + buy_column = buy_column[exit_index:] + sell_column = sell_column[exit_index:] + date_column = date_column[exit_index:] + ohlc_columns = ohlc_columns[exit_index:] + start_point += exit_index + + return result diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index acff2b2ad..e3856e200 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,6 +7,7 @@ import traceback from datetime import datetime from math import isclose from os import getpid +from threading import Lock from typing import Any, Dict, List, Optional, Tuple import arrow @@ -27,7 +28,6 @@ from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -92,6 +92,8 @@ class FreqtradeBot: # the initial state of the bot. # Keep this at the end of this initialization method. self.rpc: RPCManager = RPCManager(self) + # Protect sell-logic from forcesell and viceversa + self._sell_lock = Lock() def cleanup(self) -> None: """ @@ -132,8 +134,12 @@ class FreqtradeBot: self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist), self.strategy.informative_pairs()) - # First process current opened trades (positions) - self.exit_positions(trades) + # Protect from collisions with forcesell. + # Without this, freqtrade my try to recreate stoploss_on_exchange orders + # while selling is in process, since telegram messages arrive in an different thread. + with self._sell_lock: + # First process current opened trades (positions) + self.exit_positions(trades) # Then looking for buy opportunities if self.get_free_open_trades(): @@ -218,7 +224,7 @@ class FreqtradeBot: return trades_created - def get_target_bid(self, pair: str, tick: Dict = None) -> float: + def get_buy_rate(self, pair: str, tick: Dict = None) -> float: """ Calculates bid target between current ask price and last price :return: float: Price @@ -435,7 +441,7 @@ class FreqtradeBot: buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.get_target_bid(pair) + buy_limit_requested = self.get_buy_rate(pair) min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -748,8 +754,8 @@ class FreqtradeBot: Check and execute sell """ should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.utcnow(), buy, sell, - force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 + trade, sell_rate, datetime.utcnow(), buy, sell, + force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) if should_sell.sell_flag: diff --git a/freqtrade/main.py b/freqtrade/main.py index 811e29864..a75eeebed 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -14,7 +14,7 @@ if sys.version_info < (3, 6): import logging from typing import Any, List -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments logger = logging.getLogger('freqtrade') diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 34760372f..e69de29bb 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,102 +0,0 @@ -import logging -from typing import Any, Dict - -from freqtrade import constants -from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration - -logger = logging.getLogger(__name__) - - -def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: - """ - Prepare the configuration for the Hyperopt module - :param args: Cli args from Arguments() - :return: Configuration - """ - config = setup_utils_configuration(args, method) - - if method == RunMode.BACKTEST: - if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: - raise DependencyException('stake amount could not be "%s" for backtesting' % - constants.UNLIMITED_STAKE_AMOUNT) - - return config - - -def start_backtesting(args: Dict[str, Any]) -> None: - """ - Start Backtesting script - :param args: Cli args from Arguments() - :return: None - """ - # Import here to avoid loading backtesting module when it's not used - from freqtrade.optimize.backtesting import Backtesting - - # Initialize configuration - config = setup_configuration(args, RunMode.BACKTEST) - - logger.info('Starting freqtrade in Backtesting mode') - - # Initialize backtesting object - backtesting = Backtesting(config) - backtesting.start() - - -def start_hyperopt(args: Dict[str, Any]) -> None: - """ - Start hyperopt script - :param args: Cli args from Arguments() - :return: None - """ - # Import here to avoid loading hyperopt module when it's not used - try: - from filelock import FileLock, Timeout - from freqtrade.optimize.hyperopt import Hyperopt - except ImportError as e: - raise OperationalException( - f"{e}. Please ensure that the hyperopt dependencies are installed.") from e - # Initialize configuration - config = setup_configuration(args, RunMode.HYPEROPT) - - logger.info('Starting freqtrade in Hyperopt mode') - - lock = FileLock(Hyperopt.get_lock_filename(config)) - - try: - with lock.acquire(timeout=1): - - # Remove noisy log messages - logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) - logging.getLogger('filelock').setLevel(logging.WARNING) - - # Initialize backtesting object - hyperopt = Hyperopt(config) - hyperopt.start() - - except Timeout: - logger.info("Another running instance of freqtrade Hyperopt detected.") - logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. " - "Hyperopt module is resource hungry. Please run your Hyperopt sequentially " - "or on separate machines.") - logger.info("Quitting now.") - # TODO: return False here in order to help freqtrade to exit - # with non-zero exit code... - # Same in Edge and Backtesting start() functions. - - -def start_edge(args: Dict[str, Any]) -> None: - """ - Start Edge script - :param args: Cli args from Arguments() - :return: None - """ - from freqtrade.optimize.edge_cli import EdgeCli - # Initialize configuration - config = setup_configuration(args, RunMode.EDGE) - logger.info('Starting freqtrade in Edge mode') - - # Initialize Edge object - edge_cli = EdgeCli(config) - edge_cli.start() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0bc3dd6c0..25b7de9dc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -281,30 +281,28 @@ class Backtesting: return bt_res return None - def backtest(self, args: Dict) -> DataFrame: + def backtest(self, processed: Dict, stake_amount: float, + start_date, end_date, + max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: """ - Implements backtesting functionality + Implement backtesting functionality NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. Of course try to not have ugly code. By some accessor are sometime slower than functions. - Avoid, logging on this method + Avoid extensive logging in this method and functions it calls. - :param args: a dict containing: - stake_amount: btc amount to use for each trade - processed: a processed dictionary with format {pair, data} - max_open_trades: maximum number of concurrent trades (default: 0, disabled) - position_stacking: do we allow position stacking? (default: False) - :return: DataFrame + :param processed: a processed dictionary with format {pair, data} + :param stake_amount: amount to use for each trade + :param start_date: backtesting timerange start datetime + :param end_date: backtesting timerange end datetime + :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited + :param position_stacking: do we allow position stacking? + :return: DataFrame with trades (results of backtesting) """ - # Arguments are long and noisy, so this is commented out. - # Uncomment if you need to debug the backtest() method. - # logger.debug(f"Start backtest, args: {args}") - processed = args['processed'] - stake_amount = args['stake_amount'] - max_open_trades = args.get('max_open_trades', 0) - position_stacking = args.get('position_stacking', False) - start_date = args['start_date'] - end_date = args['end_date'] + logger.debug(f"Run backtest, stake_amount: {stake_amount}, " + f"start_date: {start_date}, end_date: {end_date}, " + f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" + ) trades = [] trade_count_lock: Dict = {} @@ -371,18 +369,21 @@ class Backtesting: def start(self) -> None: """ - Run a backtesting end-to-end + Run backtesting end-to-end :return: None """ data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + position_stacking = self.config.get('position_stacking', False) data, timerange = self.load_bt_data() @@ -405,14 +406,12 @@ class Backtesting: ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - 'start_date': min_date, - 'end_date': max_date, - } + processed=preprocessed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=max_open_trades, + position_stacking=position_stacking, ) for strategy, results in all_results.items(): diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 90ae209c7..b49db10f7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -373,14 +373,12 @@ class Hyperopt: min_date, max_date = get_timerange(processed) backtesting_results = self.backtesting.backtest( - { - 'stake_amount': self.config['stake_amount'], - 'processed': processed, - 'max_open_trades': self.max_open_trades, - 'position_stacking': self.position_stacking, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=self.max_open_trades, + position_stacking=self.position_stacking, ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 3801546b1..67056eaa9 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -70,7 +70,7 @@ def generate_text_table_sell_reason(data: Dict[str, Dict], results: DataFrame) - for reason, count in results['sell_reason'].value_counts().iteritems(): result = results.loc[results['sell_reason'] == reason] profit = len(result[result['profit_abs'] >= 0]) - loss = len(result[results['profit_abs'] < 0]) + loss = len(result[result['profit_abs'] < 0]) profit_mean = round(result['profit_percent'].mean() * 100.0, 2) tabular_data.append([reason.value, count, profit, loss, profit_mean]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d58b99f39..41097c211 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -420,24 +420,27 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - if trade_id == 'all': - # Execute sell for all open orders - for trade in Trade.get_open_trades(): - _exec_forcesell(trade) + with self._freqtrade._sell_lock: + if trade_id == 'all': + # Execute sell for all open orders + for trade in Trade.get_open_trades(): + _exec_forcesell(trade) + Trade.session.flush() + self._freqtrade.wallets.update() + return {'result': 'Created sell orders for all open trades.'} + + # Query for trade + trade = Trade.get_trades( + trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] + ).first() + if not trade: + logger.warning('forcesell: Invalid argument received') + raise RPCException('invalid argument') + + _exec_forcesell(trade) Trade.session.flush() - return {'result': 'Created sell orders for all open trades.'} - - # Query for trade - trade = Trade.get_trades( - trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] - ).first() - if not trade: - logger.warning('forcesell: Invalid argument received') - raise RPCException('invalid argument') - - _exec_forcesell(trade) - Trade.session.flush() - return {'result': f'Created sell order for trade {trade_id}.'} + self._freqtrade.wallets.update() + return {'result': f'Created sell order for trade {trade_id}.'} def _rpc_forcebuy(self, pair: str, price: Optional[float]) -> Optional[Trade]: """ diff --git a/freqtrade/templates/sample_hyperopt_loss.py b/freqtrade/templates/sample_hyperopt_loss.py index 5a2fb72b6..4173d97f5 100644 --- a/freqtrade/templates/sample_hyperopt_loss.py +++ b/freqtrade/templates/sample_hyperopt_loss.py @@ -27,7 +27,8 @@ class SampleHyperOptLoss(IHyperOptLoss): Defines the default loss function for hyperopt This is intended to give you some inspiration for your own loss function. - The Function needs to return a number (float) - which becomes for better backtest results. + The Function needs to return a number (float) - which becomes smaller for better backtest + results. """ @staticmethod diff --git a/freqtrade/utils.py b/freqtrade/utils.py deleted file mode 100644 index 4f1147691..000000000 --- a/freqtrade/utils.py +++ /dev/null @@ -1,505 +0,0 @@ -import csv -import logging -import sys -from collections import OrderedDict -from operator import itemgetter -from pathlib import Path -from typing import Any, Dict, List - -import arrow -import rapidjson -from colorama import init as colorama_init -from tabulate import tabulate - -from freqtrade.configuration import (Configuration, TimeRange, - remove_credentials, - validate_config_consistency) -from freqtrade.configuration.directory_operations import (copy_sample_files, - create_userdata_dir) -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.converter import (convert_ohlcv_format, - convert_trades_format) -from freqtrade.data.history import (convert_trades_to_ohlcv, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data) -from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) -from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.state import RunMode - -logger = logging.getLogger(__name__) - - -def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: - """ - Prepare the configuration for utils subcommands - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args, method) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - remove_credentials(config) - validate_config_consistency(config) - - return config - - -def start_trading(args: Dict[str, Any]) -> int: - """ - Main entry point for trading mode - """ - from freqtrade.worker import Worker - # Load and run worker - worker = None - try: - worker = Worker(args) - worker.run() - except KeyboardInterrupt: - logger.info('SIGINT received, aborting ...') - finally: - if worker: - logger.info("worker found ... calling exit") - worker.exit() - return 0 - - -def start_list_exchanges(args: Dict[str, Any]) -> None: - """ - Print available exchanges - :param args: Cli args from Arguments() - :return: None - """ - exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() - if args['print_one_column']: - print('\n'.join(exchanges)) - else: - if args['list_exchanges_all']: - print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") - else: - print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") - - -def start_create_userdir(args: Dict[str, Any]) -> None: - """ - Create "user_data" directory to contain user data strategies, hyperopt, ...) - :param args: Cli args from Arguments() - :return: None - """ - if "user_data_dir" in args and args["user_data_dir"]: - userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) - copy_sample_files(userdir, overwrite=args["reset"]) - else: - logger.warning("`create-userdir` requires --userdir to be set.") - sys.exit(1) - - -def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): - """ - Deploy new strategy from template to strategy_path - """ - indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) - buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) - sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) - plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_strategy.py.j2', - arguments={"strategy": strategy_name, - "indicators": indicators, - "buy_trend": buy_trend, - "sell_trend": sell_trend, - "plot_config": plot_config, - }) - - logger.info(f"Writing strategy to `{strategy_path}`.") - strategy_path.write_text(strategy_text) - - -def start_new_strategy(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "strategy" in args and args["strategy"]: - if args["strategy"] == "DefaultStrategy": - raise OperationalException("DefaultStrategy is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - - deploy_new_strategy(args['strategy'], new_path, args['template']) - - else: - raise OperationalException("`new-strategy` requires --strategy to be set.") - - -def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): - """ - Deploys a new hyperopt template to hyperopt_path - """ - buy_guards = render_template( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) - sell_guards = render_template( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) - buy_space = render_template( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) - sell_space = render_template( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "hyperopt" in args and args["hyperopt"]: - if args["hyperopt"] == "DefaultHyperopt": - raise OperationalException("DefaultHyperopt is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - -def start_download_data(args: Dict[str, Any]) -> None: - """ - Download data (former download_backtest_data.py script) - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - timerange = TimeRange() - if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") - - logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {config["datadir"]}') - - pairs_not_available: List[str] = [] - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) - try: - - if config.get('download_trades'): - pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=config.get("erase"), - data_format=config['dataformat_trades']) - - # Convert downloaded trade data to different timeframes - convert_trades_to_ohlcv( - pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase"), - data_format_ohlcv=config['dataformat_ohlcv'], - data_format_trades=config['dataformat_trades'], - ) - - else: - pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase"), - data_format=config['dataformat_ohlcv']) - - except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - - finally: - if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {exchange.name}.") - - -def start_list_strategies(args: Dict[str, Any]) -> None: - """ - Print Strategies available in a directory - """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) - strategies = StrategyResolver.search_all_objects(directory) - # Sort alphabetically - strategies = sorted(strategies, key=lambda x: x['name']) - strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] - - if args['print_one_column']: - print('\n'.join([s['name'] for s in strategies])) - else: - print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) - - -def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: - """ - Convert data from one format to another - """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - if ohlcv: - convert_ohlcv_format(config, - convert_from=args['format_from'], convert_to=args['format_to'], - erase=args['erase']) - else: - convert_trades_format(config, - convert_from=args['format_from'], convert_to=args['format_to'], - erase=args['erase']) - - -def start_list_timeframes(args: Dict[str, Any]) -> None: - """ - Print ticker intervals (timeframes) available on Exchange - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - # Do not use ticker_interval set in the config - config['ticker_interval'] = None - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - if args['print_one_column']: - print('\n'.join(exchange.timeframes)) - else: - print(f"Timeframes available for the exchange `{exchange.name}`: " - f"{', '.join(exchange.timeframes)}") - - -def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: - """ - Print pairs/markets on the exchange - :param args: Cli args from Arguments() - :param pairs_only: if True print only pairs, otherwise print all instruments (markets) - :return: None - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - # By default only active pairs/markets are to be shown - active_only = not args.get('list_pairs_all', False) - - base_currencies = args.get('base_currencies', []) - quote_currencies = args.get('quote_currencies', []) - - try: - pairs = exchange.get_markets(base_currencies=base_currencies, - quote_currencies=quote_currencies, - pairs_only=pairs_only, - active_only=active_only) - # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) - except Exception as e: - raise OperationalException(f"Cannot get markets. Reason: {e}") from e - - else: - summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + - ("active " if active_only else "") + - (plural(len(pairs), "pair" if pairs_only else "market")) + - (f" with {', '.join(base_currencies)} as base " - f"{plural(len(base_currencies), 'currency', 'currencies')}" - if base_currencies else "") + - (" and" if base_currencies and quote_currencies else "") + - (f" with {', '.join(quote_currencies)} as quote " - f"{plural(len(quote_currencies), 'currency', 'currencies')}" - if quote_currencies else "")) - - headers = ["Id", "Symbol", "Base", "Quote", "Active", - *(['Is pair'] if not pairs_only else [])] - - tabular_data = [] - for _, v in pairs.items(): - tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], - 'Base': v['base'], 'Quote': v['quote'], - 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v['symbol'])} - if not pairs_only else {})}) - - if (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - # Print summary string in the log in case of machine-readable - # regular formats. - logger.info(f"{summary_str}.") - else: - # Print empty string separating leading logs and output in case of - # human-readable formats. - print() - - if len(pairs): - if args.get('print_list', False): - # print data as a list, with human-readable summary - print(f"{summary_str}: {', '.join(pairs.keys())}.") - elif args.get('print_one_column', False): - print('\n'.join(pairs.keys())) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairs.keys()), default=str)) - elif args.get('print_csv', False): - writer = csv.DictWriter(sys.stdout, fieldnames=headers) - writer.writeheader() - writer.writerows(tabular_data) - else: - # print data as a table, with the human-readable summary - print(f"{summary_str}:") - print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) - elif not (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - print(f"{summary_str}.") - - -def start_test_pairlist(args: Dict[str, Any]) -> None: - """ - Test Pairlist configuration - """ - from freqtrade.pairlist.pairlistmanager import PairListManager - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - quote_currencies = args.get('quote_currencies') - if not quote_currencies: - quote_currencies = [config.get('stake_currency')] - results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - # Do not use ticker_interval set in the config - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist - - for curr, pairlist in results.items(): - if not args.get('print_one_column', False): - print(f"Pairs for {curr}: ") - - if args.get('print_one_column', False): - print('\n'.join(pairlist)) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairlist), default=str)) - else: - print(pairlist) - - -def start_hyperopt_list(args: Dict[str, Any]) -> None: - """ - List hyperopt epochs previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - print_colorized = config.get('print_colorized', False) - print_json = config.get('print_json', False) - no_details = config.get('hyperopt_list_no_details', False) - no_header = False - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - - # TODO: fetch the interval for epochs to print from the cli option - epoch_start, epoch_stop = 0, None - - if print_colorized: - colorama_init(autoreset=True) - - try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) - - except KeyboardInterrupt: - print('User interrupted..') - - if trials and not no_details: - sorted_trials = sorted(trials, key=itemgetter('loss')) - results = sorted_trials[0] - Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) - - -def start_hyperopt_show(args: Dict[str, Any]) -> None: - """ - Show details of a hyperopt epoch previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - no_header = config.get('hyperopt_show_no_header', False) - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - trials_epochs = len(trials) - - n = config.get('hyperopt_show_index', -1) - if n > trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") - if n < -trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") - - # Translate epoch index from human-readable format to pythonic - if n > 0: - n -= 1 - - print_json = config.get('print_json', False) - - if trials: - val = trials[n] - Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") - - -def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: - """ - Filter our items from the list of hyperopt results - """ - if only_best: - trials = [x for x in trials if x['is_best']] - if only_profitable: - trials = [x for x in trials if x['results_metrics']['profit'] > 0] - - logger.info(f"{len(trials)} " + - ("best " if only_best else "") + - ("profitable " if only_profitable else "") + - "epochs found.") - - return trials diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils.py b/tests/commands/test_commands.py similarity index 93% rename from tests/test_utils.py rename to tests/commands/test_commands.py index 0c2e2aab3..eb2b8e609 100644 --- a/tests/test_utils.py +++ b/tests/commands/test_commands.py @@ -4,15 +4,16 @@ from unittest.mock import MagicMock, PropertyMock import pytest +from freqtrade.commands import (start_convert_data, start_create_userdir, + start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_test_pairlist, + start_trading) +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import (setup_utils_configuration, start_convert_data, - start_create_userdir, start_download_data, - start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_markets, - start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_test_pairlist, start_trading) from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -451,8 +452,8 @@ def test_create_datadir(caplog, mocker): # Added assert here to analyze random test-failures ... assert len(caplog.record_tuples) == 0 - cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock()) + cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) + csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) args = [ "create-userdir", "--userdir", @@ -538,7 +539,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +557,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +575,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +595,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +614,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( @@ -639,7 +640,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent / "strategy"), + str(Path(__file__).parent.parent / "strategy"), "-1" ] pargs = get_args(args) @@ -654,7 +655,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent / "strategy"), + str(Path(__file__).parent.parent / "strategy"), ] pargs = get_args(args) # pargs['config'] = None @@ -665,11 +666,11 @@ def test_start_list_strategies(mocker, caplog, capsys): assert "DefaultStrategy" in captured.out -def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys): +def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): + patch_exchange(mocker, mock_markets=True) mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), - get_tickers=tickers + get_tickers=tickers, ) default_conf['pairlists'] = [ @@ -827,8 +828,8 @@ def test_hyperopt_show(mocker, capsys, hyperopt_results): def test_convert_data(mocker, testdatadir): - ohlcv_mock = mocker.patch("freqtrade.utils.convert_ohlcv_format", MagicMock()) - trades_mock = mocker.patch("freqtrade.utils.convert_trades_format", MagicMock()) + ohlcv_mock = mocker.patch("freqtrade.commands.data_commands.convert_ohlcv_format") + trades_mock = mocker.patch("freqtrade.commands.data_commands.convert_trades_format") args = [ "convert-data", "--format-from", @@ -849,8 +850,8 @@ def test_convert_data(mocker, testdatadir): def test_convert_data_trades(mocker, testdatadir): - ohlcv_mock = mocker.patch("freqtrade.utils.convert_ohlcv_format", MagicMock()) - trades_mock = mocker.patch("freqtrade.utils.convert_trades_format", MagicMock()) + ohlcv_mock = mocker.patch("freqtrade.commands.data_commands.convert_ohlcv_format") + trades_mock = mocker.patch("freqtrade.commands.data_commands.convert_trades_format") args = [ "convert-trade-data", "--format-from", diff --git a/tests/conftest.py b/tests/conftest.py index 295c91f56..395388f73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ import pytest from telegram import Chat, Message, Update from freqtrade import constants, persistence -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.exchange import Exchange diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 47cb9f353..bd2765430 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -382,13 +382,11 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: data_processed = {pair: frame.copy()} min_date, max_date = get_timerange({pair: frame}) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 10, - 'start_date': min_date, - 'end_date': max_date, - } + processed=data_processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=10, ) assert len(results) == len(data.trades) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 85c4d1b40..2f17467a9 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -11,13 +11,13 @@ from arrow import Arrow from freqtrade import constants from freqtrade.configuration import TimeRange +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.optimize import setup_configuration, start_backtesting from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy @@ -88,21 +88,19 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) results = backtesting.backtest( - { - 'stake_amount': config['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, ) # results :: assert len(results) == num_results # FIX: fixturize this? -def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=None): +def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) @@ -110,13 +108,12 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record= processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(processed) return { - 'stake_amount': conf['stake_amount'], 'processed': processed, - 'max_open_trades': 10, - 'position_stacking': False, - 'record': record, + 'stake_amount': conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 10, + 'position_stacking': False, } @@ -150,7 +147,7 @@ def _trend_alternate(dataframe=None, metadata=None): # Unit tests -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -159,7 +156,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -200,7 +197,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--fee', '0', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -233,7 +230,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog) -def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT patched_configuration_load_config_file(mocker, default_conf) @@ -245,7 +242,7 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog ] with pytest.raises(DependencyException, match=r'.*stake amount.*'): - setup_configuration(get_args(args), RunMode.BACKTEST) + setup_optimize_configuration(get_args(args), RunMode.BACKTEST) def test_start(mocker, fee, default_conf, caplog) -> None: @@ -389,14 +386,12 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(data_processed) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 10, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=data_processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=10, + position_stacking=False, ) assert not results.empty assert len(results) == 2 @@ -445,14 +440,12 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) - processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(processed) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, ) assert not results.empty assert len(results) == 1 @@ -492,7 +485,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert results.empty @@ -507,7 +500,7 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert results.empty @@ -520,7 +513,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) backtesting._store_backtest_result("test_.json", results) # 200 candles in backtest data # won't buy on first (shifted by 1) @@ -565,15 +558,15 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) data_processed = backtesting.strategy.tickerdata_to_dataframe(data) min_date, max_date = get_timerange(data_processed) backtest_conf = { - 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, - 'max_open_trades': 3, - 'position_stacking': False, + 'stake_amount': default_conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 3, + 'position_stacking': False, } - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) # Make sure we have parallel trades assert len(evaluate_result_multi(results, '5m', 2)) > 0 @@ -581,14 +574,14 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results, '5m', 3)) == 0 backtest_conf = { - 'stake_amount': default_conf['stake_amount'], 'processed': data_processed, - 'max_open_trades': 1, - 'position_stacking': False, + 'stake_amount': default_conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 1, + 'position_stacking': False, } - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert len(evaluate_result_multi(results, '5m', 1)) == 0 diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index acc0d2d16..96dd0899d 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,14 +3,14 @@ from unittest.mock import MagicMock -from freqtrade.optimize import setup_configuration, start_edge +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -19,7 +19,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert config['runmode'] == RunMode.EDGE assert 'max_open_trades' in config @@ -53,7 +53,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--stoplosses=-0.01,-0.10,-0.001' ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index cce924a05..1ce549ac8 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -9,9 +9,10 @@ import pytest from arrow import Arrow from filelock import Timeout +from freqtrade.commands.optimize_commands import (setup_optimize_configuration, + start_hyperopt) from freqtrade.data.history import load_data from freqtrade.exceptions import OperationalException -from freqtrade.optimize import setup_configuration, start_hyperopt from freqtrade.optimize.default_hyperopt import DefaultHyperOpt from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt @@ -76,7 +77,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca '--hyperopt', 'DefaultHyperOpt', ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -116,7 +117,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--print-all' ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config diff --git a/tests/test_arguments.py b/tests/test_arguments.py index d8fbace0f..60da0082a 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest -from freqtrade.configuration import Arguments -from freqtrade.configuration.cli_options import check_int_positive +from freqtrade.commands import Arguments +from freqtrade.commands.cli_options import check_int_positive # Parse common command-line-arguments. Used for all tools diff --git a/tests/test_configuration.py b/tests/test_configuration.py index cbcd6416a..74de166c1 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,7 +10,8 @@ from unittest.mock import MagicMock import pytest from jsonschema import ValidationError -from freqtrade.configuration import (Arguments, Configuration, check_exchange, +from freqtrade.commands import Arguments +from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, validate_config_consistency) from freqtrade.configuration.config_validation import validate_config_schema diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6c6bd1753..e0f2ecd3a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -906,30 +906,22 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0] -def test_balance_fully_ask_side(mocker, default_conf) -> None: - default_conf['bid_strategy']['ask_last_balance'] = 0.0 +@pytest.mark.parametrize("ask,last,last_ab,expected", [ + (20, 10, 0.0, 20), # Full ask side + (20, 10, 1.0, 10), # Full last side + (20, 10, 0.5, 15), # Between ask and last + (20, 10, 0.7, 13), # Between ask and last + (20, 10, 0.3, 17), # Between ask and last + (5, 10, 1.0, 5), # last bigger than ask + (5, 10, 0.5, 5), # last bigger than ask +]) +def test_get_buy_rate(mocker, default_conf, ask, last, last_ab, expected) -> None: + default_conf['bid_strategy']['ask_last_balance'] = last_ab freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={'ask': 20, 'last': 10})) + MagicMock(return_value={'ask': ask, 'last': last})) - assert freqtrade.get_target_bid('ETH/BTC') == 20 - - -def test_balance_fully_last_side(mocker, default_conf) -> None: - default_conf['bid_strategy']['ask_last_balance'] = 1.0 - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={'ask': 20, 'last': 10})) - - assert freqtrade.get_target_bid('ETH/BTC') == 10 - - -def test_balance_bigger_last_ask(mocker, default_conf) -> None: - default_conf['bid_strategy']['ask_last_balance'] = 1.0 - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - MagicMock(return_value={'ask': 5, 'last': 10})) - assert freqtrade.get_target_bid('ETH/BTC') == 5 + assert freqtrade.get_buy_rate('ETH/BTC') == expected def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: @@ -938,10 +930,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: freqtrade = FreqtradeBot(default_conf) stake_amount = 2 bid = 0.11 - get_bid = MagicMock(return_value=bid) + buy_rate_mock = MagicMock(return_value=bid) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', - get_target_bid=get_bid, + get_buy_rate=buy_rate_mock, _get_min_pair_stake_amount=MagicMock(return_value=1) ) buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) @@ -958,7 +950,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: pair = 'ETH/BTC' assert freqtrade.execute_buy(pair, stake_amount) - assert get_bid.call_count == 1 + assert buy_rate_mock.call_count == 1 assert buy_mm.call_count == 1 call_args = buy_mm.call_args_list[0][1] assert call_args['pair'] == pair @@ -975,8 +967,8 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: # Test calling with price fix_price = 0.06 assert freqtrade.execute_buy(pair, stake_amount, fix_price) - # Make sure get_target_bid wasn't called again - assert get_bid.call_count == 1 + # Make sure get_buy_rate wasn't called again + assert buy_rate_mock.call_count == 1 assert buy_mm.call_count == 2 call_args = buy_mm.call_args_list[1][1] @@ -3500,7 +3492,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: """ - test if function get_target_bid will return the order book price + test if function get_buy_rate will return the order book price instead of the ask rate """ patch_exchange(mocker) @@ -3518,13 +3510,13 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('ETH/BTC') == 0.043935 + assert freqtrade.get_buy_rate('ETH/BTC') == 0.043935 assert ticker_mock.call_count == 0 def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: """ - test if function get_target_bid will return the ask rate (since its value is lower) + test if function get_buy_rate will return the ask rate (since its value is lower) instead of the order book rate (even if enabled) """ patch_exchange(mocker) @@ -3543,7 +3535,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. - assert freqtrade.get_target_bid('ETH/BTC') != 0.042 + assert freqtrade.get_buy_rate('ETH/BTC') != 0.042 assert ticker_mock.call_count == 0 diff --git a/tests/test_main.py b/tests/test_main.py index 76b1bf658..1229f748a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments from freqtrade.exceptions import OperationalException, FreqtradeException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main @@ -26,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: Test that main() can start backtesting and also ensure we can pass some specific arguments further argument parsing is done in test_arguments.py """ - backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + backtesting_mock = mocker.patch('freqtrade.commands.start_backtesting') backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): @@ -42,7 +42,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: - hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock()) hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 78c01eb97..e7ec4ce46 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.exceptions import OperationalException -from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit +from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.plot.plotting import (add_indicators, add_profit, create_plotconfig, generate_candlestick_graph,