From 80ed1c3e146da9c3f36f978ceae0df250acc4131 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 10:51:09 +0100 Subject: [PATCH 01/14] Move utils to commands --- freqtrade/commands/__init__.py | 0 freqtrade/{ => commands}/utils.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/commands/__init__.py rename freqtrade/{ => commands}/utils.py (100%) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/utils.py b/freqtrade/commands/utils.py similarity index 100% rename from freqtrade/utils.py rename to freqtrade/commands/utils.py From 6e852804671efa0546341b5db5f35b79bc9e9278 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 10:55:39 +0100 Subject: [PATCH 02/14] Adjust imports --- freqtrade/commands/__init__.py | 6 ++++++ freqtrade/configuration/arguments.py | 12 +++++------ freqtrade/optimize/__init__.py | 2 +- freqtrade/plot/plot_utils.py | 2 +- tests/test_utils.py | 30 ++++++++++++++-------------- 5 files changed, 29 insertions(+), 23 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index e69de29bb..0c31048f2 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -0,0 +1,6 @@ +from .utils import (setup_utils_configuration, start_create_userdir, # noqa: F401 + start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_test_pairlist, start_trading) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index b2197619d..3c71fa548 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -131,12 +131,12 @@ class Arguments: self._build_args(optionlist=['version'], parser=self.parser) from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import (start_create_userdir, start_download_data, - start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_markets, - start_list_strategies, start_new_hyperopt, - start_new_strategy, start_list_timeframes, - start_test_pairlist, start_trading) + from freqtrade.commands import (start_create_userdir, start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_markets, + start_list_strategies, start_new_hyperopt, + start_new_strategy, start_list_timeframes, + start_test_pairlist, start_trading) from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 34760372f..98b521d4e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -4,7 +4,7 @@ from typing import Any, Dict from freqtrade import constants from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration +from freqtrade.commands import setup_utils_configuration logger = logging.getLogger(__name__) diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/plot/plot_utils.py index 9eff08396..b2f16dff0 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/plot/plot_utils.py @@ -2,7 +2,7 @@ from typing import Any, Dict from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration +from freqtrade.commands import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/tests/test_utils.py b/tests/test_utils.py index 7ca2cac5a..91009ec42 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,15 +4,15 @@ from unittest.mock import MagicMock, PropertyMock import pytest +from freqtrade.commands import (setup_utils_configuration, + start_create_userdir, start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_hyperopt, start_new_strategy, + start_test_pairlist, start_trading) from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import (setup_utils_configuration, start_create_userdir, - start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_hyperopt, - start_new_strategy, start_test_pairlist, - start_trading) from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -451,8 +451,8 @@ def test_create_datadir(caplog, mocker): # Added assert here to analyze random test-failures ... assert len(caplog.record_tuples) == 0 - cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock()) + cud = mocker.patch("freqtrade.commands.create_userdata_dir", MagicMock()) + csf = mocker.patch("freqtrade.commands.copy_sample_files", MagicMock()) args = [ "create-userdir", "--userdir", @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( From 926bf07df1420c2e08dadbb53e8359a1ffa20900 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:08:58 +0100 Subject: [PATCH 03/14] Seperate a few commands into specific files --- freqtrade/commands/__init__.py | 16 +- freqtrade/commands/deploy_commands.py | 113 ++++++++ freqtrade/commands/hyperopt_commands.py | 114 ++++++++ freqtrade/commands/list_commands.py | 157 +++++++++++ freqtrade/commands/trade_commands.py | 25 ++ freqtrade/commands/utils.py | 356 ------------------------ 6 files changed, 419 insertions(+), 362 deletions(-) create mode 100644 freqtrade/commands/deploy_commands.py create mode 100644 freqtrade/commands/hyperopt_commands.py create mode 100644 freqtrade/commands/list_commands.py create mode 100644 freqtrade/commands/trade_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 0c31048f2..4c57efeae 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,6 +1,10 @@ -from .utils import (setup_utils_configuration, start_create_userdir, # noqa: F401 - start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_hyperopt, - start_new_strategy, start_test_pairlist, start_trading) +from .hyperopt_commands import (start_hyperopt_list, start_hyperopt_show) # noqa: 401 + +from .list_commands import (start_list_exchanges, # noqa: F401 + start_list_markets, start_list_strategies, + start_list_timeframes) +from .utils import setup_utils_configuration # noqa: F401 +from .utils import (start_download_data, # noqa: F401 + start_test_pairlist) +from .deploy_commands import (start_new_hyperopt, start_new_strategy, start_create_userdir) # noqa: F401 +from .trade_commands import start_trading # noqa: F401 diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py new file mode 100644 index 000000000..892e15db1 --- /dev/null +++ b/freqtrade/commands/deploy_commands.py @@ -0,0 +1,113 @@ +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration.directory_operations import (copy_sample_files, + create_userdata_dir) +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY +from freqtrade.exceptions import OperationalException +from freqtrade.misc import render_template +from freqtrade.state import RunMode + +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_create_userdir(args: Dict[str, Any]) -> None: + """ + Create "user_data" directory to contain user data strategies, hyperopt, ...) + :param args: Cli args from Arguments() + :return: None + """ + if "user_data_dir" in args and args["user_data_dir"]: + userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) + copy_sample_files(userdir, overwrite=args["reset"]) + else: + logger.warning("`create-userdir` requires --userdir to be set.") + sys.exit(1) + + +def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): + """ + Deploy new strategy from template to strategy_path + """ + indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) + buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) + sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) + plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",) + + strategy_text = render_template(templatefile='base_strategy.py.j2', + arguments={"strategy": strategy_name, + "indicators": indicators, + "buy_trend": buy_trend, + "sell_trend": sell_trend, + "plot_config": plot_config, + }) + + logger.info(f"Writing strategy to `{strategy_path}`.") + strategy_path.write_text(strategy_text) + + +def start_new_strategy(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "strategy" in args and args["strategy"]: + if args["strategy"] == "DefaultStrategy": + raise OperationalException("DefaultStrategy is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + + deploy_new_strategy(args['strategy'], new_path, args['template']) + + else: + raise OperationalException("`new-strategy` requires --strategy to be set.") + + +def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): + """ + Deploys a new hyperopt template to hyperopt_path + """ + buy_guards = render_template( + templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) + sell_guards = render_template( + templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) + buy_space = render_template( + templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) + sell_space = render_template( + templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) + + strategy_text = render_template(templatefile='base_hyperopt.py.j2', + arguments={"hyperopt": hyperopt_name, + "buy_guards": buy_guards, + "sell_guards": sell_guards, + "buy_space": buy_space, + "sell_space": sell_space, + }) + + logger.info(f"Writing hyperopt to `{hyperopt_path}`.") + hyperopt_path.write_text(strategy_text) + + +def start_new_hyperopt(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "hyperopt" in args and args["hyperopt"]: + if args["hyperopt"] == "DefaultHyperopt": + raise OperationalException("DefaultHyperopt is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) + else: + raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py new file mode 100644 index 000000000..db3f69050 --- /dev/null +++ b/freqtrade/commands/hyperopt_commands.py @@ -0,0 +1,114 @@ +import logging +from operator import itemgetter +from typing import Any, Dict, List + +from colorama import init as colorama_init + +from .utils import setup_utils_configuration +from freqtrade.exceptions import OperationalException +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_hyperopt_list(args: Dict[str, Any]) -> None: + """ + List hyperopt epochs previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + only_best = config.get('hyperopt_list_best', False) + only_profitable = config.get('hyperopt_list_profitable', False) + print_colorized = config.get('print_colorized', False) + print_json = config.get('print_json', False) + no_details = config.get('hyperopt_list_no_details', False) + no_header = False + + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + trials = Hyperopt.load_previous_results(trials_file) + total_epochs = len(trials) + + trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + + # TODO: fetch the interval for epochs to print from the cli option + epoch_start, epoch_stop = 0, None + + if print_colorized: + colorama_init(autoreset=True) + + try: + # Human-friendly indexes used here (starting from 1) + for val in trials[epoch_start:epoch_stop]: + Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) + + except KeyboardInterrupt: + print('User interrupted..') + + if trials and not no_details: + sorted_trials = sorted(trials, key=itemgetter('loss')) + results = sorted_trials[0] + Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) + + +def start_hyperopt_show(args: Dict[str, Any]) -> None: + """ + Show details of a hyperopt epoch previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + only_best = config.get('hyperopt_list_best', False) + only_profitable = config.get('hyperopt_list_profitable', False) + no_header = config.get('hyperopt_show_no_header', False) + + trials_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + trials = Hyperopt.load_previous_results(trials_file) + total_epochs = len(trials) + + trials = _hyperopt_filter_trials(trials, only_best, only_profitable) + trials_epochs = len(trials) + + n = config.get('hyperopt_show_index', -1) + if n > trials_epochs: + raise OperationalException( + f"The index of the epoch to show should be less than {trials_epochs + 1}.") + if n < -trials_epochs: + raise OperationalException( + f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + + # Translate epoch index from human-readable format to pythonic + if n > 0: + n -= 1 + + print_json = config.get('print_json', False) + + if trials: + val = trials[n] + Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") + + +def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: + """ + Filter our items from the list of hyperopt results + """ + if only_best: + trials = [x for x in trials if x['is_best']] + if only_profitable: + trials = [x for x in trials if x['results_metrics']['profit'] > 0] + + logger.info(f"{len(trials)} " + + ("best " if only_best else "") + + ("profitable " if only_profitable else "") + + "epochs found.") + + return trials diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py new file mode 100644 index 000000000..90efaf66f --- /dev/null +++ b/freqtrade/commands/list_commands.py @@ -0,0 +1,157 @@ +import csv +import logging +import sys +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict + +import rapidjson +from tabulate import tabulate + +from freqtrade.constants import USERPATH_STRATEGY +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import (available_exchanges, ccxt_exchanges, + market_is_active, symbol_is_pair) +from freqtrade.misc import plural +from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.state import RunMode + +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_list_exchanges(args: Dict[str, Any]) -> None: + """ + Print available exchanges + :param args: Cli args from Arguments() + :return: None + """ + exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() + if args['print_one_column']: + print('\n'.join(exchanges)) + else: + if args['list_exchanges_all']: + print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") + else: + print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") + + +def start_list_strategies(args: Dict[str, Any]) -> None: + """ + Print Strategies available in a directory + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) + strategies = StrategyResolver.search_all_objects(directory) + # Sort alphabetically + strategies = sorted(strategies, key=lambda x: x['name']) + strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] + + if args['print_one_column']: + print('\n'.join([s['name'] for s in strategies])) + else: + print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) + + +def start_list_timeframes(args: Dict[str, Any]) -> None: + """ + Print ticker intervals (timeframes) available on Exchange + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + # Do not use ticker_interval set in the config + config['ticker_interval'] = None + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + if args['print_one_column']: + print('\n'.join(exchange.timeframes)) + else: + print(f"Timeframes available for the exchange `{exchange.name}`: " + f"{', '.join(exchange.timeframes)}") + + +def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: + """ + Print pairs/markets on the exchange + :param args: Cli args from Arguments() + :param pairs_only: if True print only pairs, otherwise print all instruments (markets) + :return: None + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + # By default only active pairs/markets are to be shown + active_only = not args.get('list_pairs_all', False) + + base_currencies = args.get('base_currencies', []) + quote_currencies = args.get('quote_currencies', []) + + try: + pairs = exchange.get_markets(base_currencies=base_currencies, + quote_currencies=quote_currencies, + pairs_only=pairs_only, + active_only=active_only) + # Sort the pairs/markets by symbol + pairs = OrderedDict(sorted(pairs.items())) + except Exception as e: + raise OperationalException(f"Cannot get markets. Reason: {e}") from e + + else: + summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + + ("active " if active_only else "") + + (plural(len(pairs), "pair" if pairs_only else "market")) + + (f" with {', '.join(base_currencies)} as base " + f"{plural(len(base_currencies), 'currency', 'currencies')}" + if base_currencies else "") + + (" and" if base_currencies and quote_currencies else "") + + (f" with {', '.join(quote_currencies)} as quote " + f"{plural(len(quote_currencies), 'currency', 'currencies')}" + if quote_currencies else "")) + + headers = ["Id", "Symbol", "Base", "Quote", "Active", + *(['Is pair'] if not pairs_only else [])] + + tabular_data = [] + for _, v in pairs.items(): + tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], + 'Base': v['base'], 'Quote': v['quote'], + 'Active': market_is_active(v), + **({'Is pair': symbol_is_pair(v['symbol'])} + if not pairs_only else {})}) + + if (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + # Print summary string in the log in case of machine-readable + # regular formats. + logger.info(f"{summary_str}.") + else: + # Print empty string separating leading logs and output in case of + # human-readable formats. + print() + + if len(pairs): + if args.get('print_list', False): + # print data as a list, with human-readable summary + print(f"{summary_str}: {', '.join(pairs.keys())}.") + elif args.get('print_one_column', False): + print('\n'.join(pairs.keys())) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairs.keys()), default=str)) + elif args.get('print_csv', False): + writer = csv.DictWriter(sys.stdout, fieldnames=headers) + writer.writeheader() + writer.writerows(tabular_data) + else: + # print data as a table, with the human-readable summary + print(f"{summary_str}:") + print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) + elif not (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + print(f"{summary_str}.") diff --git a/freqtrade/commands/trade_commands.py b/freqtrade/commands/trade_commands.py new file mode 100644 index 000000000..2c0c4c9c1 --- /dev/null +++ b/freqtrade/commands/trade_commands.py @@ -0,0 +1,25 @@ +import logging + +from typing import Any, Dict + + +logger = logging.getLogger(__name__) + + +def start_trading(args: Dict[str, Any]) -> int: + """ + Main entry point for trading mode + """ + from freqtrade.worker import Worker + # Load and run worker + worker = None + try: + worker = Worker(args) + worker.run() + except KeyboardInterrupt: + logger.info('SIGINT received, aborting ...') + finally: + if worker: + logger.info("worker found ... calling exit") + worker.exit() + return 0 diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index 2f7b2d717..ba9173cf2 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -46,139 +46,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str return config -def start_trading(args: Dict[str, Any]) -> int: - """ - Main entry point for trading mode - """ - from freqtrade.worker import Worker - # Load and run worker - worker = None - try: - worker = Worker(args) - worker.run() - except KeyboardInterrupt: - logger.info('SIGINT received, aborting ...') - finally: - if worker: - logger.info("worker found ... calling exit") - worker.exit() - return 0 - - -def start_list_exchanges(args: Dict[str, Any]) -> None: - """ - Print available exchanges - :param args: Cli args from Arguments() - :return: None - """ - exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() - if args['print_one_column']: - print('\n'.join(exchanges)) - else: - if args['list_exchanges_all']: - print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") - else: - print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") - - -def start_create_userdir(args: Dict[str, Any]) -> None: - """ - Create "user_data" directory to contain user data strategies, hyperopt, ...) - :param args: Cli args from Arguments() - :return: None - """ - if "user_data_dir" in args and args["user_data_dir"]: - userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) - copy_sample_files(userdir, overwrite=args["reset"]) - else: - logger.warning("`create-userdir` requires --userdir to be set.") - sys.exit(1) - - -def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): - """ - Deploy new strategy from template to strategy_path - """ - indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) - buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) - sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) - plot_config = render_template(templatefile=f"subtemplates/plot_config_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_strategy.py.j2', - arguments={"strategy": strategy_name, - "indicators": indicators, - "buy_trend": buy_trend, - "sell_trend": sell_trend, - "plot_config": plot_config, - }) - - logger.info(f"Writing strategy to `{strategy_path}`.") - strategy_path.write_text(strategy_text) - - -def start_new_strategy(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "strategy" in args and args["strategy"]: - if args["strategy"] == "DefaultStrategy": - raise OperationalException("DefaultStrategy is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - - deploy_new_strategy(args['strategy'], new_path, args['template']) - - else: - raise OperationalException("`new-strategy` requires --strategy to be set.") - - -def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): - """ - Deploys a new hyperopt template to hyperopt_path - """ - buy_guards = render_template( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) - sell_guards = render_template( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) - buy_space = render_template( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) - sell_space = render_template( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "hyperopt" in args and args["hyperopt"]: - if args["hyperopt"] == "DefaultHyperopt": - raise OperationalException("DefaultHyperopt is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - def start_download_data(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) @@ -227,126 +94,6 @@ def start_download_data(args: Dict[str, Any]) -> None: f"on exchange {exchange.name}.") -def start_list_strategies(args: Dict[str, Any]) -> None: - """ - Print Strategies available in a directory - """ - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGY)) - strategies = StrategyResolver.search_all_objects(directory) - # Sort alphabetically - strategies = sorted(strategies, key=lambda x: x['name']) - strats_to_print = [{'name': s['name'], 'location': s['location'].name} for s in strategies] - - if args['print_one_column']: - print('\n'.join([s['name'] for s in strategies])) - else: - print(tabulate(strats_to_print, headers='keys', tablefmt='pipe')) - - -def start_list_timeframes(args: Dict[str, Any]) -> None: - """ - Print ticker intervals (timeframes) available on Exchange - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - # Do not use ticker_interval set in the config - config['ticker_interval'] = None - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - if args['print_one_column']: - print('\n'.join(exchange.timeframes)) - else: - print(f"Timeframes available for the exchange `{exchange.name}`: " - f"{', '.join(exchange.timeframes)}") - - -def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: - """ - Print pairs/markets on the exchange - :param args: Cli args from Arguments() - :param pairs_only: if True print only pairs, otherwise print all instruments (markets) - :return: None - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - # By default only active pairs/markets are to be shown - active_only = not args.get('list_pairs_all', False) - - base_currencies = args.get('base_currencies', []) - quote_currencies = args.get('quote_currencies', []) - - try: - pairs = exchange.get_markets(base_currencies=base_currencies, - quote_currencies=quote_currencies, - pairs_only=pairs_only, - active_only=active_only) - # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) - except Exception as e: - raise OperationalException(f"Cannot get markets. Reason: {e}") from e - - else: - summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + - ("active " if active_only else "") + - (plural(len(pairs), "pair" if pairs_only else "market")) + - (f" with {', '.join(base_currencies)} as base " - f"{plural(len(base_currencies), 'currency', 'currencies')}" - if base_currencies else "") + - (" and" if base_currencies and quote_currencies else "") + - (f" with {', '.join(quote_currencies)} as quote " - f"{plural(len(quote_currencies), 'currency', 'currencies')}" - if quote_currencies else "")) - - headers = ["Id", "Symbol", "Base", "Quote", "Active", - *(['Is pair'] if not pairs_only else [])] - - tabular_data = [] - for _, v in pairs.items(): - tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], - 'Base': v['base'], 'Quote': v['quote'], - 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v['symbol'])} - if not pairs_only else {})}) - - if (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - # Print summary string in the log in case of machine-readable - # regular formats. - logger.info(f"{summary_str}.") - else: - # Print empty string separating leading logs and output in case of - # human-readable formats. - print() - - if len(pairs): - if args.get('print_list', False): - # print data as a list, with human-readable summary - print(f"{summary_str}: {', '.join(pairs.keys())}.") - elif args.get('print_one_column', False): - print('\n'.join(pairs.keys())) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairs.keys()), default=str)) - elif args.get('print_csv', False): - writer = csv.DictWriter(sys.stdout, fieldnames=headers) - writer.writeheader() - writer.writerows(tabular_data) - else: - # print data as a table, with the human-readable summary - print(f"{summary_str}:") - print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) - elif not (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - print(f"{summary_str}.") - - def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration @@ -377,106 +124,3 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: print(rapidjson.dumps(list(pairlist), default=str)) else: print(pairlist) - - -def start_hyperopt_list(args: Dict[str, Any]) -> None: - """ - List hyperopt epochs previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - print_colorized = config.get('print_colorized', False) - print_json = config.get('print_json', False) - no_details = config.get('hyperopt_list_no_details', False) - no_header = False - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - - # TODO: fetch the interval for epochs to print from the cli option - epoch_start, epoch_stop = 0, None - - if print_colorized: - colorama_init(autoreset=True) - - try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) - - except KeyboardInterrupt: - print('User interrupted..') - - if trials and not no_details: - sorted_trials = sorted(trials, key=itemgetter('loss')) - results = sorted_trials[0] - Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) - - -def start_hyperopt_show(args: Dict[str, Any]) -> None: - """ - Show details of a hyperopt epoch previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - no_header = config.get('hyperopt_show_no_header', False) - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - trials_epochs = len(trials) - - n = config.get('hyperopt_show_index', -1) - if n > trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") - if n < -trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") - - # Translate epoch index from human-readable format to pythonic - if n > 0: - n -= 1 - - print_json = config.get('print_json', False) - - if trials: - val = trials[n] - Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") - - -def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: - """ - Filter our items from the list of hyperopt results - """ - if only_best: - trials = [x for x in trials if x['is_best']] - if only_profitable: - trials = [x for x in trials if x['results_metrics']['profit'] > 0] - - logger.info(f"{len(trials)} " + - ("best " if only_best else "") + - ("profitable " if only_profitable else "") + - "epochs found.") - - return trials From 7e233041877d7c4976735e10269a2f3421c530f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:09:13 +0100 Subject: [PATCH 04/14] Adjust tests to new paths --- tests/test_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 91009ec42..c686d2117 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -451,8 +451,8 @@ def test_create_datadir(caplog, mocker): # Added assert here to analyze random test-failures ... assert len(caplog.record_tuples) == 0 - cud = mocker.patch("freqtrade.commands.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.commands.copy_sample_files", MagicMock()) + cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) + csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) args = [ "create-userdir", "--userdir", @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.commands.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.commands.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.utils.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( From 70a0346b0a2b781dda480dc1a00da9215ac80eea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:17:26 +0100 Subject: [PATCH 05/14] Move data-stuff to data-commands --- freqtrade/commands/__init__.py | 16 +++---- freqtrade/commands/data_commands.py | 64 +++++++++++++++++++++++++ freqtrade/commands/utils.py | 73 ++--------------------------- tests/test_utils.py | 12 ++--- 4 files changed, 81 insertions(+), 84 deletions(-) create mode 100644 freqtrade/commands/data_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 4c57efeae..5fa667ccc 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,10 +1,10 @@ -from .hyperopt_commands import (start_hyperopt_list, start_hyperopt_show) # noqa: 401 - -from .list_commands import (start_list_exchanges, # noqa: F401 +from freqtrade.commands.data_commands import start_download_data # noqa: F401 +from freqtrade.commands.deploy_commands import (start_create_userdir, # noqa: F401 + start_new_hyperopt, start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: F401 + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 start_list_markets, start_list_strategies, start_list_timeframes) -from .utils import setup_utils_configuration # noqa: F401 -from .utils import (start_download_data, # noqa: F401 - start_test_pairlist) -from .deploy_commands import (start_new_hyperopt, start_new_strategy, start_create_userdir) # noqa: F401 -from .trade_commands import start_trading # noqa: F401 +from freqtrade.commands.trade_commands import start_trading # noqa: F401 +from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py new file mode 100644 index 000000000..b1c847d1d --- /dev/null +++ b/freqtrade/commands/data_commands.py @@ -0,0 +1,64 @@ +import logging +import sys +from typing import Any, Dict, List + +import arrow + +from freqtrade.configuration import TimeRange +from freqtrade.data.history import (convert_trades_to_ohlcv, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data) +from freqtrade.exceptions import OperationalException +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_download_data(args: Dict[str, Any]) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {config["datadir"]}') + + pairs_not_available: List[str] = [] + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + try: + + if config.get('download_trades'): + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=config["pairs"], datadir=config['datadir'], + timerange=timerange, erase=config.get("erase")) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + else: + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.") diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index ba9173cf2..a597d5335 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -1,30 +1,11 @@ -import csv import logging -import sys -from collections import OrderedDict -from operator import itemgetter -from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict -import arrow import rapidjson -from colorama import init as colorama_init -from tabulate import tabulate -from freqtrade.configuration import (Configuration, TimeRange, - remove_credentials, +from freqtrade.configuration import (Configuration, remove_credentials, validate_config_consistency) -from freqtrade.configuration.directory_operations import (copy_sample_files, - create_userdata_dir) -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.history import (convert_trades_to_ohlcv, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data) -from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) -from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -46,54 +27,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str return config -def start_download_data(args: Dict[str, Any]) -> None: - """ - Download data (former download_backtest_data.py script) - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - timerange = TimeRange() - if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") - - logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {config["datadir"]}') - - pairs_not_available: List[str] = [] - - # Init exchange - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) - try: - - if config.get('download_trades'): - pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=config.get("erase")) - - # Convert downloaded trade data to different timeframes - convert_trades_to_ohlcv( - pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) - else: - pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=config.get("erase")) - - except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - - finally: - if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {exchange.name}.") - - def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration diff --git a/tests/test_utils.py b/tests/test_utils.py index c686d2117..1328e3981 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -538,7 +538,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -556,7 +556,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -574,7 +574,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -594,7 +594,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.commands.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -613,9 +613,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.commands.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.commands.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( From b254bdfea31c95b237b2642c6b55050d1e727b45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:20:34 +0100 Subject: [PATCH 06/14] Move plot_utils to plot_commands --- freqtrade/commands/__init__.py | 1 + freqtrade/{plot/plot_utils.py => commands/plot_commands.py} | 2 +- freqtrade/configuration/arguments.py | 2 +- tests/test_plotting.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) rename freqtrade/{plot/plot_utils.py => commands/plot_commands.py} (94%) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 5fa667ccc..45088e8a3 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -6,5 +6,6 @@ from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 start_list_markets, start_list_strategies, start_list_timeframes) +from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit # noqa: F401 from freqtrade.commands.trade_commands import start_trading # noqa: F401 from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/commands/plot_commands.py similarity index 94% rename from freqtrade/plot/plot_utils.py rename to freqtrade/commands/plot_commands.py index b2f16dff0..4c68cc3c5 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/commands/plot_commands.py @@ -2,7 +2,7 @@ from typing import Any, Dict from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.commands import setup_utils_configuration +from freqtrade.commands.utils import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 3c71fa548..c61e4ec13 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -136,8 +136,8 @@ class Arguments: start_list_exchanges, start_list_markets, start_list_strategies, start_new_hyperopt, start_new_strategy, start_list_timeframes, + start_plot_dataframe, start_plot_profit, start_test_pairlist, start_trading) - from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 78c01eb97..e7ec4ce46 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -11,7 +11,7 @@ from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data from freqtrade.exceptions import OperationalException -from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit +from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.plot.plotting import (add_indicators, add_profit, create_plotconfig, generate_candlestick_graph, From e033df6a2fce1d50333e70e599cd7dfde187b162 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:32:52 +0100 Subject: [PATCH 07/14] Move optimize_commands to commands module --- .../__init__.py => commands/optimize_commands.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename freqtrade/{optimize/__init__.py => commands/optimize_commands.py} (89%) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/commands/optimize_commands.py similarity index 89% rename from freqtrade/optimize/__init__.py rename to freqtrade/commands/optimize_commands.py index 98b521d4e..e635e61fd 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/commands/optimize_commands.py @@ -2,14 +2,14 @@ import logging from typing import Any, Dict from freqtrade import constants +from freqtrade.commands.utils import setup_utils_configuration from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode -from freqtrade.commands import setup_utils_configuration logger = logging.getLogger(__name__) -def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: +def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: """ Prepare the configuration for the Hyperopt module :param args: Cli args from Arguments() @@ -35,7 +35,7 @@ def start_backtesting(args: Dict[str, Any]) -> None: from freqtrade.optimize.backtesting import Backtesting # Initialize configuration - config = setup_configuration(args, RunMode.BACKTEST) + config = setup_optimize_configuration(args, RunMode.BACKTEST) logger.info('Starting freqtrade in Backtesting mode') @@ -58,7 +58,7 @@ def start_hyperopt(args: Dict[str, Any]) -> None: raise OperationalException( f"{e}. Please ensure that the hyperopt dependencies are installed.") from e # Initialize configuration - config = setup_configuration(args, RunMode.HYPEROPT) + config = setup_optimize_configuration(args, RunMode.HYPEROPT) logger.info('Starting freqtrade in Hyperopt mode') @@ -94,7 +94,7 @@ def start_edge(args: Dict[str, Any]) -> None: """ from freqtrade.optimize.edge_cli import EdgeCli # Initialize configuration - config = setup_configuration(args, RunMode.EDGE) + config = setup_optimize_configuration(args, RunMode.EDGE) logger.info('Starting freqtrade in Edge mode') # Initialize Edge object From f347e5934a6be5d515c983daac4755ee3a8ec9ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:33:13 +0100 Subject: [PATCH 08/14] Small adjustments for moved commands --- freqtrade/commands/__init__.py | 30 ++++++++++++++++++---------- freqtrade/configuration/arguments.py | 2 +- freqtrade/optimize/__init__.py | 0 tests/optimize/test_backtesting.py | 12 +++++------ tests/optimize/test_edge_cli.py | 8 ++++---- tests/optimize/test_hyperopt.py | 8 ++++---- tests/test_main.py | 4 ++-- 7 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 freqtrade/optimize/__init__.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 45088e8a3..8339486fb 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,11 +1,19 @@ -from freqtrade.commands.data_commands import start_download_data # noqa: F401 -from freqtrade.commands.deploy_commands import (start_create_userdir, # noqa: F401 - start_new_hyperopt, start_new_strategy) -from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, # noqa: F401 - start_hyperopt_show) -from freqtrade.commands.list_commands import (start_list_exchanges, # noqa: F401 - start_list_markets, start_list_strategies, - start_list_timeframes) -from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit # noqa: F401 -from freqtrade.commands.trade_commands import start_trading # noqa: F401 -from freqtrade.commands.utils import setup_utils_configuration, start_test_pairlist # noqa: F401 +# flake8: noqa: F401 + +from freqtrade.commands.data_commands import start_download_data +from freqtrade.commands.deploy_commands import (start_create_userdir, + start_new_hyperopt, + start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, + start_list_markets, + start_list_strategies, + start_list_timeframes) +from freqtrade.commands.optimize_commands import (start_backtesting, + start_edge, start_hyperopt) +from freqtrade.commands.plot_commands import (start_plot_dataframe, + start_plot_profit) +from freqtrade.commands.trade_commands import start_trading +from freqtrade.commands.utils import (setup_utils_configuration, + start_test_pairlist) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index c61e4ec13..21800e821 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -130,13 +130,13 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_new_hyperopt, start_new_strategy, start_list_timeframes, start_plot_dataframe, start_plot_profit, + start_backtesting, start_hyperopt, start_edge, start_test_pairlist, start_trading) subparsers = self.parser.add_subparsers(dest='command', diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index acbc44e21..07872da57 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -12,13 +12,13 @@ from arrow import Arrow from freqtrade import constants from freqtrade.configuration import TimeRange +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.optimize import setup_configuration, start_backtesting from freqtrade.optimize.backtesting import Backtesting from freqtrade.state import RunMode from freqtrade.strategy.default_strategy import DefaultStrategy @@ -177,7 +177,7 @@ def _trend_alternate(dataframe=None, metadata=None): # Unit tests -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -186,7 +186,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -227,7 +227,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--fee', '0', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -260,7 +260,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog) -def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT patched_configuration_load_config_file(mocker, default_conf) @@ -272,7 +272,7 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog ] with pytest.raises(DependencyException, match=r'.*stake amount.*'): - setup_configuration(get_args(args), RunMode.BACKTEST) + setup_optimize_configuration(get_args(args), RunMode.BACKTEST) def test_start(mocker, fee, default_conf, caplog) -> None: diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index acc0d2d16..96dd0899d 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,14 +3,14 @@ from unittest.mock import MagicMock -from freqtrade.optimize import setup_configuration, start_edge +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -19,7 +19,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert config['runmode'] == RunMode.EDGE assert 'max_open_trades' in config @@ -53,7 +53,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--stoplosses=-0.01,-0.10,-0.001' ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 473d760ac..69d110649 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -9,10 +9,10 @@ import pytest from arrow import Arrow from filelock import Timeout -from freqtrade.exceptions import OperationalException +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file -from freqtrade.optimize import setup_configuration, start_hyperopt +from freqtrade.exceptions import OperationalException from freqtrade.optimize.default_hyperopt import DefaultHyperOpt from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt @@ -77,7 +77,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca '--hyperopt', 'DefaultHyperOpt', ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -117,7 +117,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--print-all' ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config diff --git a/tests/test_main.py b/tests/test_main.py index 76b1bf658..462ac6427 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -26,7 +26,7 @@ def test_parse_args_backtesting(mocker) -> None: Test that main() can start backtesting and also ensure we can pass some specific arguments further argument parsing is done in test_arguments.py """ - backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + backtesting_mock = mocker.patch('freqtrade.commands.start_backtesting') backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): @@ -42,7 +42,7 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: - hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock()) hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): From a1c684f67ca3dcf0c6293d71511c72a0dcbd2cb3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:36:33 +0100 Subject: [PATCH 09/14] Simplify noqa setup for module imports --- freqtrade/configuration/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 63c38d8c5..a631609d7 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,7 @@ -from freqtrade.configuration.arguments import Arguments # noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials # noqa: F401 -from freqtrade.configuration.timerange import TimeRange # noqa: F401 -from freqtrade.configuration.configuration import Configuration # noqa: F401 -from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 +# flake8: noqa: F401 + +from freqtrade.configuration.arguments import Arguments +from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.timerange import TimeRange +from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.config_validation import validate_config_consistency From 7f851ad8d950d5c26da08f76ad239610136af51c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:38:56 +0100 Subject: [PATCH 10/14] Move arguments and cli_options to commands module --- freqtrade/{configuration => commands}/arguments.py | 0 freqtrade/{configuration => commands}/cli_options.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename freqtrade/{configuration => commands}/arguments.py (100%) rename freqtrade/{configuration => commands}/cli_options.py (100%) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/commands/arguments.py similarity index 100% rename from freqtrade/configuration/arguments.py rename to freqtrade/commands/arguments.py diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/commands/cli_options.py similarity index 100% rename from freqtrade/configuration/cli_options.py rename to freqtrade/commands/cli_options.py From a3e9d04383bdc37aea465561306334042adc801a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:41:04 +0100 Subject: [PATCH 11/14] Adjust imports to new place for arguments --- freqtrade/commands/__init__.py | 1 + freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 6 +++--- freqtrade/configuration/__init__.py | 1 - freqtrade/main.py | 2 +- tests/conftest.py | 2 +- tests/test_arguments.py | 4 ++-- tests/test_configuration.py | 3 ++- tests/test_main.py | 2 +- 9 files changed, 12 insertions(+), 11 deletions(-) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 8339486fb..dcb814c50 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.commands.arguments import Arguments from freqtrade.commands.data_commands import start_download_data from freqtrade.commands.deploy_commands import (start_create_userdir, start_new_hyperopt, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 21800e821..724814554 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional from freqtrade import constants -from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS +from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 1807cd591..490f26cfa 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -import argparse +from argparse import ArgumentTypeError from freqtrade import __version__, constants @@ -12,7 +12,7 @@ def check_int_positive(value: str) -> int: if uint <= 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a positive integer value" ) return uint @@ -24,7 +24,7 @@ def check_int_nonzero(value: str) -> int: if uint == 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a non-zero integer value" ) return uint diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index a631609d7..54fc4e427 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,6 +1,5 @@ # flake8: noqa: F401 -from freqtrade.configuration.arguments import Arguments from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.timerange import TimeRange from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/main.py b/freqtrade/main.py index 811e29864..a75eeebed 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -14,7 +14,7 @@ if sys.version_info < (3, 6): import logging from typing import Any, List -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments logger = logging.getLogger('freqtrade') diff --git a/tests/conftest.py b/tests/conftest.py index 295c91f56..395388f73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ import pytest from telegram import Chat, Message, Update from freqtrade import constants, persistence -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.exchange import Exchange diff --git a/tests/test_arguments.py b/tests/test_arguments.py index d8fbace0f..60da0082a 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest -from freqtrade.configuration import Arguments -from freqtrade.configuration.cli_options import check_int_positive +from freqtrade.commands import Arguments +from freqtrade.commands.cli_options import check_int_positive # Parse common command-line-arguments. Used for all tools diff --git a/tests/test_configuration.py b/tests/test_configuration.py index cbcd6416a..74de166c1 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,7 +10,8 @@ from unittest.mock import MagicMock import pytest from jsonschema import ValidationError -from freqtrade.configuration import (Arguments, Configuration, check_exchange, +from freqtrade.commands import Arguments +from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, validate_config_consistency) from freqtrade.configuration.config_validation import validate_config_schema diff --git a/tests/test_main.py b/tests/test_main.py index 462ac6427..1229f748a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments from freqtrade.exceptions import OperationalException, FreqtradeException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main From 2d02c3f2a4097aa56182ff9d56bf4f631ac06696 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:43:30 +0100 Subject: [PATCH 12/14] Split out pairlist_commands --- freqtrade/commands/__init__.py | 4 +-- freqtrade/commands/pairlist_commands.py | 43 +++++++++++++++++++++++++ freqtrade/commands/utils.py | 35 -------------------- 3 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 freqtrade/commands/pairlist_commands.py diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index dcb814c50..e16a6f203 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -13,8 +13,8 @@ from freqtrade.commands.list_commands import (start_list_exchanges, start_list_timeframes) from freqtrade.commands.optimize_commands import (start_backtesting, start_edge, start_hyperopt) +from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import (start_plot_dataframe, start_plot_profit) from freqtrade.commands.trade_commands import start_trading -from freqtrade.commands.utils import (setup_utils_configuration, - start_test_pairlist) +from freqtrade.commands.utils import setup_utils_configuration diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py new file mode 100644 index 000000000..06ba2e680 --- /dev/null +++ b/freqtrade/commands/pairlist_commands.py @@ -0,0 +1,43 @@ +import logging +from typing import Any, Dict + +import rapidjson + +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode + +from .utils import setup_utils_configuration + +logger = logging.getLogger(__name__) + + +def start_test_pairlist(args: Dict[str, Any]) -> None: + """ + Test Pairlist configuration + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + quote_currencies = args.get('quote_currencies') + if not quote_currencies: + quote_currencies = [config.get('stake_currency')] + results = {} + for curr in quote_currencies: + config['stake_currency'] = curr + # Do not use ticker_interval set in the config + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist + + for curr, pairlist in results.items(): + if not args.get('print_one_column', False): + print(f"Pairs for {curr}: ") + + if args.get('print_one_column', False): + print('\n'.join(pairlist)) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairlist), default=str)) + else: + print(pairlist) diff --git a/freqtrade/commands/utils.py b/freqtrade/commands/utils.py index a597d5335..f77a35383 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/commands/utils.py @@ -1,11 +1,8 @@ import logging from typing import Any, Dict -import rapidjson - from freqtrade.configuration import (Configuration, remove_credentials, validate_config_consistency) -from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -25,35 +22,3 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str validate_config_consistency(config) return config - - -def start_test_pairlist(args: Dict[str, Any]) -> None: - """ - Test Pairlist configuration - """ - from freqtrade.pairlist.pairlistmanager import PairListManager - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) - - quote_currencies = args.get('quote_currencies') - if not quote_currencies: - quote_currencies = [config.get('stake_currency')] - results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - # Do not use ticker_interval set in the config - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist - - for curr, pairlist in results.items(): - if not args.get('print_one_column', False): - print(f"Pairs for {curr}: ") - - if args.get('print_one_column', False): - print('\n'.join(pairlist)) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairlist), default=str)) - else: - print(pairlist) From 8c9119b4710e467d4833638d582aba5914d61d2b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:45:37 +0100 Subject: [PATCH 13/14] Add docustring to commands module --- freqtrade/commands/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index e16a6f203..1dadbec1e 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -1,5 +1,11 @@ # flake8: noqa: F401 +""" +Commands module. +Contains all start-commands, subcommands and CLI Interface creation. +Note: Be careful with file-scoped imports in these subfiles. + as they are parsed on startup, nothing containing optional modules should be loaded. +""" from freqtrade.commands.arguments import Arguments from freqtrade.commands.data_commands import start_download_data from freqtrade.commands.deploy_commands import (start_create_userdir, From 02563019fc84d559584ae007c6c9d5f4055da911 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Jan 2020 13:55:48 +0100 Subject: [PATCH 14/14] move setup_utils_config to configuration module --- freqtrade/commands/__init__.py | 1 - freqtrade/commands/data_commands.py | 3 +-- freqtrade/commands/deploy_commands.py | 3 +-- freqtrade/commands/hyperopt_commands.py | 2 +- freqtrade/commands/list_commands.py | 3 +-- freqtrade/commands/optimize_commands.py | 2 +- freqtrade/commands/pairlist_commands.py | 3 +-- freqtrade/commands/plot_commands.py | 2 +- freqtrade/configuration/__init__.py | 1 + .../{commands/utils.py => configuration/config_setup.py} | 5 +++-- tests/commands/__init__.py | 0 tests/{test_utils.py => commands/test_commands.py} | 8 ++++---- 12 files changed, 15 insertions(+), 18 deletions(-) rename freqtrade/{commands/utils.py => configuration/config_setup.py} (79%) create mode 100644 tests/commands/__init__.py rename tests/{test_utils.py => commands/test_commands.py} (99%) diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 1dadbec1e..990c1107a 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -23,4 +23,3 @@ from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import (start_plot_dataframe, start_plot_profit) from freqtrade.commands.trade_commands import start_trading -from freqtrade.commands.utils import setup_utils_configuration diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index b1c847d1d..c01772023 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -4,14 +4,13 @@ from typing import Any, Dict, List import arrow -from freqtrade.configuration import TimeRange +from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode -from .utils import setup_utils_configuration logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 892e15db1..99ae63244 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -3,6 +3,7 @@ import sys from pathlib import Path from typing import Any, Dict +from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import (copy_sample_files, create_userdata_dir) from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY @@ -10,8 +11,6 @@ from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template from freqtrade.state import RunMode -from .utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index db3f69050..5c6f25848 100644 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -4,7 +4,7 @@ from typing import Any, Dict, List from colorama import init as colorama_init -from .utils import setup_utils_configuration +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 90efaf66f..022822782 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -8,6 +8,7 @@ from typing import Any, Dict import rapidjson from tabulate import tabulate +from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_STRATEGY from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, ccxt_exchanges, @@ -16,8 +17,6 @@ from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode -from .utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index e635e61fd..a2d1b4601 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -2,7 +2,7 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.commands.utils import setup_utils_configuration +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index 06ba2e680..bf0b217a5 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -3,11 +3,10 @@ from typing import Any, Dict import rapidjson +from freqtrade.configuration import setup_utils_configuration from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode -from .utils import setup_utils_configuration - logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 4c68cc3c5..028933ba7 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -1,8 +1,8 @@ from typing import Any, Dict +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.commands.utils import setup_utils_configuration def validate_plot_args(args: Dict[str, Any]): diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 54fc4e427..d41ac97ec 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 +from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.check_exchange import check_exchange, remove_credentials from freqtrade.configuration.timerange import TimeRange from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/commands/utils.py b/freqtrade/configuration/config_setup.py similarity index 79% rename from freqtrade/commands/utils.py rename to freqtrade/configuration/config_setup.py index f77a35383..64f283e42 100644 --- a/freqtrade/commands/utils.py +++ b/freqtrade/configuration/config_setup.py @@ -1,8 +1,9 @@ import logging from typing import Any, Dict -from freqtrade.configuration import (Configuration, remove_credentials, - validate_config_consistency) +from .config_validation import validate_config_consistency +from .configuration import Configuration +from .check_exchange import remove_credentials from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils.py b/tests/commands/test_commands.py similarity index 99% rename from tests/test_utils.py rename to tests/commands/test_commands.py index 1328e3981..65d7f6eaf 100644 --- a/tests/test_utils.py +++ b/tests/commands/test_commands.py @@ -4,13 +4,13 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade.commands import (setup_utils_configuration, - start_create_userdir, start_download_data, +from freqtrade.commands import (start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_list_exchanges, start_list_markets, start_list_strategies, start_list_timeframes, start_new_hyperopt, start_new_strategy, start_test_pairlist, start_trading) +from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, @@ -639,7 +639,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent / "strategy"), + str(Path(__file__).parent.parent / "strategy"), "-1" ] pargs = get_args(args) @@ -654,7 +654,7 @@ def test_start_list_strategies(mocker, caplog, capsys): args = [ "list-strategies", "--strategy-path", - str(Path(__file__).parent / "strategy"), + str(Path(__file__).parent.parent / "strategy"), ] pargs = get_args(args) # pargs['config'] = None