From 05deb9e09bdccd0c19904ed2687cc6bd8f2bf29f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Aug 2019 14:42:44 +0200 Subject: [PATCH] Migrate download-script logic to utils.py --- freqtrade/configuration/arguments.py | 19 ++++++-- freqtrade/tests/test_arguments.py | 18 ++++---- freqtrade/utils.py | 66 ++++++++++++++++++++++++++-- 3 files changed, 87 insertions(+), 16 deletions(-) diff --git a/freqtrade/configuration/arguments.py b/freqtrade/configuration/arguments.py index 926d02f8f..8fa16318a 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/configuration/arguments.py @@ -30,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_LIST_EXCHANGES = ["print_one_column"] -ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] +ARGS_DOWNLOADER = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"] ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", @@ -40,6 +40,8 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY + ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY + ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"]) +NO_CONF_REQURIED = ["start_download_data"] + class Arguments(object): """ @@ -75,7 +77,10 @@ class Arguments(object): # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) - if not self._no_default_config and parsed_arg.config is None: + # Allow no-config for certain commands (like downloading / plotting) + if (not self._no_default_config and parsed_arg.config is None + and not (hasattr(parsed_arg, 'func') + and parsed_arg.func.__name__ in NO_CONF_REQURIED)): parsed_arg.config = [constants.DEFAULT_CONFIG] return parsed_arg @@ -93,7 +98,7 @@ class Arguments(object): :return: None """ from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import start_list_exchanges + from freqtrade.utils import start_download_data, start_list_exchanges subparsers = self.parser.add_subparsers(dest='subparser') @@ -119,3 +124,11 @@ class Arguments(object): ) list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) + + # Add download-data subcommand + download_data_cmd = subparsers.add_parser( + 'download-data', + help='Download backtesting data.' + ) + download_data_cmd.set_defaults(func=start_download_data) + self._build_args(optionlist=ARGS_DOWNLOADER, parser=download_data_cmd) diff --git a/freqtrade/tests/test_arguments.py b/freqtrade/tests/test_arguments.py index 2cb7ff6d7..31ab9dea8 100644 --- a/freqtrade/tests/test_arguments.py +++ b/freqtrade/tests/test_arguments.py @@ -50,10 +50,10 @@ def test_parse_args_verbose() -> None: def test_common_scripts_options() -> None: - arguments = Arguments(['-p', 'ETH/BTC'], '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() - assert args.pairs == 'ETH/BTC' + args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC'], '').get_parsed_arg() + + assert args.pairs == ['ETH/BTC', 'XRP/BTC'] + assert hasattr(args, "func") def test_parse_args_version() -> None: @@ -135,14 +135,14 @@ def test_parse_args_hyperopt_custom() -> None: def test_download_data_options() -> None: args = [ - '--pairs-file', 'file_with_pairs', '--datadir', 'datadir/directory', + 'download-data', + '--pairs-file', 'file_with_pairs', '--days', '30', '--exchange', 'binance' ] - arguments = Arguments(args, '') - arguments._build_args(ARGS_DOWNLOADER) - args = arguments._parse_args() + args = Arguments(args, '').get_parsed_arg() + assert args.pairs_file == 'file_with_pairs' assert args.datadir == 'datadir/directory' assert args.days == 30 @@ -162,7 +162,7 @@ def test_plot_dataframe_options() -> None: assert pargs.indicators1 == "sma10,sma100" assert pargs.indicators2 == "macd,fastd,fastk" assert pargs.plot_limit == 30 - assert pargs.pairs == "UNITTEST/BTC" + assert pargs.pairs == ["UNITTEST/BTC"] def test_check_int_positive() -> None: diff --git a/freqtrade/utils.py b/freqtrade/utils.py index d550ef43c..d2770ba1a 100644 --- a/freqtrade/utils.py +++ b/freqtrade/utils.py @@ -1,11 +1,16 @@ import logging +import sys from argparse import Namespace +from pathlib import Path from typing import Any, Dict -from freqtrade.configuration import Configuration -from freqtrade.exchange import available_exchanges -from freqtrade.state import RunMode +import arrow +from freqtrade.configuration import Configuration, TimeRange +from freqtrade.data.history import download_pair_history +from freqtrade.exchange import available_exchanges +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -17,7 +22,7 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any :return: Configuration """ configuration = Configuration(args, method) - config = configuration.load_config() + config = configuration.get_config() config['exchange']['dry_run'] = True # Ensure we do not use Exchange credentials @@ -39,3 +44,56 @@ def start_list_exchanges(args: Namespace) -> None: else: print(f"Exchanges supported by ccxt and available for Freqtrade: " f"{', '.join(available_exchanges())}") + + +def start_download_data(args: Namespace) -> None: + """ + Download data based + """ + config = setup_utils_configuration(args, RunMode.OTHER) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + dl_path = Path(config['datadir']) + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {dl_path}') + + pairs_not_available = [] + + try: + # Init exchange + exchange = ExchangeResolver(config['exchange']['name'], config).exchange + + for pair in config["pairs"]: + if pair not in exchange._api.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for ticker_interval in config["timeframes"]: + pair_print = pair.replace('/', '_') + filename = f'{pair_print}-{ticker_interval}.json' + dl_file = dl_path.joinpath(filename) + if args.erase and dl_file.exists(): + logger.info( + f'Deleting existing data for pair {pair}, interval {ticker_interval}.') + dl_file.unlink() + + logger.info(f'Downloading pair {pair}, interval {ticker_interval}.') + download_pair_history(datadir=dl_path, exchange=exchange, + pair=pair, ticker_interval=str(ticker_interval), + timerange=timerange) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info( + f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {config['exchange']['name']}.") + + # configuration.resolve_pairs_list() + print(config)