Migrate download-script logic to utils.py

freqtrade/configuration/arguments.py

@@ -30,7 +30,7 @@ ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 
 ARGS_LIST_EXCHANGES = ["print_one_column"]
 
-ARGS_DOWNLOADER = ARGS_COMMON + ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
+ARGS_DOWNLOADER = ["pairs", "pairs_file", "days", "exchange", "timeframes", "erase"]
 
 ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY +
                        ["pairs", "indicators1", "indicators2", "plot_limit", "db_url",
@@ -40,6 +40,8 @@ ARGS_PLOT_DATAFRAME = (ARGS_COMMON + ARGS_STRATEGY +
 ARGS_PLOT_PROFIT = (ARGS_COMMON + ARGS_STRATEGY +
                     ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source"])
 
+NO_CONF_REQURIED = ["start_download_data"]
+
 
 class Arguments(object):
     """
@@ -75,7 +77,10 @@ class Arguments(object):
 
         # Workaround issue in argparse with action='append' and default value
         # (see https://bugs.python.org/issue16399)
-        if not self._no_default_config and parsed_arg.config is None:
+        # Allow no-config for certain commands (like downloading / plotting)
+        if (not self._no_default_config and parsed_arg.config is None
+                and not (hasattr(parsed_arg, 'func')
+                         and parsed_arg.func.__name__ in NO_CONF_REQURIED)):
             parsed_arg.config = [constants.DEFAULT_CONFIG]
 
         return parsed_arg
@@ -93,7 +98,7 @@ class Arguments(object):
         :return: None
         """
         from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge
-        from freqtrade.utils import start_list_exchanges
+        from freqtrade.utils import start_download_data, start_list_exchanges
 
         subparsers = self.parser.add_subparsers(dest='subparser')
 
@@ -119,3 +124,11 @@ class Arguments(object):
         )
         list_exchanges_cmd.set_defaults(func=start_list_exchanges)
         self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd)
+
+        # Add download-data subcommand
+        download_data_cmd = subparsers.add_parser(
+            'download-data',
+            help='Download backtesting data.'
+        )
+        download_data_cmd.set_defaults(func=start_download_data)
+        self._build_args(optionlist=ARGS_DOWNLOADER, parser=download_data_cmd)

freqtrade/tests/test_arguments.py

@@ -50,10 +50,10 @@ def test_parse_args_verbose() -> None:
 
 
 def test_common_scripts_options() -> None:
-    arguments = Arguments(['-p', 'ETH/BTC'], '')
+    args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC'], '').get_parsed_arg()
-    arguments._build_args(ARGS_DOWNLOADER)
+
-    args = arguments._parse_args()
+    assert args.pairs == ['ETH/BTC', 'XRP/BTC']
-    assert args.pairs == 'ETH/BTC'
+    assert hasattr(args, "func")
 
 
 def test_parse_args_version() -> None:
@@ -135,14 +135,14 @@ def test_parse_args_hyperopt_custom() -> None:
 
 def test_download_data_options() -> None:
     args = [
-        '--pairs-file', 'file_with_pairs',
         '--datadir', 'datadir/directory',
+        'download-data',
+        '--pairs-file', 'file_with_pairs',
         '--days', '30',
         '--exchange', 'binance'
     ]
-    arguments = Arguments(args, '')
+    args = Arguments(args, '').get_parsed_arg()
-    arguments._build_args(ARGS_DOWNLOADER)
+
-    args = arguments._parse_args()
     assert args.pairs_file == 'file_with_pairs'
     assert args.datadir == 'datadir/directory'
     assert args.days == 30
@@ -162,7 +162,7 @@ def test_plot_dataframe_options() -> None:
     assert pargs.indicators1 == "sma10,sma100"
     assert pargs.indicators2 == "macd,fastd,fastk"
     assert pargs.plot_limit == 30
-    assert pargs.pairs == "UNITTEST/BTC"
+    assert pargs.pairs == ["UNITTEST/BTC"]
 
 
 def test_check_int_positive() -> None:

freqtrade/utils.py

@@ -1,11 +1,16 @@
 import logging
+import sys
 from argparse import Namespace
+from pathlib import Path
 from typing import Any, Dict
 
-from freqtrade.configuration import Configuration
+import arrow
-from freqtrade.exchange import available_exchanges
-from freqtrade.state import RunMode
 
+from freqtrade.configuration import Configuration, TimeRange
+from freqtrade.data.history import download_pair_history
+from freqtrade.exchange import available_exchanges
+from freqtrade.resolvers import ExchangeResolver
+from freqtrade.state import RunMode
 
 logger = logging.getLogger(__name__)
 
@@ -17,7 +22,7 @@ def setup_utils_configuration(args: Namespace, method: RunMode) -> Dict[str, Any
     :return: Configuration
     """
     configuration = Configuration(args, method)
-    config = configuration.load_config()
+    config = configuration.get_config()
 
     config['exchange']['dry_run'] = True
     # Ensure we do not use Exchange credentials
@@ -39,3 +44,56 @@ def start_list_exchanges(args: Namespace) -> None:
     else:
         print(f"Exchanges supported by ccxt and available for Freqtrade: "
               f"{', '.join(available_exchanges())}")
+
+
+def start_download_data(args: Namespace) -> None:
+    """
+    Download data based
+    """
+    config = setup_utils_configuration(args, RunMode.OTHER)
+
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+
+    dl_path = Path(config['datadir'])
+    logger.info(f'About to download pairs: {config["pairs"]}, '
+                f'intervals: {config["timeframes"]} to {dl_path}')
+
+    pairs_not_available = []
+
+    try:
+        # Init exchange
+        exchange = ExchangeResolver(config['exchange']['name'], config).exchange
+
+        for pair in config["pairs"]:
+            if pair not in exchange._api.markets:
+                pairs_not_available.append(pair)
+                logger.info(f"Skipping pair {pair}...")
+                continue
+            for ticker_interval in config["timeframes"]:
+                pair_print = pair.replace('/', '_')
+                filename = f'{pair_print}-{ticker_interval}.json'
+                dl_file = dl_path.joinpath(filename)
+                if args.erase and dl_file.exists():
+                    logger.info(
+                        f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
+                    dl_file.unlink()
+
+                logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
+                download_pair_history(datadir=dl_path, exchange=exchange,
+                                      pair=pair, ticker_interval=str(ticker_interval),
+                                      timerange=timerange)
+
+    except KeyboardInterrupt:
+        sys.exit("SIGINT received, aborting ...")
+
+    finally:
+        if pairs_not_available:
+            logger.info(
+                f"Pairs [{','.join(pairs_not_available)}] not available "
+                f"on exchange {config['exchange']['name']}.")
+
+    # configuration.resolve_pairs_list()
+    print(config)