Merge pull request #1893 from hroff-1902/refactor-download-script

refactoring download_backtest_data.py

scripts/download_backtest_data.py

@@ -1,55 +1,66 @@
 #!/usr/bin/env python3
 """
-This script generates json data
+This script generates json files with pairs history data
 """
+import arrow
 import json
 import sys
 from pathlib import Path
-import arrow
-from typing import Any, Dict
+from typing import Any, Dict, List
 
-from freqtrade.arguments import Arguments
-from freqtrade.arguments import TimeRange
-from freqtrade.exchange import Exchange
+from freqtrade.arguments import Arguments, TimeRange
+from freqtrade.configuration import Configuration
 from freqtrade.data.history import download_pair_history
-from freqtrade.configuration import Configuration, set_loggers
+from freqtrade.exchange import Exchange
 from freqtrade.misc import deep_merge_dicts
 
 import logging
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-)
-set_loggers(0)
+
+logger = logging.getLogger('download_backtest_data')
 
 DEFAULT_DL_PATH = 'user_data/data'
 
 arguments = Arguments(sys.argv[1:], 'download utility')
-arguments.testdata_dl_options()
-args = arguments.parse_args()
+arguments.download_data_options()
 
-timeframes = args.timeframes
+# Do not read the default config if config is not specified
+# in the command line options explicitely
+args = arguments.parse_args(no_default_config=True)
+
+# Use bittrex as default exchange
+exchange_name = args.exchange or 'bittrex'
+
+pairs: List = []
+
+configuration = Configuration(args)
+config: Dict[str, Any] = {}
 
 if args.config:
-    configuration = Configuration(args)
-
-    config: Dict[str, Any] = {}
     # Now expecting a list of config filenames here, not a string
     for path in args.config:
-        print(f"Using config: {path}...")
+        logger.info(f"Using config: {path}...")
         # Merge config options, overwriting old values
         config = deep_merge_dicts(configuration._load_config_file(path), config)
 
     config['stake_currency'] = ''
     # Ensure we do not use Exchange credentials
+    config['exchange']['dry_run'] = True
     config['exchange']['key'] = ''
     config['exchange']['secret'] = ''
+
+    pairs = config['exchange']['pair_whitelist']
+
+    if config.get('ticker_interval'):
+        timeframes = args.timeframes or [config.get('ticker_interval')]
+    else:
+        timeframes = args.timeframes or ['1m', '5m']
+
 else:
     config = {
         'stake_currency': '',
         'dry_run': True,
         'exchange': {
-            'name': args.exchange,
+            'name': exchange_name,
             'key': '',
             'secret': '',
             'pair_whitelist': [],
@@ -59,56 +70,72 @@ else:
             }
         }
     }
+    timeframes = args.timeframes or ['1m', '5m']
 
+configuration._load_logging_config(config)
 
-dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name'])
-if args.export:
-    dl_path = Path(args.export)
+if args.config and args.exchange:
+    logger.warning("The --exchange option is ignored, "
+                   "using exchange settings from the configuration file.")
 
-if not dl_path.is_dir():
-    sys.exit(f'Directory {dl_path} does not exist.')
+# Check if the exchange set by the user is supported
+configuration.check_exchange(config)
+
+configuration._load_datadir_config(config)
+
+dl_path = Path(config['datadir'])
 
 pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
-if not pairs_file.exists():
-    sys.exit(f'No pairs file found with path {pairs_file}.')
 
-with pairs_file.open() as file:
-    PAIRS = list(set(json.load(file)))
+if not pairs or args.pairs_file:
+    logger.info(f'Reading pairs file "{pairs_file}".')
+    # Download pairs from the pairs file if no config is specified
+    # or if pairs file is specified explicitely
+    if not pairs_file.exists():
+        sys.exit(f'No pairs file found with path "{pairs_file}".')
 
-PAIRS.sort()
+    with pairs_file.open() as file:
+        pairs = list(set(json.load(file)))
 
+    pairs.sort()
 
 timerange = TimeRange()
 if args.days:
     time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
     timerange = arguments.parse_timerange(f'{time_since}-')
 
+logger.info(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}')
 
-print(f'About to download pairs: {PAIRS} to {dl_path}')
-
-# Init exchange
-exchange = Exchange(config)
 pairs_not_available = []
 
-for pair in PAIRS:
-    if pair not in exchange._api.markets:
-        pairs_not_available.append(pair)
-        print(f"skipping pair {pair}")
-        continue
-    for ticker_interval in timeframes:
-        pair_print = pair.replace('/', '_')
-        filename = f'{pair_print}-{ticker_interval}.json'
-        dl_file = dl_path.joinpath(filename)
-        if args.erase and dl_file.exists():
-            print(f'Deleting existing data for pair {pair}, interval {ticker_interval}')
-            dl_file.unlink()
+try:
+    # Init exchange
+    exchange = Exchange(config)
 
-        print(f'downloading pair {pair}, interval {ticker_interval}')
-        download_pair_history(datadir=dl_path, exchange=exchange,
-                              pair=pair,
-                              ticker_interval=ticker_interval,
-                              timerange=timerange)
+    for pair in pairs:
+        if pair not in exchange._api.markets:
+            pairs_not_available.append(pair)
+            logger.info(f"Skipping pair {pair}...")
+            continue
+        for ticker_interval in timeframes:
+            pair_print = pair.replace('/', '_')
+            filename = f'{pair_print}-{ticker_interval}.json'
+            dl_file = dl_path.joinpath(filename)
+            if args.erase and dl_file.exists():
+                logger.info(
+                    f'Deleting existing data for pair {pair}, interval {ticker_interval}.')
+                dl_file.unlink()
 
+            logger.info(f'Downloading pair {pair}, interval {ticker_interval}.')
+            download_pair_history(datadir=dl_path, exchange=exchange,
+                                  pair=pair, ticker_interval=str(ticker_interval),
+                                  timerange=timerange)
 
-if pairs_not_available:
-    print(f"Pairs [{','.join(pairs_not_available)}] not availble.")
+except KeyboardInterrupt:
+    sys.exit("SIGINT received, aborting ...")
+
+finally:
+    if pairs_not_available:
+        logger.info(
+            f"Pairs [{','.join(pairs_not_available)}] not available "
+            f"on exchange {config['exchange']['name']}.")