refactoring download_backtest_data.py

freqtrade/arguments.py

@@ -47,7 +47,7 @@ class Arguments(object):
 
         return self.parsed_arg
 
-    def parse_args(self) -> argparse.Namespace:
+    def parse_args(self, no_default_config: bool = False) -> argparse.Namespace:
         """
         Parses given arguments and returns an argparse Namespace instance.
         """
@@ -55,7 +55,7 @@ class Arguments(object):
 
         # Workaround issue in argparse with action='append' and default value
         # (see https://bugs.python.org/issue16399)
-        if parsed_arg.config is None:
+        if parsed_arg.config is None and not no_default_config:
             parsed_arg.config = [constants.DEFAULT_CONFIG]
 
         return parsed_arg
@@ -427,26 +427,24 @@ class Arguments(object):
             default=None
         )
 
-    def testdata_dl_options(self) -> None:
+    def download_data_options(self) -> None:
         """
         Parses given arguments for testdata download
         """
         self.parser.add_argument(
-            '--pairs-file',
+            '-v', '--verbose',
-            help='File containing a list of pairs to download.',
+            help='Verbose mode (-vv for more, -vvv to get all messages).',
-            dest='pairs_file',
+            action='count',
-            default=None,
+            dest='loglevel',
-            metavar='PATH',
+            default=0,
         )
-
         self.parser.add_argument(
-            '--export',
+            '--logfile',
-            help='Export files to given dir.',
+            help='Log to the file specified',
-            dest='export',
+            dest='logfile',
-            default=None,
+            type=str,
-            metavar='PATH',
+            metavar='FILE'
         )
-
         self.parser.add_argument(
             '-c', '--config',
             help='Specify configuration file (default: %(default)s). '
@@ -456,7 +454,21 @@ class Arguments(object):
             type=str,
             metavar='PATH',
         )
-
+        self.parser.add_argument(
+            '-d', '--datadir',
+            help='Path to backtest data.',
+            dest='datadir',
+            default=None,
+            type=str,
+            metavar='PATH',
+        )
+        self.parser.add_argument(
+            '--pairs-file',
+            help='File containing a list of pairs to download.',
+            dest='pairs_file',
+            default=None,
+            metavar='PATH',
+        )
         self.parser.add_argument(
             '--days',
             help='Download data for given number of days.',
@@ -465,7 +477,6 @@ class Arguments(object):
             metavar='INT',
             default=None
         )
-
         self.parser.add_argument(
             '--exchange',
             help='Exchange name (default: %(default)s). Only valid if no config is provided.',
@@ -473,7 +484,6 @@ class Arguments(object):
             type=str,
             default='bittrex'
         )
-
         self.parser.add_argument(
             '-t', '--timeframes',
             help='Specify which tickers to download. Space separated list. \
@@ -484,7 +494,6 @@ class Arguments(object):
             nargs='+',
             dest='timeframes',
         )
-
         self.parser.add_argument(
             '--erase',
             help='Clean all existing data for the selected exchange/pairs/timeframes.',

freqtrade/configuration.py

@@ -122,12 +122,11 @@ class Configuration(object):
 
         return conf
 
-    def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
+    def _load_logging_config(self, config: Dict[str, Any]) -> None:
         """
-        Extract information for sys.argv and load common configuration
+        Extract information for sys.argv and load logging configuration:
-        :return: configuration as dictionary
+        the --loglevel, --logfile options
         """
-
         # Log level
         if 'loglevel' in self.args and self.args.loglevel:
             config.update({'verbosity': self.args.loglevel})
@@ -153,6 +152,13 @@ class Configuration(object):
         set_loggers(config['verbosity'])
         logger.info('Verbosity set to %s', config['verbosity'])
 
+    def _load_common_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Extract information for sys.argv and load common configuration
+        :return: configuration as dictionary
+        """
+        self._load_logging_config(config)
+
         # Support for sd_notify
         if self.args.sd_notify:
             config['internals'].update({'sd_notify': True})
@@ -228,6 +234,17 @@ class Configuration(object):
             else:
                 logger.info(logstring.format(config[argname]))
 
+    def _load_datadir_config(self, config: Dict[str, Any]) -> None:
+        """
+        Extract information for sys.argv and load datadir configuration:
+        the --datadir option
+        """
+        if 'datadir' in self.args and self.args.datadir:
+            config.update({'datadir': self._create_datadir(config, self.args.datadir)})
+        else:
+            config.update({'datadir': self._create_datadir(config, None)})
+        logger.info('Using data folder: %s ...', config.get('datadir'))
+
     def _load_optimize_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         """
         Extract information for sys.argv and load Optimize configuration
@@ -263,11 +280,7 @@ class Configuration(object):
         self._args_to_config(config, argname='timerange',
                              logstring='Parameter --timerange detected: {} ...')
 
-        if 'datadir' in self.args and self.args.datadir:
+        self._load_datadir_config(config)
-            config.update({'datadir': self._create_datadir(config, self.args.datadir)})
-        else:
-            config.update({'datadir': self._create_datadir(config, None)})
-        logger.info('Using data folder: %s ...', config.get('datadir'))
 
         self._args_to_config(config, argname='refresh_pairs',
                              logstring='Parameter -r/--refresh-pairs-cached detected ...')

freqtrade/tests/test_arguments.py

@@ -170,18 +170,18 @@ def test_parse_args_hyperopt_custom() -> None:
     assert call_args.func is not None
 
 
-def test_testdata_dl_options() -> None:
+def test_download_data_options() -> None:
     args = [
         '--pairs-file', 'file_with_pairs',
-        '--export', 'export/folder',
+        '--datadir', 'datadir/folder',
         '--days', '30',
         '--exchange', 'binance'
     ]
     arguments = Arguments(args, '')
-    arguments.testdata_dl_options()
+    arguments.download_data_options()
     args = arguments.parse_args()
     assert args.pairs_file == 'file_with_pairs'
-    assert args.export == 'export/folder'
+    assert args.datadir == 'datadir/folder'
     assert args.days == 30
     assert args.exchange == 'binance'
 

scripts/download_backtest_data.py

@@ -1,39 +1,39 @@
 #!/usr/bin/env python3
 """
-This script generates json data
+This script generates json files with pairs history data
 """
+import arrow
 import json
 import sys
 from pathlib import Path
-import arrow
+from typing import Any, Dict, List
-from typing import Any, Dict
 
-from freqtrade.arguments import Arguments
+from freqtrade.arguments import Arguments, TimeRange
-from freqtrade.arguments import TimeRange
+from freqtrade.configuration import Configuration
-from freqtrade.exchange import Exchange
 from freqtrade.data.history import download_pair_history
-from freqtrade.configuration import Configuration, set_loggers
+from freqtrade.exchange import Exchange
 from freqtrade.misc import deep_merge_dicts
 
 import logging
-logging.basicConfig(
+
-    level=logging.INFO,
+logger = logging.getLogger('download_backtest_data')
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-)
-set_loggers(0)
 
 DEFAULT_DL_PATH = 'user_data/data'
 
 arguments = Arguments(sys.argv[1:], 'download utility')
-arguments.testdata_dl_options()
+arguments.download_data_options()
-args = arguments.parse_args()
+
+# Do not read the default config if config is not specified
+# in the command line options explicitely
+args = arguments.parse_args(no_default_config=True)
 
 timeframes = args.timeframes
+pairs: List = []
+
+configuration = Configuration(args)
+config: Dict[str, Any] = {}
 
 if args.config:
-    configuration = Configuration(args)
-
-    config: Dict[str, Any] = {}
     # Now expecting a list of config filenames here, not a string
     for path in args.config:
         print(f"Using config: {path}...")
@@ -42,9 +42,19 @@ if args.config:
 
     config['stake_currency'] = ''
     # Ensure we do not use Exchange credentials
+    config['exchange']['dry_run'] = True
     config['exchange']['key'] = ''
     config['exchange']['secret'] = ''
+
+    if args.exchange:
+        config['exchange']['name'] = args.exchange
+
+    pairs = config['exchange']['pair_whitelist']
+    timeframes = [config['ticker_interval']]
+
 else:
+    if not args.exchange:
+        sys.exit("No exchange specified.")
     config = {
         'stake_currency': '',
         'dry_run': True,
@@ -60,55 +70,59 @@ else:
         }
     }
 
+configuration._load_logging_config(config)
+configuration._load_datadir_config(config)
 
-dl_path = Path(DEFAULT_DL_PATH).joinpath(config['exchange']['name'])
+dl_path = Path(config['datadir'])
-if args.export:
-    dl_path = Path(args.export)
-
-if not dl_path.is_dir():
-    sys.exit(f'Directory {dl_path} does not exist.')
 
 pairs_file = Path(args.pairs_file) if args.pairs_file else dl_path.joinpath('pairs.json')
-if not pairs_file.exists():
-    sys.exit(f'No pairs file found with path {pairs_file}.')
 
-with pairs_file.open() as file:
+if not pairs or args.pairs_file:
-    PAIRS = list(set(json.load(file)))
+    print(f'Reading pairs file "{pairs_file}".')
+    # Download pairs from the pairs file if no config is specified
+    # or if pairs file is specified explicitely
+    if not pairs_file.exists():
+        sys.exit(f'No pairs file found with path "{pairs_file}".')
 
-PAIRS.sort()
+    with pairs_file.open() as file:
+        pairs = list(set(json.load(file)))
 
+    pairs.sort()
 
 timerange = TimeRange()
 if args.days:
     time_since = arrow.utcnow().shift(days=-args.days).strftime("%Y%m%d")
     timerange = arguments.parse_timerange(f'{time_since}-')
 
+print(f'About to download pairs: {pairs}, intervals: {timeframes} to {dl_path}')
 
-print(f'About to download pairs: {PAIRS} to {dl_path}')
-
-# Init exchange
-exchange = Exchange(config)
 pairs_not_available = []
 
-for pair in PAIRS:
+try:
-    if pair not in exchange._api.markets:
+    # Init exchange
-        pairs_not_available.append(pair)
+    exchange = Exchange(config)
-        print(f"skipping pair {pair}")
-        continue
-    for ticker_interval in timeframes:
-        pair_print = pair.replace('/', '_')
-        filename = f'{pair_print}-{ticker_interval}.json'
-        dl_file = dl_path.joinpath(filename)
-        if args.erase and dl_file.exists():
-            print(f'Deleting existing data for pair {pair}, interval {ticker_interval}')
-            dl_file.unlink()
 
-        print(f'downloading pair {pair}, interval {ticker_interval}')
+    for pair in pairs:
-        download_pair_history(datadir=dl_path, exchange=exchange,
+        if pair not in exchange._api.markets:
-                              pair=pair,
+            pairs_not_available.append(pair)
-                              ticker_interval=ticker_interval,
+            print(f"skipping pair {pair}")
-                              timerange=timerange)
+            continue
+        for ticker_interval in timeframes:
+            pair_print = pair.replace('/', '_')
+            filename = f'{pair_print}-{ticker_interval}.json'
+            dl_file = dl_path.joinpath(filename)
+            if args.erase and dl_file.exists():
+                print(f'Deleting existing data for pair {pair}, interval {ticker_interval}')
+                dl_file.unlink()
 
+            print(f'downloading pair {pair}, interval {ticker_interval}')
+            download_pair_history(datadir=dl_path, exchange=exchange,
+                                  pair=pair, ticker_interval=ticker_interval,
+                                  timerange=timerange)
 
-if pairs_not_available:
+except KeyboardInterrupt:
-    print(f"Pairs [{','.join(pairs_not_available)}] not availble.")
+    sys.exit("SIGINT received, aborting ...")
+
+finally:
+    if pairs_not_available:
+        print(f"Pairs [{','.join(pairs_not_available)}] not availble.")