Merge pull request #840 from freqtrade/improve_downloader

Improve ticker downloader

.travis.yml

@@ -21,10 +21,10 @@ jobs:
     - script: pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/
     - script:
       - cp config.json.example config.json
-      - python freqtrade/main.py backtesting
+      - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting
     - script:
       - cp config.json.example config.json
-      - python freqtrade/main.py hyperopt -e 5
+      - python freqtrade/main.py --datadir freqtrade/tests/testdata hyperopt -e 5
     - script: flake8 freqtrade
     - script: mypy freqtrade
 after_success:

docs/backtesting.md

@@ -93,22 +93,30 @@ The full timerange specification:
                                                 `--timerange=1527595200-1527618600`
 
 
-**Update testdata directory**
+**Downloading new set of ticker data**
-To update your testdata directory, or download into another testdata directory:
+To download new set of backtesting ticker data, you can use a download script.
-```bash
-mkdir -p user_data/data/testdata-20180113
-cp freqtrade/tests/testdata/pairs.json user_data/data/testdata-20180113
-cd user_data/data/testdata-20180113
-```
 
-Possibly edit `pairs.json` file to include/exclude pairs
+If you are using Binance for example:
+- create a folder `user_data/data/binance` and copy `pairs.json` in that folder.
+- update the `pairs.json` to contain the currency pairs you are interested in.
 
 ```bash
-python3 freqtrade/tests/testdata/download_backtest_data.py -p pairs.json
+mkdir -p user_data/data/binance
+cp freqtrade/tests/testdata/pairs.json user_data/data/binance
 ```
 
-The script will read your `pairs.json` file, and download ticker data
+Then run:
-into the current working directory.
+
+```bash
+python scripts/download_backtest_data --exchange binance
+```
+
+This will download ticker data for all the currency pairs you defined in `pairs.json`.
+
+- To use a different folder than the exchange specific default, use `--export user_data/data/some_directory`.
+- To change the exchange used to download the tickers, use `--exchange`. Default is `bittrex`.
+- To use `pairs.json` from some other folder, use `--pairs-file some_other_dir/pairs.json`.
+- To download ticker data for only 10 days, use `--days 10`.
 
 
 For help about backtesting usage, please refer to 

freqtrade/arguments.py

@@ -4,7 +4,6 @@ This module contains the argument manager class
 
 import argparse
 import logging
-import os
 import re
 import arrow
 from typing import List, Tuple, Optional
@@ -72,9 +71,9 @@ class Arguments(object):
         )
         self.parser.add_argument(
             '-d', '--datadir',
-            help='path to backtest data (default: %(default)s',
+            help='path to backtest data',
             dest='datadir',
-            default=os.path.join('freqtrade', 'tests', 'testdata'),
+            default=None,
             type=str,
             metavar='PATH',
         )
@@ -309,7 +308,7 @@ class Arguments(object):
 
         self.parser.add_argument(
             '--exchange',
-            help='Exchange name',
+            help='Exchange name (default: %(default)s)',
             dest='exchange',
             type=str,
             default='bittrex')

freqtrade/configuration.py

@@ -1,7 +1,7 @@
 """
 This module contains the configuration class
 """
-
+import os
 import json
 import logging
 from argparse import Namespace
@@ -113,6 +113,14 @@ class Configuration(object):
 
         return config
 
+    def _create_default_datadir(self, config: Dict[str, Any]) -> str:
+        exchange_name = config.get('exchange', {}).get('name').lower()
+        default_path = os.path.join('user_data', 'data', exchange_name)
+        if not os.path.isdir(default_path):
+            os.makedirs(default_path)
+            logger.info(f'Created data directory: {default_path}')
+        return default_path
+
     def _load_backtesting_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
         """
         Extract information for sys.argv and load Backtesting configuration
@@ -145,7 +153,9 @@ class Configuration(object):
         # If --datadir is used we add it to the configuration
         if 'datadir' in self.args and self.args.datadir:
             config.update({'datadir': self.args.datadir})
-            logger.info('Using data folder: %s ...', self.args.datadir)
+        else:
+            config.update({'datadir': self._create_default_datadir(config)})
+        logger.info('Using data folder: %s ...', config.get('datadir'))
 
         # If -r/--refresh-pairs-cached is used we add it to the configuration
         if 'refresh_pairs' in self.args and self.args.refresh_pairs:

scripts/download_backtest_data.py

@@ -8,24 +8,28 @@ import arrow
 
 from freqtrade import (exchange, arguments, misc)
 
-DEFAULT_DL_PATH = 'freqtrade/tests/testdata'
+DEFAULT_DL_PATH = 'user_data/data'
 
 arguments = arguments.Arguments(sys.argv[1:], 'download utility')
 arguments.testdata_dl_options()
 args = arguments.parse_args()
 
 TICKER_INTERVALS = ['1m', '5m']
-PAIRS = []
 
-if args.pairs_file:
+dl_path = os.path.join(DEFAULT_DL_PATH, args.exchange)
-    with open(args.pairs_file) as file:
+if args.export:
-        PAIRS = json.load(file)
-PAIRS = list(set(PAIRS))
-
-dl_path = DEFAULT_DL_PATH
-if args.export and os.path.exists(args.export):
     dl_path = args.export
 
+if not os.path.isdir(dl_path):
+    sys.exit(f'Directory {dl_path} does not exist.')
+
+pairs_file = args.pairs_file if args.pairs_file else os.path.join(dl_path, 'pairs.json')
+if not os.path.isfile(pairs_file):
+    sys.exit(f'No pairs file found with path {pairs_file}.')
+
+with open(pairs_file) as file:
+    PAIRS = list(set(json.load(file)))
+
 since_time = None
 if args.days:
     since_time = arrow.utcnow().shift(days=-args.days).timestamp * 1000