Merge pull request #379 from kryofly/testdata-download2

support download for multiple testdata sets

docs/backtesting.md

@@ -51,6 +51,21 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --live
 python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101
 ```
 
+To update your testdata directory, or download into another testdata directory:
+```bash
+mkdir freqtrade/tests/testdata-20180113
+cp freqtrade/tests/testdata/pairs.json freqtrade/tests/testdata-20180113
+cd freqtrade/tests/testdata-20180113
+
+Possibly edit pairs.json file to include/exclude pairs
+
+python download_backtest_data.py -p pairs.json
+```
+
+The script will read your pairs.json file, and download ticker data
+into the current working directory.
+
+
 For help about backtesting usage, please refer to 
 [Backtesting commands](#backtesting-commands).
 

freqtrade/misc.py

@@ -15,6 +15,11 @@ from freqtrade import __version__
 logger = logging.getLogger(__name__)
 
 
+def file_dump_json(filename, data):
+    with open(filename, 'w') as fp:
+        json.dump(data, fp)
+
+
 class State(enum.Enum):
     RUNNING = 0
     STOPPED = 1

freqtrade/tests/testdata/download_backtest_data.py

@@ -1,29 +1,38 @@
 #!/usr/bin/env python3
 
 """This script generate json data from bittrex"""
+import sys
 import json
-from os import path
 
 from freqtrade import exchange
 from freqtrade.exchange import Bittrex
+from freqtrade import misc
 
-PAIRS = [
-    'BTC_BCC', 'BTC_ETH', 'BTC_MER', 'BTC_POWR', 'BTC_ETC',
-    'BTC_OK', 'BTC_NEO', 'BTC_EMC2', 'BTC_DASH', 'BTC_LSK',
-    'BTC_LTC', 'BTC_XZC', 'BTC_OMG', 'BTC_STRAT', 'BTC_XRP',
-    'BTC_QTUM', 'BTC_WAVES', 'BTC_VTC', 'BTC_XLM', 'BTC_MCO'
-]
-TICKER_INTERVAL = 5  # ticker interval in minutes (currently implemented: 1 and 5)
-OUTPUT_DIR = path.dirname(path.realpath(__file__))
+parser = misc.common_args_parser('download utility')
+parser.add_argument(
+        '-p', '--pair',
+        help='JSON file containing pairs to download',
+        dest='pair',
+        default=None
+)
+args = parser.parse_args(sys.argv[1:])
+
+TICKER_INTERVALS = [1, 5]  # ticker interval in minutes (currently implemented: 1 and 5)
+PAIRS = []
+
+if args.pair:
+    with open(args.pair) as file:
+        PAIRS = json.load(file)
+PAIRS = list(set(PAIRS))
+
+print('About to download pairs:', PAIRS)
 
 # Init Bittrex exchange
 exchange._API = Bittrex({'key': '', 'secret': ''})
 
 for pair in PAIRS:
-    data = exchange.get_ticker_history(pair, TICKER_INTERVAL)
-    filename = path.join(OUTPUT_DIR, '{}-{}.json'.format(
-        pair,
-        TICKER_INTERVAL,
-    ))
-    with open(filename, 'w') as fp:
-        json.dump(data, fp)
+    for tick_interval in TICKER_INTERVALS:
+        print('downloading pair %s, interval %s' % (pair, tick_interval))
+        data = exchange.get_ticker_history(pair, tick_interval)
+        filename = '{}-{}.json'.format(pair, tick_interval)
+        misc.file_dump_json(filename, data)

freqtrade/tests/testdata/pairs.json

@@ -0,0 +1,23 @@
+[
+ "BTC_ADA",
+ "BTC_BAT",
+ "BTC_DASH",
+ "BTC_ETC",
+ "BTC_ETH",
+ "BTC_GBYTE",
+ "BTC_LSK",
+ "BTC_LTC",
+ "BTC_NEO",
+ "BTC_NXT",
+ "BTC_POWR",
+ "BTC_STORJ",
+ "BTC_QTUM",
+ "BTC_WAVES",
+ "BTC_VTC",
+ "BTC_XLM",
+ "BTC_XMR",
+ "BTC_XVG",
+ "BTC_XRP",
+ "BTC_ZEC"
+]
+