From 3277e491f18835fd7697bc4a4467277b5ba27703 Mon Sep 17 00:00:00 2001
From: kryofly <kryofly@space>
Date: Sat, 13 Jan 2018 17:39:36 +0100
Subject: [PATCH] support download for multiple testdata sets

---
 docs/backtesting.md                           | 15 +++++++
 freqtrade/misc.py                             |  5 +++
 .../tests/testdata/download_backtest_data.py  | 41 +++++++++++--------
 freqtrade/tests/testdata/pairs.json           | 23 +++++++++++
 4 files changed, 68 insertions(+), 16 deletions(-)
 create mode 100644 freqtrade/tests/testdata/pairs.json

diff --git a/docs/backtesting.md b/docs/backtesting.md
index c426e2b5c..8574d9dc2 100644
--- a/docs/backtesting.md
+++ b/docs/backtesting.md
@@ -51,6 +51,21 @@ python3 ./freqtrade/main.py backtesting --realistic-simulation --live
 python3 ./freqtrade/main.py backtesting --datadir freqtrade/tests/testdata-20180101
 ```
 
+To update your testdata directory, or download into another testdata directory:
+```bash
+mkdir freqtrade/tests/testdata-20180113
+cp freqtrade/tests/testdata/pairs.json freqtrade/tests/testdata-20180113
+cd freqtrade/tests/testdata-20180113
+
+Possibly edit pairs.json file to include/exclude pairs
+
+python download_backtest_data.py -p pairs.json
+```
+
+The script will read your pairs.json file, and download ticker data
+into the current working directory.
+
+
 For help about backtesting usage, please refer to 
 [Backtesting commands](#backtesting-commands).
 
diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 3d70f6b25..979174f8d 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -15,6 +15,11 @@ from freqtrade import __version__
 logger = logging.getLogger(__name__)
 
 
+def file_dump_json(filename, data):
+    with open(filename, 'w') as fp:
+        json.dump(data, fp)
+
+
 class State(enum.Enum):
     RUNNING = 0
     STOPPED = 1
diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py
index 37cd4c95f..0cb545b3a 100755
--- a/freqtrade/tests/testdata/download_backtest_data.py
+++ b/freqtrade/tests/testdata/download_backtest_data.py
@@ -1,29 +1,38 @@
 #!/usr/bin/env python3
 
 """This script generate json data from bittrex"""
+import sys
 import json
-from os import path
 
 from freqtrade import exchange
 from freqtrade.exchange import Bittrex
+from freqtrade import misc
 
-PAIRS = [
-    'BTC_BCC', 'BTC_ETH', 'BTC_MER', 'BTC_POWR', 'BTC_ETC',
-    'BTC_OK', 'BTC_NEO', 'BTC_EMC2', 'BTC_DASH', 'BTC_LSK',
-    'BTC_LTC', 'BTC_XZC', 'BTC_OMG', 'BTC_STRAT', 'BTC_XRP',
-    'BTC_QTUM', 'BTC_WAVES', 'BTC_VTC', 'BTC_XLM', 'BTC_MCO'
-]
-TICKER_INTERVAL = 5  # ticker interval in minutes (currently implemented: 1 and 5)
-OUTPUT_DIR = path.dirname(path.realpath(__file__))
+parser = misc.common_args_parser('download utility')
+parser.add_argument(
+        '-p', '--pair',
+        help='JSON file containing pairs to download',
+        dest='pair',
+        default=None
+)
+args = parser.parse_args(sys.argv[1:])
+
+TICKER_INTERVALS = [1, 5]  # ticker interval in minutes (currently implemented: 1 and 5)
+PAIRS = []
+
+if args.pair:
+    with open(args.pair) as file:
+        PAIRS = json.load(file)
+PAIRS = list(set(PAIRS))
+
+print('About to download pairs:', PAIRS)
 
 # Init Bittrex exchange
 exchange._API = Bittrex({'key': '', 'secret': ''})
 
 for pair in PAIRS:
-    data = exchange.get_ticker_history(pair, TICKER_INTERVAL)
-    filename = path.join(OUTPUT_DIR, '{}-{}.json'.format(
-        pair,
-        TICKER_INTERVAL,
-    ))
-    with open(filename, 'w') as fp:
-        json.dump(data, fp)
+    for tick_interval in TICKER_INTERVALS:
+        print('downloading pair %s, interval %s' % (pair, tick_interval))
+        data = exchange.get_ticker_history(pair, tick_interval)
+        filename = '{}-{}.json'.format(pair, tick_interval)
+        misc.file_dump_json(filename, data)
diff --git a/freqtrade/tests/testdata/pairs.json b/freqtrade/tests/testdata/pairs.json
new file mode 100644
index 000000000..c3e339b4c
--- /dev/null
+++ b/freqtrade/tests/testdata/pairs.json
@@ -0,0 +1,23 @@
+[
+ "BTC_ADA",
+ "BTC_BAT",
+ "BTC_DASH",
+ "BTC_ETC",
+ "BTC_ETH",
+ "BTC_GBYTE",
+ "BTC_LSK",
+ "BTC_LTC",
+ "BTC_NEO",
+ "BTC_NXT",
+ "BTC_POWR",
+ "BTC_STORJ",
+ "BTC_QTUM",
+ "BTC_WAVES",
+ "BTC_VTC",
+ "BTC_XLM",
+ "BTC_XMR",
+ "BTC_XVG",
+ "BTC_XRP",
+ "BTC_ZEC"
+]
+