From a107c4c7b4203c2f0232faf6f6040d03e7477293 Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Fri, 10 Aug 2018 11:08:28 +0200
Subject: [PATCH] Download using asyncio

---
 freqtrade/exchange/__init__.py    | 36 +++++++++++++++++++++++++++++--
 freqtrade/optimize/__init__.py    |  4 ++--
 scripts/download_backtest_data.py |  9 ++++----
 3 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py
index 0cc707772..2cfcfbde7 100644
--- a/freqtrade/exchange/__init__.py
+++ b/freqtrade/exchange/__init__.py
@@ -142,6 +142,7 @@ class Exchange(object):
 
         try:
             markets = self._api.load_markets()
+            asyncio.get_event_loop().run_until_complete(self._api_async.load_markets())
         except ccxt.BaseError as e:
             logger.warning('Unable to validate pairs (assuming they are correct). Reason: %s', e)
             return
@@ -341,12 +342,43 @@ class Exchange(object):
             logger.info("returning cached ticker-data for %s", pair)
             return self._cached_ticker[pair]
 
+    def get_history(self, pair: str, tick_interval: str,
+                    since_ms: int) -> List:
+        """
+        Gets candle history using asyncio and returns the list of candles.
+        Handles all async doing.
+        """
+        return asyncio.get_event_loop().run_until_complete(
+            self._async_get_history(pair=pair, tick_interval=tick_interval,
+                                    since_ms=since_ms))
+
+    async def _async_get_history(self, pair: str,
+                                 tick_interval: str,
+                                 since_ms: int) -> List:
+        # Assume exchange returns 500 candles
+        _LIMIT = 500
+
+        one_call = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60 * _LIMIT * 1000
+        logger.debug("one_call: %s", one_call)
+        input_coroutines = [self.async_get_candle_history(
+            pair, tick_interval, since) for since in
+                range(since_ms, int(time.time() * 1000), one_call)]
+        tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
+
+        # Combine tickers
+        data = []
+        for tick in tickers:
+            if tick[0] == pair:
+                data.extend(tick[1])
+        logger.info("downloaded %s with length %s.", pair, len(data))
+        return data
+
     async def async_get_candles_history(self, pairs: List[str],
                                         tick_interval: str) -> List[Tuple[str, List]]:
         # COMMENTED CODE IS FOR DISCUSSION: where should we close the loop on async ?
         # loop = asyncio.new_event_loop()
         # asyncio.set_event_loop(loop)
-        await self._api_async.load_markets()
+        # await self._api_async.load_markets()
         input_coroutines = [self.async_get_candle_history(
             symbol, tick_interval) for symbol in pairs]
         tickers = await asyncio.gather(*input_coroutines, return_exceptions=True)
@@ -357,7 +389,7 @@ class Exchange(object):
                                        since_ms: Optional[int] = None) -> Tuple[str, List]:
         try:
             # fetch ohlcv asynchronously
-            logger.debug("fetching %s ...", pair)
+            logger.debug("fetching %s since %s ...", pair, since_ms)
 
             # Calculating ticker interval in second
             interval_in_sec = constants.TICKER_INTERVAL_MINUTES[tick_interval] * 60
diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py
index 502407f07..49b286fe8 100644
--- a/freqtrade/optimize/__init__.py
+++ b/freqtrade/optimize/__init__.py
@@ -218,8 +218,8 @@ def download_backtesting_testdata(datadir: str,
     logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
     logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
 
-    new_data = exchange.get_candle_history(pair=pair, tick_interval=tick_interval,
-                                           since_ms=since_ms)
+    new_data = exchange.get_history(pair=pair, tick_interval=tick_interval,
+                                    since_ms=since_ms)
     data.extend(new_data)
 
     logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
diff --git a/scripts/download_backtest_data.py b/scripts/download_backtest_data.py
index 686098f94..27c4c1e1c 100755
--- a/scripts/download_backtest_data.py
+++ b/scripts/download_backtest_data.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-"""This script generate json data from bittrex"""
+"""This script generate json data"""
 import json
 import sys
 from pathlib import Path
@@ -52,9 +52,10 @@ exchange = Exchange({'key': '',
                      'stake_currency': '',
                      'dry_run': True,
                      'exchange': {
-                        'name': args.exchange,
-                        'pair_whitelist': []
-                        }
+                         'name': args.exchange,
+                         'pair_whitelist': [],
+                         'ccxt_rate_limit': False
+                     }
                      })
 pairs_not_available = []