Parse ccxt ohlcv history into dict in get_ticker_history

freqtrade/analyze.py

@@ -9,6 +9,7 @@ from typing import Dict, List
 import arrow
 from pandas import DataFrame, to_datetime
 
+from freqtrade.misc import ticker_interval_to_minutes
 from freqtrade.exchange import get_ticker_history
 from freqtrade.strategy.strategy import Strategy
 
@@ -32,7 +33,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame:
         .rename(columns=columns)
     if 'BV' in frame:
         frame.drop('BV', 1, inplace=True)
-    frame['date'] = to_datetime(frame['date'], unit='ms')
+    frame['date'] = to_datetime(frame['date'], format='%Y-%m-%dT%H:%M:%S.%f')
     frame.sort_values('date', inplace=True)
     return frame
 
@@ -84,10 +85,11 @@ def analyze_ticker(ticker_history: List[Dict]) -> DataFrame:
 
 # FIX: Maybe return False, if an error has occured,
 #      Otherwise we might mask an error as an non-signal-scenario
-def get_signal(pair: str, interval: int) -> (bool, bool):
+def get_signal(pair: str, interval: str) -> (bool, bool):
     """
     Calculates current signal based several technical analysis indicators
     :param pair: pair in format ANT/BTC
+    :param interval: interval in string format (1m, 5m, 1h,...)
     :return: (Buy, Sell) A bool-tuple indicating buy/sell signal
     """
     ticker_hist = get_ticker_history(pair, interval)
@@ -112,7 +114,7 @@ def get_signal(pair: str, interval: int) -> (bool, bool):
 
     # Check if dataframe is out of date
     signal_date = arrow.get(latest['date'])
-    if signal_date < arrow.now() - timedelta(minutes=(interval + 5)):
+    if signal_date < arrow.now() - timedelta(minutes=(ticker_interval_to_minutes(interval) + 5)):
         logger.warning('Too old dataframe for pair %s', pair)
         return (False, False)  # return False ?
 

freqtrade/exchange/__init__.py

@@ -4,6 +4,7 @@ import logging
 import ccxt
 from random import randint
 from typing import List, Dict, Any, Optional
+from datetime import datetime
 
 import arrow
 from cachetools import cached, TTLCache
@@ -150,9 +151,24 @@ def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict:
 
 
 @cached(TTLCache(maxsize=100, ttl=30))
-def get_ticker_history(pair: str, tick_interval) -> List[Dict]:
+def get_ticker_history(pair: str, tick_interval: str) -> List[Dict]:
     # TODO: check if exchange supports fetch_ohlcv
-    return _API.fetch_ohlcv(pair, timeframe=tick_interval)
+    history = _API.fetch_ohlcv(pair, timeframe=tick_interval)
+    history_json = []
+    try:
+        for candlestick in history:
+            history_json.append({
+                'T': datetime.fromtimestamp(candlestick[0]/1000.0).strftime('%Y-%m-%dT%H:%M:%S.%f'),
+                'O': candlestick[1],
+                'H': candlestick[2],
+                'L': candlestick[3],
+                'C': candlestick[4],
+                'V': candlestick[5],
+            })
+        return history_json
+    except IndexError as e:
+        logger.warning('Empty ticker history. Msg %s', str(e))
+        return []
 
 
 def cancel_order(order_id: str) -> None:

freqtrade/optimize/__init__.py

@@ -108,7 +108,7 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: str) -> bool:
         try:
             download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
         except BaseException:
-            logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format(
+            logger.info('Failed to download the pair: "{pair}", Interval: {interval}'.format(
                 pair=pair,
                 interval=ticker_interval,
             ))
@@ -153,20 +153,11 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m')
         logger.debug("Current End: None")
 
     new_data = get_ticker_history(pair=pair, tick_interval=interval)
-    data_json = []
+
-    for candlestick in new_data:
+    logger.debug("New Start: {}".format(new_data[1]['T']))
-        data_json.append({
+    logger.debug("New End: {}".format(new_data[-1]['T']))
-            'T': candlestick[0],
-            'O': candlestick[1],
-            'H': candlestick[2],
-            'L': candlestick[3],
-            'C': candlestick[4],
-            'V': candlestick[5],
-        })
-    logger.debug("New Start: {}".format(data_json[1]['T']))
-    logger.debug("New End: {}".format(data_json[-1]['T']))
     data = sorted(data, key=lambda data_json: data_json['T'])
 
-    misc.file_dump_json(filename, data_json)
+    misc.file_dump_json(filename, new_data)
 
     return True

freqtrade/tests/testdata/download_backtest_data.py

@@ -4,7 +4,6 @@
 import sys
 import json
 import ccxt
-import datetime
 
 from freqtrade import exchange
 from freqtrade import misc
@@ -37,18 +36,5 @@ for pair in PAIRS:
         data = exchange.get_ticker_history(pair, tick_interval)
         pair_print = pair.replace('/', '_')
         filename = '{}-{}.json'.format(pair_print, tick_interval)
-
+        data = sorted(data, key=lambda d: d['T'])
-        data_json = []
+        misc.file_dump_json(filename, data)
-        for candlestick in data:
-            # Timestamp in unix milliseconds formatted
-            data_json.append({
-                'T': datetime.datetime.fromtimestamp(candlestick[0]/1000.0).strftime('%Y-%m-%dT%H:%M:%S.%f'),
-                'O': candlestick[1],
-                'H': candlestick[2],
-                'L': candlestick[3],
-                'C': candlestick[4],
-                'V': candlestick[5],
-            })
-        data_json = sorted(data_json, key=lambda d: d['T'])
-
-        misc.file_dump_json(filename, data_json)