From f9338be455dd0dd0c7980a4aecc81160200fca5d Mon Sep 17 00:00:00 2001 From: enenn Date: Sun, 4 Feb 2018 19:43:05 +0100 Subject: [PATCH] Parse ccxt ohlcv history into dict in get_ticker_history --- freqtrade/analyze.py | 8 +++++--- freqtrade/exchange/__init__.py | 20 +++++++++++++++++-- freqtrade/optimize/__init__.py | 19 +++++------------- .../tests/testdata/download_backtest_data.py | 18 ++--------------- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/freqtrade/analyze.py b/freqtrade/analyze.py index 0be5245f8..9e7d95218 100644 --- a/freqtrade/analyze.py +++ b/freqtrade/analyze.py @@ -9,6 +9,7 @@ from typing import Dict, List import arrow from pandas import DataFrame, to_datetime +from freqtrade.misc import ticker_interval_to_minutes from freqtrade.exchange import get_ticker_history from freqtrade.strategy.strategy import Strategy @@ -32,7 +33,7 @@ def parse_ticker_dataframe(ticker: list) -> DataFrame: .rename(columns=columns) if 'BV' in frame: frame.drop('BV', 1, inplace=True) - frame['date'] = to_datetime(frame['date'], unit='ms') + frame['date'] = to_datetime(frame['date'], format='%Y-%m-%dT%H:%M:%S.%f') frame.sort_values('date', inplace=True) return frame @@ -84,10 +85,11 @@ def analyze_ticker(ticker_history: List[Dict]) -> DataFrame: # FIX: Maybe return False, if an error has occured, # Otherwise we might mask an error as an non-signal-scenario -def get_signal(pair: str, interval: int) -> (bool, bool): +def get_signal(pair: str, interval: str) -> (bool, bool): """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC + :param interval: interval in string format (1m, 5m, 1h,...) :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ ticker_hist = get_ticker_history(pair, interval) @@ -112,7 +114,7 @@ def get_signal(pair: str, interval: int) -> (bool, bool): # Check if dataframe is out of date signal_date = arrow.get(latest['date']) - if signal_date < arrow.now() - timedelta(minutes=(interval + 5)): + if signal_date < arrow.now() - timedelta(minutes=(ticker_interval_to_minutes(interval) + 5)): logger.warning('Too old dataframe for pair %s', pair) return (False, False) # return False ? diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 694d8bab3..d47199fb2 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -4,6 +4,7 @@ import logging import ccxt from random import randint from typing import List, Dict, Any, Optional +from datetime import datetime import arrow from cachetools import cached, TTLCache @@ -150,9 +151,24 @@ def get_ticker(pair: str, refresh: Optional[bool] = True) -> dict: @cached(TTLCache(maxsize=100, ttl=30)) -def get_ticker_history(pair: str, tick_interval) -> List[Dict]: +def get_ticker_history(pair: str, tick_interval: str) -> List[Dict]: # TODO: check if exchange supports fetch_ohlcv - return _API.fetch_ohlcv(pair, timeframe=tick_interval) + history = _API.fetch_ohlcv(pair, timeframe=tick_interval) + history_json = [] + try: + for candlestick in history: + history_json.append({ + 'T': datetime.fromtimestamp(candlestick[0]/1000.0).strftime('%Y-%m-%dT%H:%M:%S.%f'), + 'O': candlestick[1], + 'H': candlestick[2], + 'L': candlestick[3], + 'C': candlestick[4], + 'V': candlestick[5], + }) + return history_json + except IndexError as e: + logger.warning('Empty ticker history. Msg %s', str(e)) + return [] def cancel_order(order_id: str) -> None: diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index ea4ab7976..1a299eb30 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -108,7 +108,7 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: str) -> bool: try: download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval) except BaseException: - logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format( + logger.info('Failed to download the pair: "{pair}", Interval: {interval}'.format( pair=pair, interval=ticker_interval, )) @@ -153,20 +153,11 @@ def download_backtesting_testdata(datadir: str, pair: str, interval: str = '5m') logger.debug("Current End: None") new_data = get_ticker_history(pair=pair, tick_interval=interval) - data_json = [] - for candlestick in new_data: - data_json.append({ - 'T': candlestick[0], - 'O': candlestick[1], - 'H': candlestick[2], - 'L': candlestick[3], - 'C': candlestick[4], - 'V': candlestick[5], - }) - logger.debug("New Start: {}".format(data_json[1]['T'])) - logger.debug("New End: {}".format(data_json[-1]['T'])) + + logger.debug("New Start: {}".format(new_data[1]['T'])) + logger.debug("New End: {}".format(new_data[-1]['T'])) data = sorted(data, key=lambda data_json: data_json['T']) - misc.file_dump_json(filename, data_json) + misc.file_dump_json(filename, new_data) return True diff --git a/freqtrade/tests/testdata/download_backtest_data.py b/freqtrade/tests/testdata/download_backtest_data.py index 02dba7628..9e1016c3e 100755 --- a/freqtrade/tests/testdata/download_backtest_data.py +++ b/freqtrade/tests/testdata/download_backtest_data.py @@ -4,7 +4,6 @@ import sys import json import ccxt -import datetime from freqtrade import exchange from freqtrade import misc @@ -37,18 +36,5 @@ for pair in PAIRS: data = exchange.get_ticker_history(pair, tick_interval) pair_print = pair.replace('/', '_') filename = '{}-{}.json'.format(pair_print, tick_interval) - - data_json = [] - for candlestick in data: - # Timestamp in unix milliseconds formatted - data_json.append({ - 'T': datetime.datetime.fromtimestamp(candlestick[0]/1000.0).strftime('%Y-%m-%dT%H:%M:%S.%f'), - 'O': candlestick[1], - 'H': candlestick[2], - 'L': candlestick[3], - 'C': candlestick[4], - 'V': candlestick[5], - }) - data_json = sorted(data_json, key=lambda d: d['T']) - - misc.file_dump_json(filename, data_json) + data = sorted(data, key=lambda d: d['T']) + misc.file_dump_json(filename, data)