Merge pull request #85 from gcarq/datetime_fixes

Performance improvements for backtesting

freqtrade/analyze.py

@@ -4,7 +4,7 @@ from datetime import timedelta
 
 import arrow
 import talib.abstract as ta
-from pandas import DataFrame
+from pandas import DataFrame, to_datetime
 from qtpylib.indicators import awesome_oscillator, crossed_above
 
 from freqtrade import exchange
@@ -15,7 +15,7 @@ logging.basicConfig(level=logging.DEBUG,
 logger = logging.getLogger(__name__)
 
 
-def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame:
+def parse_ticker_dataframe(ticker: list) -> DataFrame:
     """
     Analyses the trend for the given pair
     :param pair: pair as str in format BTC_ETH or BTC-ETH
@@ -23,8 +23,9 @@ def parse_ticker_dataframe(ticker: list, minimum_date: arrow.Arrow) -> DataFrame
     """
     df = DataFrame(ticker) \
         .drop('BV', 1) \
-        .rename(columns={'C':'close', 'V':'volume', 'O':'open', 'H':'high', 'L':'low', 'T':'date'}) \
-        .sort_values('date')
+        .rename(columns={'C':'close', 'V':'volume', 'O':'open', 'H':'high', 'L':'low', 'T':'date'})
+    df['date'] = to_datetime(df['date'], utc=True, infer_datetime_format=True)
+    df.sort_values('date', inplace=True)
     return df
 
 
@@ -53,14 +54,14 @@ def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
     :param dataframe: DataFrame
     :return: DataFrame with buy column
     """
-    dataframe.loc[
+    dataframe.ix[
         (dataframe['close'] < dataframe['sma']) &
         (dataframe['tema'] <= dataframe['blower']) &
         (dataframe['mfi'] < 25) &
         (dataframe['fastd'] < 25) &
         (dataframe['adx'] > 30),
         'buy'] = 1
-    dataframe.loc[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
+    dataframe.ix[dataframe['buy'] == 1, 'buy_price'] = dataframe['close']
 
     return dataframe
 
@@ -73,7 +74,7 @@ def analyze_ticker(pair: str) -> DataFrame:
     """
     minimum_date = arrow.utcnow().shift(hours=-24)
     data = get_ticker_history(pair, minimum_date)
-    dataframe = parse_ticker_dataframe(data['result'], minimum_date)
+    dataframe = parse_ticker_dataframe(data['result'])
 
     if dataframe.empty:
         logger.warning('Empty dataframe for pair %s', pair)

freqtrade/tests/test_analyze.py

@@ -1,7 +1,7 @@
 # pragma pylint: disable=missing-docstring
+from datetime import datetime
 import json
 import pytest
-import arrow
 from pandas import DataFrame
 
 from freqtrade.analyze import parse_ticker_dataframe, populate_buy_trend, populate_indicators, \
@@ -12,7 +12,7 @@ def result():
     with open('freqtrade/tests/testdata/btc-eth.json') as data_file:
         data = json.load(data_file)
 
-    return parse_ticker_dataframe(data['result'], arrow.get('2017-08-30T10:00:00'))
+    return parse_ticker_dataframe(data['result'])
 
 def test_dataframe_has_correct_columns(result):
     assert result.columns.tolist() == \
@@ -27,10 +27,10 @@ def test_populates_buy_trend(result):
     assert 'buy_price' in dataframe.columns
 
 def test_returns_latest_buy_signal(mocker):
-    buydf = DataFrame([{'buy': 1, 'date': arrow.utcnow()}])
+    buydf = DataFrame([{'buy': 1, 'date': datetime.today()}])
     mocker.patch('freqtrade.analyze.analyze_ticker', return_value=buydf)
     assert get_buy_signal('BTC-ETH')
 
-    buydf = DataFrame([{'buy': 0, 'date': arrow.utcnow()}])
+    buydf = DataFrame([{'buy': 0, 'date': datetime.today()}])
     mocker.patch('freqtrade.analyze.analyze_ticker', return_value=buydf)
     assert not get_buy_signal('BTC-ETH')

freqtrade/tests/test_backtesting.py

@@ -51,20 +51,16 @@ def backtest(conf, pairs, mocker):
 
             mocker.patch('freqtrade.analyze.get_ticker_history', return_value=data)
             mocker.patch('arrow.utcnow', return_value=arrow.get('2017-08-20T14:50:00'))
-            ticker = analyze_ticker(pair)
+            ticker = analyze_ticker(pair)[['close', 'date', 'buy']].copy()
             # for each buy point
-            for index, row in ticker[ticker.buy == 1].iterrows():
-                trade = Trade(
-                    open_rate=row['close'],
-                    open_date=arrow.get(row['date']).datetime,
-                    amount=1,
-                )
+            for row in ticker[ticker.buy == 1].itertuples(index=True):
+                trade = Trade(open_rate=row.close, open_date=row.date, amount=1)
                 # calculate win/lose forwards from buy point
-                for index2, row2 in ticker[index:].iterrows():
-                    if should_sell(trade, row2['close'], arrow.get(row2['date']).datetime):
-                        current_profit = (row2['close'] - trade.open_rate) / trade.open_rate
+                for row2 in ticker[row.Index:].itertuples(index=True):
+                    if should_sell(trade, row2.close, row2.date):
+                        current_profit = (row2.close - trade.open_rate) / trade.open_rate
 
-                        trades.append((pair, current_profit, index2 - index))
+                        trades.append((pair, current_profit, row2.Index - row.Index))
                         break
     labels = ['currency', 'profit', 'duration']
     results = DataFrame.from_records(trades, columns=labels)