Merge pull request #591 from gcarq/feature/remove-duplicate-ticks

Aggregate ticks in parse_ticker_dataframe

freqtrade/analyze.py

@@ -46,12 +46,20 @@ class Analyze(object):
         :return: DataFrame
         """
         columns = {'C': 'close', 'V': 'volume', 'O': 'open', 'H': 'high', 'L': 'low', 'T': 'date'}
-        frame = DataFrame(ticker) \
-            .rename(columns=columns)
+        frame = DataFrame(ticker).rename(columns=columns)
         if 'BV' in frame:
-            frame.drop('BV', 1, inplace=True)
+            frame.drop('BV', axis=1, inplace=True)
+
         frame['date'] = to_datetime(frame['date'], utc=True, infer_datetime_format=True)
-        frame.sort_values('date', inplace=True)
+
+        # group by index and aggregate results to eliminate duplicate ticks
+        frame = frame.groupby(by='date', as_index=False, sort=True).agg({
+            'close': 'last',
+            'high': 'max',
+            'low': 'min',
+            'open': 'first',
+            'volume': 'max',
+        })
         return frame
 
     def populate_indicators(self, dataframe: DataFrame) -> DataFrame:

freqtrade/optimize/__init__.py

@@ -111,45 +111,38 @@ def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool:
 
 
 # FIX: 20180110, suggest rename interval to tick_interval
-def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool:
+def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> None:
     """
     Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
     Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
-    :param pairs: list of pairs to download
-    :return: bool
     """
 
     path = make_testdata_path(datadir)
     logger.info(
-        'Download the pair: "%s", Interval: %s min',
-        pair,
-        interval
+        'Download the pair: "%s", Interval: %s min', pair, interval
     )
 
-    filepair = pair.replace("-", "_")
     filename = os.path.join(path, '{pair}-{interval}.json'.format(
-        pair=filepair,
+        pair=pair.replace("-", "_"),
         interval=interval,
     ))
 
     if os.path.isfile(filename):
         with open(filename, "rt") as file:
             data = json.load(file)
-        logger.debug("Current Start: %s", data[1]['T'])
-        logger.debug("Current End: %s", data[-1:][0]['T'])
     else:
         data = []
-        logger.debug("Current Start: None")
-        logger.debug("Current End: None")
 
-    new_data = get_ticker_history(pair=pair, tick_interval=int(interval))
-    for row in new_data:
-        if row not in data:
-            data.append(row)
-    logger.debug("New Start: %s", data[1]['T'])
-    logger.debug("New End: %s", data[-1:][0]['T'])
-    data = sorted(data, key=lambda data: data['T'])
+    logger.debug('Current Start: %s', data[1]['T'] if data else None)
+    logger.debug('Current End: %s', data[-1:][0]['T'] if data else None)
 
+    # Extend data with new ticker history
+    data.extend([
+        row for row in get_ticker_history(pair=pair, tick_interval=int(interval))
+        if row not in data
+    ])
+
+    data = sorted(data, key=lambda _data: _data['T'])
+    logger.debug('New Start: %s', data[1]['T'])
+    logger.debug('New End: %s', data[-1:][0]['T'])
     misc.file_dump_json(filename, data)
-
-    return True

freqtrade/optimize/backtesting.py

@@ -4,11 +4,12 @@
 This module contains the backtesting logic
 """
 import logging
+import operator
 from argparse import Namespace
 from typing import Dict, Tuple, Any, List, Optional
 
 import arrow
-from pandas import DataFrame, Series
+from pandas import DataFrame
 from tabulate import tabulate
 
 import freqtrade.optimize as optimize
@@ -60,11 +61,12 @@ class Backtesting(object):
         :param data: dictionary with preprocessed backtesting data
         :return: tuple containing min_date, max_date
         """
-        all_dates = Series([])
-        for pair_data in data.values():
-            all_dates = all_dates.append(pair_data['date'])
-        all_dates.sort_values(inplace=True)
-        return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1])
+        timeframe = [
+            (arrow.get(min(frame.date)), arrow.get(max(frame.date)))
+            for frame in data.values()
+        ]
+        return min(timeframe, key=operator.itemgetter(0))[0], \
+            max(timeframe, key=operator.itemgetter(1))[1]
 
     def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str:
         """

freqtrade/tests/optimize/test_optimize.py

@@ -182,10 +182,11 @@ def test_download_backtesting_testdata(ticker_history, mocker) -> None:
 
 def test_download_backtesting_testdata2(mocker) -> None:
     tick = [{'T': 'bar'}, {'T': 'foo'}]
-    mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
+    json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None)
     mocker.patch('freqtrade.optimize.__init__.get_ticker_history', return_value=tick)
-    assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1)
-    assert download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3)
+    download_backtesting_testdata(None, pair="BTC-UNITEST", interval=1)
+    download_backtesting_testdata(None, pair="BTC-UNITEST", interval=3)
+    assert json_dump_mock.call_count == 2
 
 
 def test_load_tickerdata_file() -> None:

freqtrade/tests/test_analyze.py

@@ -50,7 +50,7 @@ def test_dataframe_correct_length(result):
 
 def test_dataframe_correct_columns(result):
     assert result.columns.tolist() == \
-        ['close', 'high', 'low', 'open', 'date', 'volume']
+        ['date', 'close', 'high', 'low', 'open', 'volume']
 
 
 def test_populates_buy_trend(result):
@@ -170,7 +170,7 @@ def test_get_signal_handles_exceptions(mocker):
 
 
 def test_parse_ticker_dataframe(ticker_history, ticker_history_without_bv):
-    columns = ['close', 'high', 'low', 'open', 'date', 'volume']
+    columns = ['date', 'close', 'high', 'low', 'open', 'volume']
 
     # Test file with BV data
     dataframe = Analyze.parse_ticker_dataframe(ticker_history)