From ff9caf790be4118f18ebf2f9c8779cf717ae507d Mon Sep 17 00:00:00 2001
From: Matthias <xmatthias@outlook.com>
Date: Wed, 1 Apr 2020 07:58:39 +0200
Subject: [PATCH] remove trade duplicates in datahandler

---
 freqtrade/data/history/history_utils.py   |  6 +++++-
 freqtrade/data/history/idatahandler.py    | 15 +++++++++++++--
 freqtrade/data/history/jsondatahandler.py |  2 +-
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py
index a7490f17b..74c8e8b96 100644
--- a/freqtrade/data/history/history_utils.py
+++ b/freqtrade/data/history/history_utils.py
@@ -9,7 +9,9 @@ from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
-from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv
+from freqtrade.data.converter import (ohlcv_to_dataframe,
+                                      trades_remove_duplicates,
+                                      trades_to_ohlcv)
 from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import Exchange
@@ -282,6 +284,8 @@ def _download_trades_history(exchange: Exchange,
                                                   from_id=from_id,
                                                   )
         trades.extend(new_trades[1])
+        # Remove duplicates to make sure we're not storing data we don't need
+        trades = trades_remove_duplicates(trades)
         data_handler.trades_store(pair, data=trades)
 
         logger.debug("New Start: %s", trades[0][0])
diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py
index 4663c29ac..d5d7c16db 100644
--- a/freqtrade/data/history/idatahandler.py
+++ b/freqtrade/data/history/idatahandler.py
@@ -13,7 +13,8 @@ from typing import List, Optional, Type
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
+from freqtrade.data.converter import (clean_ohlcv_dataframe,
+                                      trades_remove_duplicates, trim_dataframe)
 from freqtrade.exchange import timeframe_to_seconds
 
 logger = logging.getLogger(__name__)
@@ -110,7 +111,7 @@ class IDataHandler(ABC):
         """
 
     @abstractmethod
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
         """
         Load a pair from file, either .json.gz or .json
         :param pair: Load trades for this pair
@@ -126,6 +127,16 @@ class IDataHandler(ABC):
         :return: True when deleted, false if file did not exist.
         """
 
+    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+        """
+        Load a pair from file, either .json.gz or .json
+        Removes duplicates in the process.
+        :param pair: Load trades for this pair
+        :param timerange: Timerange to load trades for - currently not implemented
+        :return: List of trades
+        """
+        return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
+
     def ohlcv_load(self, pair, timeframe: str,
                    timerange: Optional[TimeRange] = None,
                    fill_missing: bool = True,
diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py
index 50a32603b..01320f129 100644
--- a/freqtrade/data/history/jsondatahandler.py
+++ b/freqtrade/data/history/jsondatahandler.py
@@ -136,7 +136,7 @@ class JsonDataHandler(IDataHandler):
         """
         raise NotImplementedError()
 
-    def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
+    def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
         """
         Load a pair from file, either .json.gz or .json
         # TODO: respect timerange ...