diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index a7490f17b..74c8e8b96 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -9,7 +9,9 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS -from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv +from freqtrade.data.converter import (ohlcv_to_dataframe, + trades_remove_duplicates, + trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange @@ -282,6 +284,8 @@ def _download_trades_history(exchange: Exchange, from_id=from_id, ) trades.extend(new_trades[1]) + # Remove duplicates to make sure we're not storing data we don't need + trades = trades_remove_duplicates(trades) data_handler.trades_store(pair, data=trades) logger.debug("New Start: %s", trades[0][0]) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 4663c29ac..d5d7c16db 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -13,7 +13,8 @@ from typing import List, Optional, Type from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe +from freqtrade.data.converter import (clean_ohlcv_dataframe, + trades_remove_duplicates, trim_dataframe) from freqtrade.exchange import timeframe_to_seconds logger = logging.getLogger(__name__) @@ -110,7 +111,7 @@ class IDataHandler(ABC): """ @abstractmethod - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ Load a pair from file, either .json.gz or .json :param pair: Load trades for this pair @@ -126,6 +127,16 @@ class IDataHandler(ABC): :return: True when deleted, false if file did not exist. """ + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + Removes duplicates in the process. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 50a32603b..01320f129 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -136,7 +136,7 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ...