adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove trade duplicates in datahandler

Browse Source
This commit is contained in:
Matthias 2020-04-01 07:58:39 +02:00
parent 0506caf986
commit ff9caf790b
3 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
freqtrade/data/history/history_utils.py
View File

@ -9,7 +9,9 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import ohlcv_to_dataframe, trades_to_ohlcv from freqtrade.data.converter import (ohlcv_to_dataframe,
trades_remove_duplicates,
trades_to_ohlcv)
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
@ -282,6 +284,8 @@ def _download_trades_history(exchange: Exchange,
from_id=from_id, from_id=from_id,
) )
trades.extend(new_trades[1]) trades.extend(new_trades[1])
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_remove_duplicates(trades)
data_handler.trades_store(pair, data=trades) data_handler.trades_store(pair, data=trades)
logger.debug("New Start: %s", trades[0][0]) logger.debug("New Start: %s", trades[0][0])

15
freqtrade/data/history/idatahandler.py
View File

@ -13,7 +13,8 @@ from typing import List, Optional, Type
from pandas import DataFrame from pandas import DataFrame
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe from freqtrade.data.converter import (clean_ohlcv_dataframe,
trades_remove_duplicates, trim_dataframe)
from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange import timeframe_to_seconds
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -110,7 +111,7 @@ class IDataHandler(ABC):
""" """
@abstractmethod @abstractmethod
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
""" """
Load a pair from file, either .json.gz or .json Load a pair from file, either .json.gz or .json
:param pair: Load trades for this pair :param pair: Load trades for this pair
@ -126,6 +127,16 @@ class IDataHandler(ABC):
:return: True when deleted, false if file did not exist. :return: True when deleted, false if file did not exist.
""" """
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
"""
Load a pair from file, either .json.gz or .json
Removes duplicates in the process.
:param pair: Load trades for this pair
:param timerange: Timerange to load trades for - currently not implemented
:return: List of trades
"""
return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
def ohlcv_load(self, pair, timeframe: str, def ohlcv_load(self, pair, timeframe: str,
timerange: Optional[TimeRange] = None, timerange: Optional[TimeRange] = None,
fill_missing: bool = True, fill_missing: bool = True,

2
freqtrade/data/history/jsondatahandler.py
View File

@ -136,7 +136,7 @@ class JsonDataHandler(IDataHandler):
""" """
raise NotImplementedError() raise NotImplementedError()
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
""" """
Load a pair from file, either .json.gz or .json Load a pair from file, either .json.gz or .json
# TODO: respect timerange ... # TODO: respect timerange ...