adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move dataframe validation to abstract class

Browse Source
This commit is contained in:
Matthias 2019-12-26 09:56:42 +01:00
parent e861f05b75
commit 37c5b68987
2 changed files with 19 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
freqtrade/data/datahandlers/idatahandler.py
View File

@ -10,6 +10,7 @@ from pathlib import Path
from typing import Dict, List, Optional from typing import Dict, List, Optional
from datetime import datetime, timezone from datetime import datetime, timezone
from pandas import DataFrame from pandas import DataFrame
from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange import timeframe_to_seconds
@ -47,15 +48,26 @@ class IDataHandler(ABC):
timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
pairdf = self._ohlcv_load(pair, timeframe, pairdf = self._ohlcv_load(pair, timeframe,
timerange=timerange_startup, timerange=timerange_startup)
fill_missing=fill_missing,
drop_incomplete=drop_incomplete)
if pairdf.empty: if pairdf.empty:
logger.warning( logger.warning(
f'No history data for pair: "{pair}", timeframe: {timeframe}. ' f'No history data for pair: "{pair}", timeframe: {timeframe}. '
'Use `freqtrade download-data` to download the data' 'Use `freqtrade download-data` to download the data'
) )
return pairdf return pairdf
else:
enddate = pairdf.iloc[-1]['date']
if timerange_startup:
self._validate_pairdata(pair, pairdf, timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
# incomplete candles should only be dropped if we didn't trim the end beforehand.
return clean_ohlcv_dataframe(pairdf, timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete and
enddate == pairdf.iloc[-1]['date']))
def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange):
""" """

23
freqtrade/data/datahandlers/jsondatahandler.py
View File

@ -7,7 +7,6 @@ from pandas import DataFrame, read_json, to_datetime
from freqtrade import misc from freqtrade import misc
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.converter import clean_ohlcv_dataframe, trim_dataframe
from .idatahandler import IDataHandler from .idatahandler import IDataHandler
@ -54,18 +53,14 @@ class JsonDataHandler(IDataHandler):
def _ohlcv_load(self, pair: str, timeframe: str, def _ohlcv_load(self, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None, timerange: Optional[TimeRange] = None,
fill_missing: bool = True,
drop_incomplete: bool = True,
) -> DataFrame: ) -> DataFrame:
""" """
Internal method used to load data for one pair from disk. Internal method used to load data for one pair from disk.
Implements the loading and conversation to a Pandas dataframe. Implements the loading and conversation to a Pandas dataframe.
Timerange trimming and dataframe validation happens outside of this method.
:param pair: Pair to load data :param pair: Pair to load data
:param timeframe: Ticker timeframe (e.g. "5m") :param timeframe: Ticker timeframe (e.g. "5m")
:param timerange: Limit data to be loaded to this timerange :param timerange: Limit data to be loaded to this timerange.
:param fill_missing: Fill missing values with "No action"-candles
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:return: DataFrame with ohlcv data, or empty DataFrame :return: DataFrame with ohlcv data, or empty DataFrame
""" """
filename = self._pair_data_filename(self._datadir, pair, timeframe) filename = self._pair_data_filename(self._datadir, pair, timeframe)
@ -77,19 +72,7 @@ class JsonDataHandler(IDataHandler):
unit='ms', unit='ms',
utc=True, utc=True,
infer_datetime_format=True) infer_datetime_format=True)
return pairdata
enddate = pairdata.iloc[-1]['date']
if timerange:
self._validate_pairdata(pair, pairdata, timerange)
pairdata = trim_dataframe(pairdata, timerange)
# incomplete candles should only be dropped if we didn't trim the end beforehand.
return clean_ohlcv_dataframe(pairdata, timeframe,
pair=pair,
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete and
enddate == pairdata.iloc[-1]['date']))
def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None:
""" """