Merge pull request #6848 from freqtrade/datahandler_case

Datahandler case insensitive
This commit is contained in:
Matthias 2022-05-17 09:34:11 +02:00 committed by GitHub
commit 8d46e16c46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 45 additions and 19 deletions

View File

@ -40,7 +40,7 @@ class HDF5DataHandler(IDataHandler):
return [ return [
( (
cls.rebuild_pair_from_filename(match[1]), cls.rebuild_pair_from_filename(match[1]),
match[2], cls.rebuild_timeframe_from_filename(match[2]),
CandleType.from_string(match[3]) CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1] ) for match in _tmp if match and len(match.groups()) > 1]
@ -109,7 +109,11 @@ class HDF5DataHandler(IDataHandler):
) )
if not filename.exists(): if not filename.exists():
return pd.DataFrame(columns=self._columns) # Fallback mode for 1M files
filename = self._pair_data_filename(
self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
if not filename.exists():
return pd.DataFrame(columns=self._columns)
where = [] where = []
if timerange: if timerange:
if timerange.starttype == 'date': if timerange.starttype == 'date':

View File

@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
class IDataHandler(ABC): class IDataHandler(ABC):
_OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)'
def __init__(self, datadir: Path) -> None: def __init__(self, datadir: Path) -> None:
self._datadir = datadir self._datadir = datadir
@ -193,10 +193,14 @@ class IDataHandler(ABC):
datadir: Path, datadir: Path,
pair: str, pair: str,
timeframe: str, timeframe: str,
candle_type: CandleType candle_type: CandleType,
no_timeframe_modify: bool = False
) -> Path: ) -> Path:
pair_s = misc.pair_to_filename(pair) pair_s = misc.pair_to_filename(pair)
candle = "" candle = ""
if not no_timeframe_modify:
timeframe = cls.timeframe_to_file(timeframe)
if candle_type != CandleType.SPOT: if candle_type != CandleType.SPOT:
datadir = datadir.joinpath('futures') datadir = datadir.joinpath('futures')
candle = f"-{candle_type}" candle = f"-{candle_type}"
@ -210,6 +214,18 @@ class IDataHandler(ABC):
filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
return filename return filename
@staticmethod
def timeframe_to_file(timeframe: str):
return timeframe.replace('M', 'Mo')
@staticmethod
def rebuild_timeframe_from_filename(timeframe: str) -> str:
"""
converts timeframe from disk to file
Replaces mo with M (to avoid problems on case-insensitive filesystems)
"""
return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE)
@staticmethod @staticmethod
def rebuild_pair_from_filename(pair: str) -> str: def rebuild_pair_from_filename(pair: str) -> str:
""" """

View File

@ -41,7 +41,7 @@ class JsonDataHandler(IDataHandler):
return [ return [
( (
cls.rebuild_pair_from_filename(match[1]), cls.rebuild_pair_from_filename(match[1]),
match[2], cls.rebuild_timeframe_from_filename(match[2]),
CandleType.from_string(match[3]) CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1] ) for match in _tmp if match and len(match.groups()) > 1]
@ -103,9 +103,14 @@ class JsonDataHandler(IDataHandler):
:param candle_type: Any of the enum CandleType (must match trading mode!) :param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame :return: DataFrame with ohlcv data, or empty DataFrame
""" """
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) filename = self._pair_data_filename(
self._datadir, pair, timeframe, candle_type=candle_type)
if not filename.exists(): if not filename.exists():
return DataFrame(columns=self._columns) # Fallback mode for 1M files
filename = self._pair_data_filename(
self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
if not filename.exists():
return DataFrame(columns=self._columns)
try: try:
pairdata = read_json(filename, orient='values') pairdata = read_json(filename, orient='values')
pairdata.columns = self._columns pairdata.columns = self._columns

View File

@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None:
assert str(Path('tests') / 'testdata') in str(testdatadir) assert str(Path('tests') / 'testdata') in str(testdatadir)
@pytest.mark.parametrize("pair,expected_result,candle_type", [ @pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [
("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""), ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""),
("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""), ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""),
("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""), ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""),
(".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""),
("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""),
("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""),
("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"), ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""),
("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"), ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"),
("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"),
]) ])
def test_json_pair_data_filename(pair, expected_result, candle_type): def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
fn = JsonDataHandler._pair_data_filename( fn = JsonDataHandler._pair_data_filename(
Path('freqtrade/hello/world'), Path('freqtrade/hello/world'),
pair, pair,
'5m', timeframe,
CandleType.from_string(candle_type) CandleType.from_string(candle_type)
) )
assert isinstance(fn, Path) assert isinstance(fn, Path)
@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type):
fn = JsonGzDataHandler._pair_data_filename( fn = JsonGzDataHandler._pair_data_filename(
Path('freqtrade/hello/world'), Path('freqtrade/hello/world'),
pair, pair,
'5m', timeframe,
candle_type=CandleType.from_string(candle_type) candle_type=CandleType.from_string(candle_type)
) )
assert isinstance(fn, Path) assert isinstance(fn, Path)