From a1048fb619e186d83ff83ad25ba0e02e818d6fd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 17:00:00 +0200 Subject: [PATCH 1/3] Store monthly candles as "Mo" --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/idatahandler.py | 16 ++++++++++++++-- freqtrade/data/history/jsondatahandler.py | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 23120a4ba..165685960 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -40,7 +40,7 @@ class HDF5DataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 2e6b070ca..bd795f480 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -201,7 +201,7 @@ class IDataHandler(ABC): datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') + f'{pair_s}-{cls.timeframe_to_file(timeframe)}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -210,6 +210,18 @@ class IDataHandler(ABC): filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename + @staticmethod + def timeframe_to_file(timeframe: str): + return timeframe.replace('M', 'Mo') + + @staticmethod + def rebuild_timeframe_from_filename(timeframe: str) -> str: + """ + converts timeframe from disk to file + Replaces mo with M (to avoid problems on case-insensitive filesystems) + """ + return re.sub('mo', 'M', timeframe, flags=re.IGNORECASE) + @staticmethod def rebuild_pair_from_filename(pair: str) -> str: """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 23054ac51..fa02c770b 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -41,7 +41,7 @@ class JsonDataHandler(IDataHandler): return [ ( cls.rebuild_pair_from_filename(match[1]), - match[2], + cls.rebuild_timeframe_from_filename(match[2]), CandleType.from_string(match[3]) ) for match in _tmp if match and len(match.groups()) > 1] From 2e65a1793d086ecbdf904d74a0dc3dc3b4ddeac1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 1 May 2022 19:51:25 +0200 Subject: [PATCH 2/3] Add fallback to load 1M files as well as 1Mo files --- freqtrade/data/history/hdf5datahandler.py | 11 +++++++--- freqtrade/data/history/idatahandler.py | 7 ++++--- freqtrade/data/history/jsondatahandler.py | 12 ++++++++--- tests/data/test_history.py | 25 ++++++++++++----------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 165685960..6099c22bc 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -77,7 +77,8 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( @@ -104,12 +105,16 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename( self._datadir, pair, - timeframe, + self.timeframe_to_file(timeframe), candle_type=candle_type ) if not filename.exists(): - return pd.DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) + if not filename.exists(): + return pd.DataFrame(columns=self._columns) where = [] if timerange: if timerange.starttype == 'date': diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index bd795f480..69d6212ee 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -98,7 +98,8 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) if filename.exists(): filename.unlink() return True @@ -201,7 +202,7 @@ class IDataHandler(ABC): datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{cls.timeframe_to_file(timeframe)}{candle}.{cls._get_file_extension()}') + f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -220,7 +221,7 @@ class IDataHandler(ABC): converts timeframe from disk to file Replaces mo with M (to avoid problems on case-insensitive filesystems) """ - return re.sub('mo', 'M', timeframe, flags=re.IGNORECASE) + return re.sub('1mo', '1M', timeframe, flags=re.IGNORECASE) @staticmethod def rebuild_pair_from_filename(pair: str) -> str: diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index fa02c770b..38402a113 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -77,7 +77,8 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int @@ -103,9 +104,14 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) + filename = self._pair_data_filename( + self._datadir, pair, self.timeframe_to_file(timeframe), candle_type=candle_type) if not filename.exists(): - return DataFrame(columns=self._columns) + # Fallback mode for 1M files + filename = self._pair_data_filename( + self._datadir, pair, timeframe, candle_type=candle_type) + if not filename.exists(): + return DataFrame(columns=self._columns) try: pairdata = read_json(filename, orient='values') pairdata.columns = self._columns diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 82d4a841c..1e7d8855e 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -158,21 +158,22 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -@pytest.mark.parametrize("pair,expected_result,candle_type", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""), - ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), - ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), - ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"), - ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"), +@pytest.mark.parametrize("pair,timeframe,expected_result,candle_type", [ + ("ETH/BTC", "5m", "freqtrade/hello/world/ETH_BTC-5m.json", ""), + ("ETH/USDT", "1M", "freqtrade/hello/world/ETH_USDT-1Mo.json", ""), + ("Fabric Token/ETH", "5m", "freqtrade/hello/world/Fabric_Token_ETH-5m.json", ""), + ("ETHH20", "5m", "freqtrade/hello/world/ETHH20-5m.json", ""), + (".XBTBON2H", "5m", "freqtrade/hello/world/_XBTBON2H-5m.json", ""), + ("ETHUSD.d", "5m", "freqtrade/hello/world/ETHUSD_d-5m.json", ""), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/ACC_OLD_BTC-5m.json", ""), + ("ETH/BTC", "5m", "freqtrade/hello/world/futures/ETH_BTC-5m-mark.json", "mark"), + ("ACC_OLD/BTC", "5m", "freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json", "index"), ]) -def test_json_pair_data_filename(pair, expected_result, candle_type): +def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + JsonDataHandler.timeframe_to_file(timeframe), CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -180,7 +181,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - '5m', + JsonGzDataHandler.timeframe_to_file(timeframe), candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path) From 76637d3939994219e1ec15e1cdf7e513217536a5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 May 2022 19:53:01 +0200 Subject: [PATCH 3/3] Simplify timeframe-transition --- freqtrade/data/history/hdf5datahandler.py | 7 +++---- freqtrade/data/history/idatahandler.py | 9 ++++++--- freqtrade/data/history/jsondatahandler.py | 7 +++---- tests/data/test_history.py | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 6099c22bc..dadc9c7e6 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -77,8 +77,7 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) self.create_dir_if_needed(filename) _data.loc[:, self._columns].to_hdf( @@ -105,14 +104,14 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename( self._datadir, pair, - self.timeframe_to_file(timeframe), + timeframe, candle_type=candle_type ) if not filename.exists(): # Fallback mode for 1M files filename = self._pair_data_filename( - self._datadir, pair, timeframe, candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) if not filename.exists(): return pd.DataFrame(columns=self._columns) where = [] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 69d6212ee..07dc7c763 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -98,8 +98,7 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) if filename.exists(): filename.unlink() return True @@ -194,10 +193,14 @@ class IDataHandler(ABC): datadir: Path, pair: str, timeframe: str, - candle_type: CandleType + candle_type: CandleType, + no_timeframe_modify: bool = False ) -> Path: pair_s = misc.pair_to_filename(pair) candle = "" + if not no_timeframe_modify: + timeframe = cls.timeframe_to_file(timeframe) + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 38402a113..83ec183df 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -77,8 +77,7 @@ class JsonDataHandler(IDataHandler): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ - filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int @@ -105,11 +104,11 @@ class JsonDataHandler(IDataHandler): :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename( - self._datadir, pair, self.timeframe_to_file(timeframe), candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type) if not filename.exists(): # Fallback mode for 1M files filename = self._pair_data_filename( - self._datadir, pair, timeframe, candle_type=candle_type) + self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True) if not filename.exists(): return DataFrame(columns=self._columns) try: diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 1e7d8855e..9709e7ad0 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -173,7 +173,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - JsonDataHandler.timeframe_to_file(timeframe), + timeframe, CandleType.from_string(candle_type) ) assert isinstance(fn, Path) @@ -181,7 +181,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): fn = JsonGzDataHandler._pair_data_filename( Path('freqtrade/hello/world'), pair, - JsonGzDataHandler.timeframe_to_file(timeframe), + timeframe, candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path)