From 4eaaec9d1a0bee87a6ce2b60b2044b5bd52164b7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jan 2020 10:36:08 +0100 Subject: [PATCH] Implement pair_to_filename to datahandler includes tests - taken from #2744 and modified to adapt to new structure --- freqtrade/data/history/jsondatahandler.py | 8 ++--- freqtrade/misc.py | 2 +- tests/data/test_history.py | 36 ++++++++++++++++------- tests/test_misc.py | 25 +++++++++++++++- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 7f0643862..7da1477af 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -30,7 +30,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] # Check if regex found something and only return these results - return [match[0].replace('_', '/') for match in _tmp if match] + return [misc.pair_to_filename(match[0]) for match in _tmp if match] def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ @@ -109,7 +109,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] # Check if regex found something and only return these results to avoid exceptions. - return [match[0].replace('_', '/') for match in _tmp if match] + return [misc.pair_to_filename(match[0]) for match in _tmp if match] def trades_store(self, pair: str, data: List[Dict]) -> None: """ @@ -157,7 +157,7 @@ class JsonDataHandler(IDataHandler): @classmethod def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = pair.replace("/", "_") + pair_s = misc.pair_to_filename(pair) filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') return filename @@ -167,7 +167,7 @@ class JsonDataHandler(IDataHandler): @classmethod def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: - pair_s = pair.replace("/", "_") + pair_s = misc.pair_to_filename(pair) filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename diff --git a/freqtrade/misc.py b/freqtrade/misc.py index e6ebc8d65..f012400c4 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -93,7 +93,7 @@ def file_load_json(file): def pair_to_filename(pair: str) -> str: - for ch in ['/', ' ', '.']: + for ch in ['/', '-', ' ', '.', '@', '$', '+', ':']: pair = pair.replace(ch, '_') return pair diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 2341673db..39000e508 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -144,23 +144,39 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -def test_json_pair_data_filename(): - fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json'), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json'), + ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json'), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json'), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json'), + ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json'), +]) +def test_json_pair_data_filename(pair, expected_result): + fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json') - fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') + assert fn == Path(expected_result) + fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json.gz') + assert fn == Path(expected_result + '.gz') -def test_json_pair_trades_filename(): - fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), + ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'), + ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), +]) +def test_json_pair_trades_filename(pair, expected_result): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json') + assert fn == Path(expected_result) - fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz') + assert fn == Path(expected_result + '.gz') def test_load_cached_data_for_updating(mocker, testdatadir) -> None: diff --git a/tests/test_misc.py b/tests/test_misc.py index c5bf06311..83e008466 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -4,9 +4,12 @@ import datetime from pathlib import Path from unittest.mock import MagicMock +import pytest + from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, - file_load_json, format_ms_time, plural, shorten_date) + file_load_json, format_ms_time, pair_to_filename, + plural, shorten_date) def test_shorten_date() -> None: @@ -57,6 +60,26 @@ def test_file_load_json(mocker, testdatadir) -> None: assert ret +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'ETH_BTC'), + ("Fabric Token/ETH", 'Fabric_Token_ETH'), + ("ETHH20", 'ETHH20'), + (".XBTBON2H", '_XBTBON2H'), + ("ETHUSD.d", 'ETHUSD_d'), + ("ADA-0327", 'ADA_0327'), + ("BTC-USD-200110", 'BTC_USD_200110'), + ("F-AKRO/USDT", 'F_AKRO_USDT'), + ("LC+/ETH", 'LC__ETH'), + ("CMT@18/ETH", 'CMT_18_ETH'), + ("LBTC:1022/SAI", 'LBTC_1022_SAI'), + ("$PAC/BTC", '_PAC_BTC'), + ("ACC_OLD/BTC", 'ACC_OLD_BTC'), +]) +def test_pair_to_filename(pair, expected_result): + pair_s = pair_to_filename(pair) + assert pair_s == expected_result + + def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000