diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e54b5a657..70c76536a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -67,18 +67,10 @@ def load_tickerdata_file( path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - gzipfile = file.with_suffix(file.suffix + '.gz') - # Try gzip file first, otherwise regular json file. - if gzipfile.is_file(): - logger.debug('Loading ticker data from file %s', gzipfile) - with gzip.open(gzipfile) as tickerdata: - pairdata = misc.json_load(tickerdata) - elif file.is_file(): - logger.debug('Loading ticker data from file %s', file) - with open(file) as tickerdata: - pairdata = misc.json_load(tickerdata) - else: + pairdata = misc.file_load_json(file) + + if not pairdata: return None if timerange: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 7fcfeec44..77b901be0 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -83,13 +83,31 @@ def file_dump_json(filename, data, is_zip=False) -> None: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) -def json_load(data): +def json_load(datafile): """ load data with rapidjson Use this to have a consistent experience, sete number_mode to "NM_NATIVE" for greatest speed """ - return rapidjson.load(data, number_mode=rapidjson.NM_NATIVE) + return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE) + + +def file_load_json(file): + + gzipfile = file.with_suffix(file.suffix + '.gz') + + # Try gzip file first, otherwise regular json file. + if gzipfile.is_file(): + logger.debug('Loading ticker data from file %s', gzipfile) + with gzip.open(gzipfile) as tickerdata: + pairdata = json_load(tickerdata) + elif file.is_file(): + logger.debug('Loading ticker data from file %s', file) + with open(file) as tickerdata: + pairdata = json_load(tickerdata) + else: + return None + return pairdata def format_ms_time(date: int) -> str: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 991e03fa0..4a630af14 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -2,11 +2,12 @@ import datetime from unittest.mock import MagicMock +from pathlib import Path from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, - file_dump_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file + file_dump_json, file_load_json, format_ms_time, shorten_date) +from freqtrade.data.history import load_tickerdata_file, make_testdata_path from freqtrade.strategy.default_strategy import DefaultStrategy @@ -57,6 +58,19 @@ def test_file_dump_json(mocker) -> None: assert json_dump.call_count == 1 +def test_file_load_json(mocker) -> None: + + # 7m .json does not exist + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json')) + assert not ret + # 1m json exists (but no .gz exists) + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json')) + assert ret + # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json')) + assert ret + + def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000