diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index f4ff46a1a..ae56aa6c7 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -5,15 +5,12 @@ includes: * download data from exchange and store to disk """ -import gzip - import logging from pathlib import Path from typing import Optional, List, Dict, Tuple, Any import arrow from pandas import DataFrame -import ujson from freqtrade import misc, constants, OperationalException from freqtrade.data.converter import parse_ticker_dataframe @@ -23,15 +20,6 @@ from freqtrade.arguments import TimeRange logger = logging.getLogger(__name__) -def json_load(data): - """ - load data with ujson - Use this to have a consistent experience, - otherwise "precise_float" needs to be passed to all load operations - """ - return ujson.load(data, precise_float=True) - - def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: """ Trim tickerlist based on given timerange @@ -77,18 +65,10 @@ def load_tickerdata_file( path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - gzipfile = file.with_suffix(file.suffix + '.gz') - # Try gzip file first, otherwise regular json file. - if gzipfile.is_file(): - logger.debug('Loading ticker data from file %s', gzipfile) - with gzip.open(gzipfile) as tickerdata: - pairdata = json_load(tickerdata) - elif file.is_file(): - logger.debug('Loading ticker data from file %s', file) - with open(file) as tickerdata: - pairdata = json_load(tickerdata) - else: + pairdata = misc.file_load_json(file) + + if not pairdata: return None if timerange: @@ -185,7 +165,7 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, # read the cached file if filename.is_file(): with open(filename, "rt") as file: - data = json_load(file) + data = misc.json_load(file) # remove the last item, could be incomplete candle if data: data.pop() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 832437951..23d8732a0 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,7 +3,6 @@ Various tool function for Freqtrade and scripts """ import gzip -import json import logging import re from datetime import datetime @@ -11,6 +10,7 @@ from typing import Dict import numpy as np from pandas import DataFrame +import rapidjson logger = logging.getLogger(__name__) @@ -71,16 +71,45 @@ def file_dump_json(filename, data, is_zip=False) -> None: :param data: JSON Data to save :return: """ - print(f'dumping json to "{filename}"') + logger.info(f'dumping json to "{filename}"') if is_zip: if not filename.endswith('.gz'): filename = filename + '.gz' with gzip.open(filename, 'w') as fp: - json.dump(data, fp, default=str) + rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: with open(filename, 'w') as fp: - json.dump(data, fp, default=str) + rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) + + logger.debug(f'done json to "{filename}"') + + +def json_load(datafile): + """ + load data with rapidjson + Use this to have a consistent experience, + sete number_mode to "NM_NATIVE" for greatest speed + """ + return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE) + + +def file_load_json(file): + + gzipfile = file.with_suffix(file.suffix + '.gz') + + # Try gzip file first, otherwise regular json file. + if gzipfile.is_file(): + logger.debug('Loading ticker data from file %s', gzipfile) + with gzip.open(gzipfile) as tickerdata: + pairdata = json_load(tickerdata) + elif file.is_file(): + logger.debug('Loading ticker data from file %s', file) + with open(file) as tickerdata: + pairdata = json_load(tickerdata) + else: + return None + return pairdata def format_ms_time(date: int) -> str: diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 8923a60b6..bc859b325 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -450,7 +450,7 @@ def test_trim_tickerlist() -> None: assert not ticker -def test_file_dump_json() -> None: +def test_file_dump_json_tofile() -> None: file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'test_{id}.json'.format(id=str(uuid.uuid4()))) data = {'bar': 'foo'} diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 017bf372f..33a59effd 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, - file_dump_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file + file_dump_json, file_load_json, format_ms_time, shorten_date) +from freqtrade.data.history import load_tickerdata_file, make_testdata_path from freqtrade.strategy.default_strategy import DefaultStrategy @@ -46,17 +46,30 @@ def test_common_datearray(default_conf) -> None: def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) + json_dump = mocker.patch('rapidjson.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) + json_dump = mocker.patch('rapidjson.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3], True) assert file_open.call_count == 1 assert json_dump.call_count == 1 +def test_file_load_json(mocker) -> None: + + # 7m .json does not exist + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json')) + assert not ret + # 1m json exists (but no .gz exists) + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json')) + assert ret + # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json')) + assert ret + + def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 diff --git a/requirements.txt b/requirements.txt index d0fe69970..d7a6f748e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,4 @@ scikit-optimize==0.5.2 py_find_1st==1.1.3 #Load ticker files 30% faster -ujson==1.35 +python-rapidjson==0.6.3 diff --git a/setup.py b/setup.py index b9e3620df..11f957e67 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', - 'ujson', + 'python-rapidjson', 'py_find_1st' ], include_package_data=True,