From c955415cc3b13a9cda40a10b13f898fbf9342b81 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Dec 2018 10:01:16 +0100 Subject: [PATCH 1/6] Switch from ujson to rapidjson --- freqtrade/data/history.py | 8 ++++---- freqtrade/misc.py | 6 +++--- freqtrade/tests/test_misc.py | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index f4ff46a1a..54d9abf2b 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -13,7 +13,7 @@ from typing import Optional, List, Dict, Tuple, Any import arrow from pandas import DataFrame -import ujson +import rapidjson from freqtrade import misc, constants, OperationalException from freqtrade.data.converter import parse_ticker_dataframe @@ -25,11 +25,11 @@ logger = logging.getLogger(__name__) def json_load(data): """ - load data with ujson + load data with rapidjson Use this to have a consistent experience, - otherwise "precise_float" needs to be passed to all load operations + sete number_mode to "NM_NATIVE" for greatest speed """ - return ujson.load(data, precise_float=True) + return rapidjson.load(data, number_mode=rapidjson.NM_NATIVE) def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 832437951..7bb9ddced 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -3,7 +3,6 @@ Various tool function for Freqtrade and scripts """ import gzip -import json import logging import re from datetime import datetime @@ -11,6 +10,7 @@ from typing import Dict import numpy as np from pandas import DataFrame +import rapidjson logger = logging.getLogger(__name__) @@ -77,10 +77,10 @@ def file_dump_json(filename, data, is_zip=False) -> None: if not filename.endswith('.gz'): filename = filename + '.gz' with gzip.open(filename, 'w') as fp: - json.dump(data, fp, default=str) + rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: with open(filename, 'w') as fp: - json.dump(data, fp, default=str) + rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) def format_ms_time(date: int) -> str: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 017bf372f..991e03fa0 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -46,12 +46,12 @@ def test_common_datearray(default_conf) -> None: def test_file_dump_json(mocker) -> None: file_open = mocker.patch('freqtrade.misc.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) + json_dump = mocker.patch('rapidjson.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) - json_dump = mocker.patch('json.dump', MagicMock()) + json_dump = mocker.patch('rapidjson.dump', MagicMock()) file_dump_json('somefile', [1, 2, 3], True) assert file_open.call_count == 1 assert json_dump.call_count == 1 diff --git a/requirements.txt b/requirements.txt index d0fe69970..d7a6f748e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,4 @@ scikit-optimize==0.5.2 py_find_1st==1.1.3 #Load ticker files 30% faster -ujson==1.35 +python-rapidjson==0.6.3 diff --git a/setup.py b/setup.py index b9e3620df..11f957e67 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', - 'ujson', + 'python-rapidjson', 'py_find_1st' ], include_package_data=True, From 065b469a106d0cb978667ca9a82173d1d31aa89a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Dec 2018 10:04:07 +0100 Subject: [PATCH 2/6] rename test to avoid naming collision --- freqtrade/tests/data/test_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/data/test_history.py b/freqtrade/tests/data/test_history.py index 8923a60b6..bc859b325 100644 --- a/freqtrade/tests/data/test_history.py +++ b/freqtrade/tests/data/test_history.py @@ -450,7 +450,7 @@ def test_trim_tickerlist() -> None: assert not ticker -def test_file_dump_json() -> None: +def test_file_dump_json_tofile() -> None: file = os.path.join(os.path.dirname(__file__), '..', 'testdata', 'test_{id}.json'.format(id=str(uuid.uuid4()))) data = {'bar': 'foo'} From 27abdd97888524ca76634e0f7a86bb8f2e3f055e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Dec 2018 10:04:28 +0100 Subject: [PATCH 3/6] Move load_json to misc --- freqtrade/data/history.py | 16 +++------------- freqtrade/misc.py | 9 +++++++++ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 54d9abf2b..e54b5a657 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -13,7 +13,6 @@ from typing import Optional, List, Dict, Tuple, Any import arrow from pandas import DataFrame -import rapidjson from freqtrade import misc, constants, OperationalException from freqtrade.data.converter import parse_ticker_dataframe @@ -23,15 +22,6 @@ from freqtrade.arguments import TimeRange logger = logging.getLogger(__name__) -def json_load(data): - """ - load data with rapidjson - Use this to have a consistent experience, - sete number_mode to "NM_NATIVE" for greatest speed - """ - return rapidjson.load(data, number_mode=rapidjson.NM_NATIVE) - - def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: """ Trim tickerlist based on given timerange @@ -83,11 +73,11 @@ def load_tickerdata_file( if gzipfile.is_file(): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json_load(tickerdata) + pairdata = misc.json_load(tickerdata) elif file.is_file(): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json_load(tickerdata) + pairdata = misc.json_load(tickerdata) else: return None @@ -185,7 +175,7 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str, # read the cached file if filename.is_file(): with open(filename, "rt") as file: - data = json_load(file) + data = misc.json_load(file) # remove the last item, could be incomplete candle if data: data.pop() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 7bb9ddced..7fcfeec44 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -83,6 +83,15 @@ def file_dump_json(filename, data, is_zip=False) -> None: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) +def json_load(data): + """ + load data with rapidjson + Use this to have a consistent experience, + sete number_mode to "NM_NATIVE" for greatest speed + """ + return rapidjson.load(data, number_mode=rapidjson.NM_NATIVE) + + def format_ms_time(date: int) -> str: """ convert MS date to readable format. From 7dc40cdac53faee936910c317f08ab0acc61a86c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Dec 2018 10:25:12 +0100 Subject: [PATCH 4/6] refactor file_load_json to be standalone --- freqtrade/data/history.py | 14 +++----------- freqtrade/misc.py | 22 ++++++++++++++++++++-- freqtrade/tests/test_misc.py | 18 ++++++++++++++++-- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index e54b5a657..70c76536a 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -67,18 +67,10 @@ def load_tickerdata_file( path = make_testdata_path(datadir) pair_s = pair.replace('/', '_') file = path.joinpath(f'{pair_s}-{ticker_interval}.json') - gzipfile = file.with_suffix(file.suffix + '.gz') - # Try gzip file first, otherwise regular json file. - if gzipfile.is_file(): - logger.debug('Loading ticker data from file %s', gzipfile) - with gzip.open(gzipfile) as tickerdata: - pairdata = misc.json_load(tickerdata) - elif file.is_file(): - logger.debug('Loading ticker data from file %s', file) - with open(file) as tickerdata: - pairdata = misc.json_load(tickerdata) - else: + pairdata = misc.file_load_json(file) + + if not pairdata: return None if timerange: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 7fcfeec44..77b901be0 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -83,13 +83,31 @@ def file_dump_json(filename, data, is_zip=False) -> None: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) -def json_load(data): +def json_load(datafile): """ load data with rapidjson Use this to have a consistent experience, sete number_mode to "NM_NATIVE" for greatest speed """ - return rapidjson.load(data, number_mode=rapidjson.NM_NATIVE) + return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE) + + +def file_load_json(file): + + gzipfile = file.with_suffix(file.suffix + '.gz') + + # Try gzip file first, otherwise regular json file. + if gzipfile.is_file(): + logger.debug('Loading ticker data from file %s', gzipfile) + with gzip.open(gzipfile) as tickerdata: + pairdata = json_load(tickerdata) + elif file.is_file(): + logger.debug('Loading ticker data from file %s', file) + with open(file) as tickerdata: + pairdata = json_load(tickerdata) + else: + return None + return pairdata def format_ms_time(date: int) -> str: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 991e03fa0..4a630af14 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -2,11 +2,12 @@ import datetime from unittest.mock import MagicMock +from pathlib import Path from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, - file_dump_json, format_ms_time, shorten_date) -from freqtrade.data.history import load_tickerdata_file + file_dump_json, file_load_json, format_ms_time, shorten_date) +from freqtrade.data.history import load_tickerdata_file, make_testdata_path from freqtrade.strategy.default_strategy import DefaultStrategy @@ -57,6 +58,19 @@ def test_file_dump_json(mocker) -> None: assert json_dump.call_count == 1 +def test_file_load_json(mocker) -> None: + + # 7m .json does not exist + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json')) + assert not ret + # 1m json exists (but no .gz exists) + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json')) + assert ret + # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json + ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json')) + assert ret + + def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 From 61f8ce5c0e552cdf2644e094066b4bfbaa2f0dbf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Dec 2018 10:44:24 +0100 Subject: [PATCH 5/6] remove unused imports --- freqtrade/data/history.py | 2 -- freqtrade/tests/test_misc.py | 1 - 2 files changed, 3 deletions(-) diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py index 70c76536a..ae56aa6c7 100644 --- a/freqtrade/data/history.py +++ b/freqtrade/data/history.py @@ -5,8 +5,6 @@ includes: * download data from exchange and store to disk """ -import gzip - import logging from pathlib import Path from typing import Optional, List, Dict, Tuple, Any diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 4a630af14..33a59effd 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -2,7 +2,6 @@ import datetime from unittest.mock import MagicMock -from pathlib import Path from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.misc import (common_datearray, datesarray_to_datetimearray, From fab7663ab3810ae0ba2722cf855fcf618a9236cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 Dec 2018 10:46:48 +0100 Subject: [PATCH 6/6] Log when dumping to file (instead of print) --- freqtrade/misc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 77b901be0..23d8732a0 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -71,7 +71,7 @@ def file_dump_json(filename, data, is_zip=False) -> None: :param data: JSON Data to save :return: """ - print(f'dumping json to "{filename}"') + logger.info(f'dumping json to "{filename}"') if is_zip: if not filename.endswith('.gz'): @@ -82,6 +82,8 @@ def file_dump_json(filename, data, is_zip=False) -> None: with open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) + logger.debug(f'done json to "{filename}"') + def json_load(datafile): """