Merge pull request #1444 from freqtrade/replace_ujson

Replace ujson with rapidjson

freqtrade/data/history.py

@@ -5,15 +5,12 @@ includes:
 * download data from exchange and store to disk
 """
 
-import gzip
-
 import logging
 from pathlib import Path
 from typing import Optional, List, Dict, Tuple, Any
 
 import arrow
 from pandas import DataFrame
-import ujson
 
 from freqtrade import misc, constants, OperationalException
 from freqtrade.data.converter import parse_ticker_dataframe
@@ -23,15 +20,6 @@ from freqtrade.arguments import TimeRange
 logger = logging.getLogger(__name__)
 
 
-def json_load(data):
-    """
-    load data with ujson
-    Use this to have a consistent experience,
-    otherwise "precise_float" needs to be passed to all load operations
-    """
-    return ujson.load(data, precise_float=True)
-
-
 def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     """
     Trim tickerlist based on given timerange
@@ -77,18 +65,10 @@ def load_tickerdata_file(
     path = make_testdata_path(datadir)
     pair_s = pair.replace('/', '_')
     file = path.joinpath(f'{pair_s}-{ticker_interval}.json')
-    gzipfile = file.with_suffix(file.suffix + '.gz')
 
-    # Try gzip file first, otherwise regular json file.
-    if gzipfile.is_file():
-        logger.debug('Loading ticker data from file %s', gzipfile)
-        with gzip.open(gzipfile) as tickerdata:
-            pairdata = json_load(tickerdata)
-    elif file.is_file():
-        logger.debug('Loading ticker data from file %s', file)
-        with open(file) as tickerdata:
-            pairdata = json_load(tickerdata)
-    else:
+    pairdata = misc.file_load_json(file)
+
+    if not pairdata:
         return None
 
     if timerange:
@@ -185,7 +165,7 @@ def load_cached_data_for_updating(filename: Path, tick_interval: str,
     # read the cached file
     if filename.is_file():
         with open(filename, "rt") as file:
-            data = json_load(file)
+            data = misc.json_load(file)
         # remove the last item, could be incomplete candle
         if data:
             data.pop()

freqtrade/misc.py

@@ -3,7 +3,6 @@ Various tool function for Freqtrade and scripts
 """
 
 import gzip
-import json
 import logging
 import re
 from datetime import datetime
@@ -11,6 +10,7 @@ from typing import Dict
 
 import numpy as np
 from pandas import DataFrame
+import rapidjson
 
 logger = logging.getLogger(__name__)
 
@@ -71,16 +71,45 @@ def file_dump_json(filename, data, is_zip=False) -> None:
     :param data: JSON Data to save
     :return:
     """
-    print(f'dumping json to "{filename}"')
+    logger.info(f'dumping json to "{filename}"')
 
     if is_zip:
         if not filename.endswith('.gz'):
             filename = filename + '.gz'
         with gzip.open(filename, 'w') as fp:
-            json.dump(data, fp, default=str)
+            rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)
     else:
         with open(filename, 'w') as fp:
-            json.dump(data, fp, default=str)
+            rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)
+
+    logger.debug(f'done json to "{filename}"')
+
+
+def json_load(datafile):
+    """
+    load data with rapidjson
+    Use this to have a consistent experience,
+    sete number_mode to "NM_NATIVE" for greatest speed
+    """
+    return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE)
+
+
+def file_load_json(file):
+
+    gzipfile = file.with_suffix(file.suffix + '.gz')
+
+    # Try gzip file first, otherwise regular json file.
+    if gzipfile.is_file():
+        logger.debug('Loading ticker data from file %s', gzipfile)
+        with gzip.open(gzipfile) as tickerdata:
+            pairdata = json_load(tickerdata)
+    elif file.is_file():
+        logger.debug('Loading ticker data from file %s', file)
+        with open(file) as tickerdata:
+            pairdata = json_load(tickerdata)
+    else:
+        return None
+    return pairdata
 
 
 def format_ms_time(date: int) -> str:

freqtrade/tests/data/test_history.py

@@ -450,7 +450,7 @@ def test_trim_tickerlist() -> None:
     assert not ticker
 
 
-def test_file_dump_json() -> None:
+def test_file_dump_json_tofile() -> None:
     file = os.path.join(os.path.dirname(__file__), '..', 'testdata',
                         'test_{id}.json'.format(id=str(uuid.uuid4())))
     data = {'bar': 'foo'}

freqtrade/tests/test_misc.py

@@ -5,8 +5,8 @@ from unittest.mock import MagicMock
 
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.misc import (common_datearray, datesarray_to_datetimearray,
-                            file_dump_json, format_ms_time, shorten_date)
-from freqtrade.data.history import load_tickerdata_file
+                            file_dump_json, file_load_json, format_ms_time, shorten_date)
+from freqtrade.data.history import load_tickerdata_file, make_testdata_path
 from freqtrade.strategy.default_strategy import DefaultStrategy
 
 
@@ -46,17 +46,30 @@ def test_common_datearray(default_conf) -> None:
 
 def test_file_dump_json(mocker) -> None:
     file_open = mocker.patch('freqtrade.misc.open', MagicMock())
-    json_dump = mocker.patch('json.dump', MagicMock())
+    json_dump = mocker.patch('rapidjson.dump', MagicMock())
     file_dump_json('somefile', [1, 2, 3])
     assert file_open.call_count == 1
     assert json_dump.call_count == 1
     file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock())
-    json_dump = mocker.patch('json.dump', MagicMock())
+    json_dump = mocker.patch('rapidjson.dump', MagicMock())
     file_dump_json('somefile', [1, 2, 3], True)
     assert file_open.call_count == 1
     assert json_dump.call_count == 1
 
 
+def test_file_load_json(mocker) -> None:
+
+    # 7m .json does not exist
+    ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-7m.json'))
+    assert not ret
+    # 1m json exists (but no .gz exists)
+    ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-1m.json'))
+    assert ret
+    # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json
+    ret = file_load_json(make_testdata_path(None).joinpath('UNITTEST_BTC-8m.json'))
+    assert ret
+
+
 def test_format_ms_time() -> None:
     # Date 2018-04-10 18:02:01
     date_in_epoch_ms = 1523383321000

requirements.txt

@@ -23,4 +23,4 @@ scikit-optimize==0.5.2
 py_find_1st==1.1.3
 
 #Load ticker files 30% faster
-ujson==1.35
+python-rapidjson==0.6.3

setup.py

@@ -38,7 +38,7 @@ setup(name='freqtrade',
           'cachetools',
           'coinmarketcap',
           'scikit-optimize',
-          'ujson',
+          'python-rapidjson',
           'py_find_1st'
       ],
       include_package_data=True,