From af345404506b83a077abd0541cc646b462239be7 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 12 Jul 2018 12:24:28 +0000 Subject: [PATCH 1/6] Using ujson to load ticker files from disk during backtest is over 30% faster. --- freqtrade/optimize/__init__.py | 5 +++-- requirements.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e806ff2b8..3188b9a5f 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -2,6 +2,7 @@ import gzip import json +import ujson import logging import os from typing import Optional, List, Dict, Tuple, Any @@ -67,7 +68,7 @@ def load_tickerdata_file( elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json.load(tickerdata) + pairdata = ujson.load(tickerdata) else: return None @@ -163,7 +164,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + data = ujson.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: diff --git a/requirements.txt b/requirements.txt index 3fb91888c..d118f3be4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 coinmarketcap==5.0.3 +ujson==1.35 # Required for hyperopt scikit-optimize==0.5.2 From 87f7d9972f72ab79d80f64192c26c006270b94fa Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 12 Jul 2018 12:40:25 +0000 Subject: [PATCH 2/6] Using ujson to load ticker files from disk during backtest is over 30% faster. --- freqtrade/optimize/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 3188b9a5f..f451c68b8 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -164,7 +164,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = ujson.load(file) + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From 801b77cf8c7c48f04b918e0518ce28bb0abe5fc2 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 12 Jul 2018 13:05:09 +0000 Subject: [PATCH 3/6] Make module optional, hashed out by default in requirements.txt optimise tests for module to decide how to load tickers. --- freqtrade/optimize/__init__.py | 11 +++++++++-- requirements.txt | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index f451c68b8..ac5cda4d0 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -2,7 +2,6 @@ import gzip import json -import ujson import logging import os from typing import Optional, List, Dict, Tuple, Any @@ -12,6 +11,11 @@ from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +import importlib +ujson_found = importlib.util.find_spec("ujson") +if ujson_found is not None: + import ujson + logger = logging.getLogger(__name__) @@ -68,7 +72,10 @@ def load_tickerdata_file( elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = ujson.load(tickerdata) + if ujson_found is not None: + pairdata = ujson.load(tickerdata) + else: + pairdata = json.load(tickerdata) else: return None diff --git a/requirements.txt b/requirements.txt index d118f3be4..948c2a1c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,10 @@ pytest-mock==1.10.0 pytest-cov==2.5.1 tabulate==0.8.2 coinmarketcap==5.0.3 -ujson==1.35 + +# Optional, speeds up ticker load +# Requires visual studio in windows. +#ujson==1.35 # Required for hyperopt scikit-optimize==0.5.2 From 3b7bcf9b352b201b9bd2020babb2cda8d33a7bb2 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 12 Jul 2018 16:06:02 +0000 Subject: [PATCH 4/6] Added KWarg to calls to ujson, this aligns it FP presicion handing with how python json manages imports. Added log. All DFs are now aligns, tests passes. We cannot call import ujson as json as the kwarg breaks regular json. --- freqtrade/optimize/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index ac5cda4d0..7bdaef852 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -11,13 +11,13 @@ from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +logger = logging.getLogger(__name__) + import importlib ujson_found = importlib.util.find_spec("ujson") if ujson_found is not None: import ujson - -logger = logging.getLogger(__name__) - + logger.debug('Loaded UltraJson ujson in optimize.py') def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: @@ -68,12 +68,15 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json.load(tickerdata) + if ujson_found is not None: + pairdata = ujson.load(tickerdata, precise_float=True) + else: + pairdata = json.load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: if ujson_found is not None: - pairdata = ujson.load(tickerdata) + pairdata = ujson.load(tickerdata, precise_float=True) else: pairdata = json.load(tickerdata) else: @@ -171,7 +174,10 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + if ujson_found is not None: + data = ujson.load(file, precise_float=True) + else: + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From 297d52228ff8f03954225f91a4b72a9fb2350c9e Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 12 Jul 2018 16:17:28 +0000 Subject: [PATCH 5/6] flake 8 fix --- freqtrade/optimize/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 7bdaef852..fca061596 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,15 +10,17 @@ import arrow from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange - -logger = logging.getLogger(__name__) - import importlib ujson_found = importlib.util.find_spec("ujson") if ujson_found is not None: import ujson + +logger = logging.getLogger(__name__) + +if ujson_found is not None: logger.debug('Loaded UltraJson ujson in optimize.py') + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: return tickerlist From 0f36e42c2e455a26754d245913279f84b322d677 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 12 Jul 2018 16:27:17 +0000 Subject: [PATCH 6/6] Update requirements.txt clarified comment --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 948c2a1c5..833fe0e56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ tabulate==0.8.2 coinmarketcap==5.0.3 # Optional, speeds up ticker load -# Requires visual studio in windows. +# Requires visual studio if on windows. #ujson==1.35 # Required for hyperopt