From a4906c477edf44db9070e731ebb3476c6291ba8e Mon Sep 17 00:00:00 2001 From: Matthias Voppichler Date: Fri, 30 Mar 2018 23:30:23 +0200 Subject: [PATCH] Add handling for gzip files --- freqtrade/misc.py | 13 +++++++++--- freqtrade/tests/test_misc.py | 5 +++++ scripts/convert_backtestdata.py | 36 ++++++++++++++++----------------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bc04d6b88..fac379c18 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts import json import logging import re +import gzip from datetime import datetime from typing import Dict @@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: return np.sort(arr, axis=0) -def file_dump_json(filename, data) -> None: +def file_dump_json(filename, data, is_zip=False) -> None: """ Dump JSON data into a file :param filename: file to create :param data: JSON Data to save :return: """ - with open(filename, 'w') as fp: - json.dump(data, fp, default=str) + if not is_zip: + with open(filename, 'w') as fp: + json.dump(data, fp, default=str) + else: + if not filename.endswith('.gz'): + filename = filename + '.gz' + with gzip.open(filename, 'w') as fp: + json.dump(data, fp, default=str) def format_ms_time(date: str) -> str: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 3560b2db1..91c34b620 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None: file_dump_json('somefile', [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 + file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) + json_dump = mocker.patch('json.dump', MagicMock()) + file_dump_json('somefile', [1, 2, 3], True) + assert file_open.call_count == 1 + assert json_dump.call_count == 1 diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py index 051ff24cc..1ec553666 100755 --- a/scripts/convert_backtestdata.py +++ b/scripts/convert_backtestdata.py @@ -18,6 +18,7 @@ import glob import json import re from typing import List, Dict +import gzip from freqtrade.arguments import Arguments from freqtrade import misc @@ -26,24 +27,27 @@ from pandas import DataFrame import dateutil.parser -logger = Logger(name="Convert data").get_logger() +logger = Logger(name="Convert data", level=10).get_logger() -def load_old_file(filename) -> List[Dict]: +def load_old_file(filename) -> (List[Dict], bool): if not path.isfile(filename): logger.warning("filename %s does not exist", filename) - return None + return (None, False) logger.debug('Loading ticker data from file %s', filename) - # as in optimize/__init__.py::load_tickerdata_file - # if os.path.isfile(gzipfile): - # logger.debug('Loading ticker data from file %s', gzipfile) - # with gzip.open(gzipfile) as tickerdata: - # pairdata = json.load(tickerdata) pairdata = None - with open(filename) as tickerdata: - pairdata = json.load(tickerdata) - return pairdata + + if filename.endswith('.gz'): + logger.debug('Loading ticker data from file %s', filename) + is_zip = True + with gzip.open(filename) as tickerdata: + pairdata = json.load(tickerdata) + else: + is_zip = False + with open(filename) as tickerdata: + pairdata = json.load(tickerdata) + return (pairdata, is_zip) def parse_old_backtest_data(ticker) -> DataFrame: @@ -86,7 +90,7 @@ def convert_dataframe(frame: DataFrame): def convert_file(filename: str, filename_new: str): """Converts a file from old format to ccxt format""" - pairdata = load_old_file(filename) + (pairdata, is_zip) = load_old_file(filename) if pairdata and type(pairdata) is list and len(pairdata) > 0: if type(pairdata[0]) is list: logger.error("pairdata for %s already in new format", filename) @@ -95,8 +99,7 @@ def convert_file(filename: str, filename_new: str): frame = parse_old_backtest_data(pairdata) # Convert frame to new format frame1 = convert_dataframe(frame) - - misc.file_dump_json(filename_new, frame1) + misc.file_dump_json(filename_new, frame1, is_zip) def convert_main(args: Namespace) -> None: @@ -105,7 +108,7 @@ def convert_main(args: Namespace) -> None: """ workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/" - print(workdir) + logger.info("Workdir: %s", workdir) for filename in glob.glob(workdir + "*.json"): # swap currency names @@ -173,8 +176,5 @@ def main(sysargv: List[str]) -> None: convert_main(convert_parse_args(sysargv)) - - - if __name__ == '__main__': main(sys.argv[1:])