Add handling for gzip files

freqtrade/misc.py

@@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts
 import json
 import logging
 import re
+import gzip
 from datetime import datetime
 from typing import Dict
 
@@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray:
     return np.sort(arr, axis=0)
 
 
-def file_dump_json(filename, data) -> None:
+def file_dump_json(filename, data, is_zip=False) -> None:
     """
     Dump JSON data into a file
     :param filename: file to create
     :param data: JSON Data to save
     :return:
     """
-    with open(filename, 'w') as fp:
+    if not is_zip:
-        json.dump(data, fp, default=str)
+        with open(filename, 'w') as fp:
+            json.dump(data, fp, default=str)
+    else:
+        if not filename.endswith('.gz'):
+            filename = filename + '.gz'
+        with gzip.open(filename, 'w') as fp:
+            json.dump(data, fp, default=str)
 
 
 def format_ms_time(date: str) -> str:

freqtrade/tests/test_misc.py

@@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None:
     file_dump_json('somefile', [1, 2, 3])
     assert file_open.call_count == 1
     assert json_dump.call_count == 1
+    file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock())
+    json_dump = mocker.patch('json.dump', MagicMock())
+    file_dump_json('somefile', [1, 2, 3], True)
+    assert file_open.call_count == 1
+    assert json_dump.call_count == 1

scripts/convert_backtestdata.py

@@ -18,6 +18,7 @@ import glob
 import json
 import re
 from typing import List, Dict
+import gzip
 
 from freqtrade.arguments import Arguments
 from freqtrade import misc
@@ -26,24 +27,27 @@ from pandas import DataFrame
 
 import dateutil.parser
 
-logger = Logger(name="Convert data").get_logger()
+logger = Logger(name="Convert data", level=10).get_logger()
 
 
-def load_old_file(filename) -> List[Dict]:
+def load_old_file(filename) -> (List[Dict], bool):
     if not path.isfile(filename):
         logger.warning("filename %s does not exist", filename)
-        return None
+        return (None, False)
     logger.debug('Loading ticker data from file %s', filename)
-    # as in optimize/__init__.py::load_tickerdata_file
-    # if os.path.isfile(gzipfile):
-    #     logger.debug('Loading ticker data from file %s', gzipfile)
-    #     with gzip.open(gzipfile) as tickerdata:
-    #         pairdata = json.load(tickerdata)
 
     pairdata = None
-    with open(filename) as tickerdata:
+
-        pairdata = json.load(tickerdata)
+    if filename.endswith('.gz'):
-    return pairdata
+        logger.debug('Loading ticker data from file %s', filename)
+        is_zip = True
+        with gzip.open(filename) as tickerdata:
+            pairdata = json.load(tickerdata)
+    else:
+        is_zip = False
+        with open(filename) as tickerdata:
+            pairdata = json.load(tickerdata)
+    return (pairdata, is_zip)
 
 
 def parse_old_backtest_data(ticker) -> DataFrame:
@@ -86,7 +90,7 @@ def convert_dataframe(frame: DataFrame):
 
 def convert_file(filename: str, filename_new: str):
     """Converts a file from old format to ccxt format"""
-    pairdata = load_old_file(filename)
+    (pairdata, is_zip) = load_old_file(filename)
     if pairdata and type(pairdata) is list and len(pairdata) > 0:
         if type(pairdata[0]) is list:
             logger.error("pairdata for %s already in new format", filename)
@@ -95,8 +99,7 @@ def convert_file(filename: str, filename_new: str):
     frame = parse_old_backtest_data(pairdata)
     # Convert frame to new format
     frame1 = convert_dataframe(frame)
-
+    misc.file_dump_json(filename_new, frame1, is_zip)
-    misc.file_dump_json(filename_new, frame1)
 
 
 def convert_main(args: Namespace) -> None:
@@ -105,7 +108,7 @@ def convert_main(args: Namespace) -> None:
     """
 
     workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/"
-    print(workdir)
+    logger.info("Workdir: %s", workdir)
 
     for filename in glob.glob(workdir + "*.json"):
         # swap currency names
@@ -173,8 +176,5 @@ def main(sysargv: List[str]) -> None:
     convert_main(convert_parse_args(sysargv))
 
 
-
-
-
 if __name__ == '__main__':
     main(sys.argv[1:])