Add handling for gzip files

This commit is contained in:
Matthias Voppichler 2018-03-30 23:30:23 +02:00
parent 756bd63e1d
commit a4906c477e
3 changed files with 33 additions and 21 deletions

View File

@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts
import json import json
import logging import logging
import re import re
import gzip
from datetime import datetime from datetime import datetime
from typing import Dict from typing import Dict
@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray:
return np.sort(arr, axis=0) return np.sort(arr, axis=0)
def file_dump_json(filename, data) -> None: def file_dump_json(filename, data, is_zip=False) -> None:
""" """
Dump JSON data into a file Dump JSON data into a file
:param filename: file to create :param filename: file to create
:param data: JSON Data to save :param data: JSON Data to save
:return: :return:
""" """
if not is_zip:
with open(filename, 'w') as fp: with open(filename, 'w') as fp:
json.dump(data, fp, default=str) json.dump(data, fp, default=str)
else:
if not filename.endswith('.gz'):
filename = filename + '.gz'
with gzip.open(filename, 'w') as fp:
json.dump(data, fp, default=str)
def format_ms_time(date: str) -> str: def format_ms_time(date: str) -> str:

View File

@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None:
file_dump_json('somefile', [1, 2, 3]) file_dump_json('somefile', [1, 2, 3])
assert file_open.call_count == 1 assert file_open.call_count == 1
assert json_dump.call_count == 1 assert json_dump.call_count == 1
file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock())
json_dump = mocker.patch('json.dump', MagicMock())
file_dump_json('somefile', [1, 2, 3], True)
assert file_open.call_count == 1
assert json_dump.call_count == 1

View File

@ -18,6 +18,7 @@ import glob
import json import json
import re import re
from typing import List, Dict from typing import List, Dict
import gzip
from freqtrade.arguments import Arguments from freqtrade.arguments import Arguments
from freqtrade import misc from freqtrade import misc
@ -26,24 +27,27 @@ from pandas import DataFrame
import dateutil.parser import dateutil.parser
logger = Logger(name="Convert data").get_logger() logger = Logger(name="Convert data", level=10).get_logger()
def load_old_file(filename) -> List[Dict]: def load_old_file(filename) -> (List[Dict], bool):
if not path.isfile(filename): if not path.isfile(filename):
logger.warning("filename %s does not exist", filename) logger.warning("filename %s does not exist", filename)
return None return (None, False)
logger.debug('Loading ticker data from file %s', filename) logger.debug('Loading ticker data from file %s', filename)
# as in optimize/__init__.py::load_tickerdata_file
# if os.path.isfile(gzipfile):
# logger.debug('Loading ticker data from file %s', gzipfile)
# with gzip.open(gzipfile) as tickerdata:
# pairdata = json.load(tickerdata)
pairdata = None pairdata = None
if filename.endswith('.gz'):
logger.debug('Loading ticker data from file %s', filename)
is_zip = True
with gzip.open(filename) as tickerdata:
pairdata = json.load(tickerdata)
else:
is_zip = False
with open(filename) as tickerdata: with open(filename) as tickerdata:
pairdata = json.load(tickerdata) pairdata = json.load(tickerdata)
return pairdata return (pairdata, is_zip)
def parse_old_backtest_data(ticker) -> DataFrame: def parse_old_backtest_data(ticker) -> DataFrame:
@ -86,7 +90,7 @@ def convert_dataframe(frame: DataFrame):
def convert_file(filename: str, filename_new: str): def convert_file(filename: str, filename_new: str):
"""Converts a file from old format to ccxt format""" """Converts a file from old format to ccxt format"""
pairdata = load_old_file(filename) (pairdata, is_zip) = load_old_file(filename)
if pairdata and type(pairdata) is list and len(pairdata) > 0: if pairdata and type(pairdata) is list and len(pairdata) > 0:
if type(pairdata[0]) is list: if type(pairdata[0]) is list:
logger.error("pairdata for %s already in new format", filename) logger.error("pairdata for %s already in new format", filename)
@ -95,8 +99,7 @@ def convert_file(filename: str, filename_new: str):
frame = parse_old_backtest_data(pairdata) frame = parse_old_backtest_data(pairdata)
# Convert frame to new format # Convert frame to new format
frame1 = convert_dataframe(frame) frame1 = convert_dataframe(frame)
misc.file_dump_json(filename_new, frame1, is_zip)
misc.file_dump_json(filename_new, frame1)
def convert_main(args: Namespace) -> None: def convert_main(args: Namespace) -> None:
@ -105,7 +108,7 @@ def convert_main(args: Namespace) -> None:
""" """
workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/" workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/"
print(workdir) logger.info("Workdir: %s", workdir)
for filename in glob.glob(workdir + "*.json"): for filename in glob.glob(workdir + "*.json"):
# swap currency names # swap currency names
@ -173,8 +176,5 @@ def main(sysargv: List[str]) -> None:
convert_main(convert_parse_args(sysargv)) convert_main(convert_parse_args(sysargv))
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[1:]) main(sys.argv[1:])