Add handling for gzip files

This commit is contained in:
Matthias Voppichler 2018-03-30 23:30:23 +02:00
parent 756bd63e1d
commit a4906c477e
3 changed files with 33 additions and 21 deletions

View File

@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts
import json
import logging
import re
import gzip
from datetime import datetime
from typing import Dict
@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray:
return np.sort(arr, axis=0)
def file_dump_json(filename, data) -> None:
def file_dump_json(filename, data, is_zip=False) -> None:
"""
Dump JSON data into a file
:param filename: file to create
:param data: JSON Data to save
:return:
"""
if not is_zip:
with open(filename, 'w') as fp:
json.dump(data, fp, default=str)
else:
if not filename.endswith('.gz'):
filename = filename + '.gz'
with gzip.open(filename, 'w') as fp:
json.dump(data, fp, default=str)
def format_ms_time(date: str) -> str:

View File

@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None:
file_dump_json('somefile', [1, 2, 3])
assert file_open.call_count == 1
assert json_dump.call_count == 1
file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock())
json_dump = mocker.patch('json.dump', MagicMock())
file_dump_json('somefile', [1, 2, 3], True)
assert file_open.call_count == 1
assert json_dump.call_count == 1

View File

@ -18,6 +18,7 @@ import glob
import json
import re
from typing import List, Dict
import gzip
from freqtrade.arguments import Arguments
from freqtrade import misc
@ -26,24 +27,27 @@ from pandas import DataFrame
import dateutil.parser
logger = Logger(name="Convert data").get_logger()
logger = Logger(name="Convert data", level=10).get_logger()
def load_old_file(filename) -> List[Dict]:
def load_old_file(filename) -> (List[Dict], bool):
if not path.isfile(filename):
logger.warning("filename %s does not exist", filename)
return None
return (None, False)
logger.debug('Loading ticker data from file %s', filename)
# as in optimize/__init__.py::load_tickerdata_file
# if os.path.isfile(gzipfile):
# logger.debug('Loading ticker data from file %s', gzipfile)
# with gzip.open(gzipfile) as tickerdata:
# pairdata = json.load(tickerdata)
pairdata = None
if filename.endswith('.gz'):
logger.debug('Loading ticker data from file %s', filename)
is_zip = True
with gzip.open(filename) as tickerdata:
pairdata = json.load(tickerdata)
else:
is_zip = False
with open(filename) as tickerdata:
pairdata = json.load(tickerdata)
return pairdata
return (pairdata, is_zip)
def parse_old_backtest_data(ticker) -> DataFrame:
@ -86,7 +90,7 @@ def convert_dataframe(frame: DataFrame):
def convert_file(filename: str, filename_new: str):
"""Converts a file from old format to ccxt format"""
pairdata = load_old_file(filename)
(pairdata, is_zip) = load_old_file(filename)
if pairdata and type(pairdata) is list and len(pairdata) > 0:
if type(pairdata[0]) is list:
logger.error("pairdata for %s already in new format", filename)
@ -95,8 +99,7 @@ def convert_file(filename: str, filename_new: str):
frame = parse_old_backtest_data(pairdata)
# Convert frame to new format
frame1 = convert_dataframe(frame)
misc.file_dump_json(filename_new, frame1)
misc.file_dump_json(filename_new, frame1, is_zip)
def convert_main(args: Namespace) -> None:
@ -105,7 +108,7 @@ def convert_main(args: Namespace) -> None:
"""
workdir = args.datadir if args.datadir.endswith("/") else args.datadir + "/"
print(workdir)
logger.info("Workdir: %s", workdir)
for filename in glob.glob(workdir + "*.json"):
# swap currency names
@ -173,8 +176,5 @@ def main(sysargv: List[str]) -> None:
convert_main(convert_parse_args(sysargv))
if __name__ == '__main__':
main(sys.argv[1:])