diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bc04d6b88..7546dba8f 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,6 +5,7 @@ Various tool function for Freqtrade and scripts import json import logging import re +import gzip from datetime import datetime from typing import Dict @@ -63,15 +64,21 @@ def common_datearray(dfs: Dict[str, DataFrame]) -> np.ndarray: return np.sort(arr, axis=0) -def file_dump_json(filename, data) -> None: +def file_dump_json(filename, data, is_zip=False) -> None: """ Dump JSON data into a file :param filename: file to create :param data: JSON Data to save :return: """ - with open(filename, 'w') as fp: - json.dump(data, fp, default=str) + if is_zip: + if not filename.endswith('.gz'): + filename = filename + '.gz' + with gzip.open(filename, 'w') as fp: + json.dump(data, fp, default=str) + else: + with open(filename, 'w') as fp: + json.dump(data, fp, default=str) def format_ms_time(date: str) -> str: diff --git a/freqtrade/tests/test_misc.py b/freqtrade/tests/test_misc.py index 3560b2db1..91c34b620 100644 --- a/freqtrade/tests/test_misc.py +++ b/freqtrade/tests/test_misc.py @@ -71,3 +71,8 @@ def test_file_dump_json(mocker) -> None: file_dump_json('somefile', [1, 2, 3]) assert file_open.call_count == 1 assert json_dump.call_count == 1 + file_open = mocker.patch('freqtrade.misc.gzip.open', MagicMock()) + json_dump = mocker.patch('json.dump', MagicMock()) + file_dump_json('somefile', [1, 2, 3], True) + assert file_open.call_count == 1 + assert json_dump.call_count == 1 diff --git a/scripts/convert_backtestdata.py b/scripts/convert_backtestdata.py new file mode 100755 index 000000000..a6f6ccc60 --- /dev/null +++ b/scripts/convert_backtestdata.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +""" +Script to display when the bot will buy a specific pair + +Mandatory Cli parameters: +-p / --pair: pair to examine + +Optional Cli parameters +-d / --datadir: path to pair backtest data +--timerange: specify what timerange of data to use. +-l / --live: Live, to download the latest ticker for the pair +""" + +import sys +from argparse import Namespace +from os import path +import glob +import json +import re +from typing import List, Dict +import gzip + +from freqtrade.arguments import Arguments +from freqtrade import misc +from freqtrade.logger import Logger +from pandas import DataFrame + +import dateutil.parser + +logger = Logger(name="freqtrade").get_logger() + + +def load_old_file(filename) -> (List[Dict], bool): + if not path.isfile(filename): + logger.warning("filename %s does not exist", filename) + return (None, False) + logger.debug('Loading ticker data from file %s', filename) + + pairdata = None + + if filename.endswith('.gz'): + logger.debug('Loading ticker data from file %s', filename) + is_zip = True + with gzip.open(filename) as tickerdata: + pairdata = json.load(tickerdata) + else: + is_zip = False + with open(filename) as tickerdata: + pairdata = json.load(tickerdata) + return (pairdata, is_zip) + + +def parse_old_backtest_data(ticker) -> DataFrame: + """ + Reads old backtest data + Format: "O": 8.794e-05, + "H": 8.948e-05, + "L": 8.794e-05, + "C": 8.88e-05, + "V": 991.09056638, + "T": "2017-11-26T08:50:00", + "BV": 0.0877869 + """ + + columns = {'C': 'close', 'V': 'volume', 'O': 'open', + 'H': 'high', 'L': 'low', 'T': 'date'} + + frame = DataFrame(ticker) \ + .rename(columns=columns) + if 'BV' in frame: + frame.drop('BV', 1, inplace=True) + if not 'date' in frame: + logger.warning("Date not in frame - probably not a Ticker file") + return None + frame.sort_values('date', inplace=True) + return frame + + +def convert_dataframe(frame: DataFrame): + """Convert dataframe to new format""" + # reorder columns: + cols = ['date', 'open', 'high', 'low', 'close', 'volume'] + frame = frame[cols] + + frame['date'] = frame['date'].apply( + lambda d: int(dateutil.parser.parse(d).timestamp()) * 1000) + frame['date'] = frame['date'].astype(int) + # Convert columns one by one to preserve type. + by_column = [frame[x].values.tolist() for x in frame.columns] + return list(list(x) for x in zip(*by_column)) + + +def convert_file(filename: str, filename_new: str) -> None: + """Converts a file from old format to ccxt format""" + (pairdata, is_zip) = load_old_file(filename) + if pairdata and type(pairdata) is list: + if type(pairdata[0]) is list: + logger.error("pairdata for %s already in new format", filename) + return + + frame = parse_old_backtest_data(pairdata) + # Convert frame to new format + if frame is not None: + frame1 = convert_dataframe(frame) + misc.file_dump_json(filename_new, frame1, is_zip) + + +def convert_main(args: Namespace) -> None: + """ + converts a folder given in --datadir from old to new format to support ccxt + """ + + workdir = path.join(args.datadir, "") + logger.info("Workdir: %s", workdir) + + for filename in glob.glob(workdir + "*.json"): + # swap currency names + ret = re.search(r'[A-Z_]{7,}', path.basename(filename)) + if args.norename: + filename_new = filename + else: + if not ret: + logger.warning("file %s could not be converted, could not extract currencies", + filename) + continue + pair = ret.group(0) + currencies = pair.split("_") + if len(currencies) != 2: + logger.warning("file %s could not be converted, could not extract currencies", + filename) + continue + + ret = re.search(r'\d+(?=\.json)', path.basename(filename)) + if not ret: + logger.warning("file %s could not be converted, interval not found", filename) + continue + interval = ret.group(0) + + filename_new = path.join(path.dirname(filename), + "{}_{}-{}.json".format(currencies[1], + currencies[0], interval)) + logger.debug("Converting and renaming %s to %s", filename, filename_new) + convert_file(filename, filename_new) + + +def convert_parse_args(args: List[str]) -> Namespace: + """ + Parse args passed to the script + :param args: Cli arguments + :return: args: Array with all arguments + """ + arguments = Arguments(args, 'Convert datafiles') + arguments.parser.add_argument( + '-d', '--datadir', + help='path to backtest data (default: %(default)s', + dest='datadir', + default=path.join('freqtrade', 'tests', 'testdata'), + type=str, + metavar='PATH', + ) + arguments.parser.add_argument( + '-n', '--norename', + help='don''t rename files from BTC_ to _BTC - ' + 'Note that not renaming will overwrite source files', + dest='norename', + default=False, + action='store_true' + ) + + return arguments.parse_args() + + +def main(sysargv: List[str]) -> None: + """ + This function will initiate the bot and start the trading loop. + :return: None + """ + logger.info('Starting Dataframe conversation') + convert_main(convert_parse_args(sysargv)) + + +if __name__ == '__main__': + main(sys.argv[1:])