adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1436 from freqtrade/remove_convert_dataframe

Browse Source 
Remove convert_backtestdata - this is not usefull anymore
This commit is contained in:
Misagh 2018-12-26 15:53:25 +01:00 committed by GitHub
commit 3411d3d5fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 0 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
scripts/convert_backtestdata.py
View File

@ -1,200 +0,0 @@
#!/usr/bin/env python3
"""
Script to display when the bot will buy a specific pair
Mandatory Cli parameters:
-p / --pair: pair to examine
Optional Cli parameters
-d / --datadir: path to pair backtest data
--timerange: specify what timerange of data to use.
-l / --live: Live, to download the latest ticker for the pair
"""
import logging
import sys
from argparse import Namespace
from os import path
import glob
import json
import re
from typing import List, Dict
import gzip
from freqtrade.arguments import Arguments
from freqtrade import misc, constants
from pandas import DataFrame
import dateutil.parser
logger = logging.getLogger('freqtrade')
def load_old_file(filename) -> (List[Dict], bool):
if not path.isfile(filename):
logger.warning("filename %s does not exist", filename)
return (None, False)
logger.debug('Loading ticker data from file %s', filename)
pairdata = None
if filename.endswith('.gz'):
logger.debug('Loading ticker data from file %s', filename)
is_zip = True
with gzip.open(filename) as tickerdata:
pairdata = json.load(tickerdata)
else:
is_zip = False
with open(filename) as tickerdata:
pairdata = json.load(tickerdata)
return (pairdata, is_zip)
def parse_old_backtest_data(ticker) -> DataFrame:
"""
Reads old backtest data
Format: "O": 8.794e-05,
"H": 8.948e-05,
"L": 8.794e-05,
"C": 8.88e-05,
"V": 991.09056638,
"T": "2017-11-26T08:50:00",
"BV": 0.0877869
"""
columns = {'C': 'close', 'V': 'volume', 'O': 'open',
'H': 'high', 'L': 'low', 'T': 'date'}
frame = DataFrame(ticker) \
.rename(columns=columns)
if 'BV' in frame:
frame.drop('BV', 1, inplace=True)
if 'date' not in frame:
logger.warning("Date not in frame - probably not a Ticker file")
return None
frame.sort_values('date', inplace=True)
return frame
def convert_dataframe(frame: DataFrame):
"""Convert dataframe to new format"""
# reorder columns:
cols = ['date', 'open', 'high', 'low', 'close', 'volume']
frame = frame[cols]
# Make sure parsing/printing data is assumed to be UTC
frame['date'] = frame['date'].apply(
lambda d: int(dateutil.parser.parse(d+'+00:00').timestamp()) * 1000)
frame['date'] = frame['date'].astype('int64')
# Convert columns one by one to preserve type.
by_column = [frame[x].values.tolist() for x in frame.columns]
return list(list(x) for x in zip(*by_column))
def convert_file(filename: str, filename_new: str) -> None:
"""Converts a file from old format to ccxt format"""
(pairdata, is_zip) = load_old_file(filename)
if pairdata and type(pairdata) is list:
if type(pairdata[0]) is list:
logger.error("pairdata for %s already in new format", filename)
return
frame = parse_old_backtest_data(pairdata)
# Convert frame to new format
if frame is not None:
frame1 = convert_dataframe(frame)
misc.file_dump_json(filename_new, frame1, is_zip)
def convert_main(args: Namespace) -> None:
"""
converts a folder given in --datadir from old to new format to support ccxt
"""
workdir = path.join(args.datadir, "")
logger.info("Workdir: %s", workdir)
for filename in glob.glob(workdir + "*.json"):
# swap currency names
ret = re.search(r'[A-Z_]{7,}', path.basename(filename))
if args.norename:
filename_new = filename
else:
if not ret:
logger.warning("file %s could not be converted, could not extract currencies",
filename)
continue
pair = ret.group(0)
currencies = pair.split("_")
if len(currencies) != 2:
logger.warning("file %s could not be converted, could not extract currencies",
filename)
continue
ret_integer = re.search(r'\d+(?=\.json)', path.basename(filename))
ret_string = re.search(r'(\d+[mhdw])(?=\.json)', path.basename(filename))
if ret_integer:
minutes = int(ret_integer.group(0))
# default to adding 'm' to end of minutes for new interval name
interval = str(minutes) + 'm'
# but check if there is a mapping between int and string also
for str_interval, minutes_interval in constants.TICKER_INTERVAL_MINUTES.items():
if minutes_interval == minutes:
interval = str_interval
break
# change order on pairs if old ticker interval found
filename_new = path.join(path.dirname(filename),
f"{currencies[1]}_{currencies[0]}-{interval}.json")
elif ret_string:
interval = ret_string.group(0)
filename_new = path.join(path.dirname(filename),
f"{currencies[0]}_{currencies[1]}-{interval}.json")
else:
logger.warning("file %s could not be converted, interval not found", filename)
continue
logger.debug("Converting and renaming %s to %s", filename, filename_new)
convert_file(filename, filename_new)
def convert_parse_args(args: List[str]) -> Namespace:
"""
Parse args passed to the script
:param args: Cli arguments
:return: args: Array with all arguments
"""
arguments = Arguments(args, 'Convert datafiles')
arguments.parser.add_argument(
'-d', '--datadir',
help='path to backtest data (default: %(default)s',
dest='datadir',
default=path.join('freqtrade', 'tests', 'testdata'),
type=str,
metavar='PATH',
)
arguments.parser.add_argument(
'-n', '--norename',
help='don''t rename files from BTC_<PAIR> to <PAIR>_BTC - '
'Note that not renaming will overwrite source files',
dest='norename',
default=False,
action='store_true'
)
return arguments.parse_args()
def main(sysargv: List[str]) -> None:
"""
This function will initiate the bot and start the trading loop.
:return: None
"""
logger.info('Starting Dataframe conversation')
convert_main(convert_parse_args(sysargv))
if __name__ == '__main__':
main(sys.argv[1:])