stable/freqtrade/optimize/__init__.py

149 lines
4.8 KiB
Python
Raw Normal View History

# pragma pylint: disable=missing-docstring
2018-03-17 21:44:47 +00:00
import gzip
import json
2018-03-25 19:37:14 +00:00
import logging
import os
2018-03-17 21:43:36 +00:00
from typing import Optional, List, Dict, Tuple
2018-01-18 07:10:48 +00:00
2018-01-11 14:49:04 +00:00
from freqtrade import misc
2018-03-17 21:44:47 +00:00
from freqtrade.exchange import get_ticker_history
2018-01-18 07:10:48 +00:00
from user_data.hyperopt_conf import hyperopt_optimize_conf
2018-03-25 19:37:14 +00:00
logger = logging.getLogger(__name__)
2018-03-17 21:43:36 +00:00
def trim_tickerlist(tickerlist: List[Dict], timerange: Tuple[Tuple, int, int]) -> List[Dict]:
stype, start, stop = timerange
2018-01-15 21:25:02 +00:00
if stype == (None, 'line'):
return tickerlist[stop:]
elif stype == ('line', None):
return tickerlist[0:start]
elif stype == ('index', 'index'):
return tickerlist[start:stop]
2018-01-26 16:41:41 +00:00
return tickerlist
2018-01-15 21:25:02 +00:00
2018-03-17 21:43:36 +00:00
def load_tickerdata_file(
datadir: str, pair: str,
ticker_interval: int,
timerange: Optional[Tuple[Tuple, int, int]] = None) -> Optional[List[Dict]]:
2018-01-05 09:20:48 +00:00
"""
Load a pair from file,
:return dict OR empty if unsuccesful
"""
path = make_testdata_path(datadir)
file = os.path.join(path, '{pair}-{ticker_interval}.json'.format(
2018-01-05 09:20:48 +00:00
pair=pair,
ticker_interval=ticker_interval,
))
gzipfile = file + '.gz'
# If the file does not exist we download it when None is returned.
# If file exists, read the file, load the json
if os.path.isfile(gzipfile):
logger.debug('Loading ticker data from file %s', gzipfile)
with gzip.open(gzipfile) as tickerdata:
pairdata = json.load(tickerdata)
elif os.path.isfile(file):
logger.debug('Loading ticker data from file %s', file)
with open(file) as tickerdata:
pairdata = json.load(tickerdata)
else:
2018-01-05 09:20:48 +00:00
return None
2018-01-15 21:25:02 +00:00
if timerange:
pairdata = trim_tickerlist(pairdata, timerange)
2018-01-05 09:20:48 +00:00
return pairdata
2018-03-17 21:43:36 +00:00
def load_data(datadir: str, ticker_interval: int,
pairs: Optional[List[str]] = None,
refresh_pairs: Optional[bool] = False,
timerange: Optional[Tuple[Tuple, int, int]] = None) -> Dict[str, List]:
"""
Loads ticker history data for the given parameters
:return: dict
"""
result = {}
2017-12-21 07:31:26 +00:00
_pairs = pairs or hyperopt_optimize_conf()['exchange']['pair_whitelist']
# If the user force the refresh of pairs
if refresh_pairs:
logger.info('Download data for all pairs and store them in %s', datadir)
download_pairs(datadir, _pairs, ticker_interval)
2017-12-21 07:31:26 +00:00
for pair in _pairs:
2018-01-15 21:25:02 +00:00
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
2018-01-05 09:20:48 +00:00
if not pairdata:
# download the tickerdata from exchange
download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
2018-01-05 09:20:48 +00:00
# and retry reading the pair
2018-01-15 21:25:02 +00:00
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
2018-01-05 09:20:48 +00:00
result[pair] = pairdata
return result
def make_testdata_path(datadir: str) -> str:
"""Return the path where testdata files are stored"""
return datadir or os.path.abspath(
os.path.join(
os.path.dirname(__file__), '..', 'tests', 'testdata'
)
)
def download_pairs(datadir, pairs: List[str], ticker_interval: int) -> bool:
"""For each pairs passed in parameters, download the ticker intervals"""
for pair in pairs:
try:
download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
except BaseException:
2018-03-02 15:22:00 +00:00
logger.info(
'Failed to download the pair: "%s", Interval: %s min',
pair,
ticker_interval
)
return False
return True
2018-01-10 11:00:00 +00:00
# FIX: 20180110, suggest rename interval to tick_interval
2018-03-29 18:15:32 +00:00
def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> None:
"""
Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
"""
path = make_testdata_path(datadir)
2018-03-02 15:22:00 +00:00
logger.info(
2018-03-29 18:15:32 +00:00
'Download the pair: "%s", Interval: %s min', pair, interval
2018-03-02 15:22:00 +00:00
)
filename = os.path.join(path, '{pair}-{interval}.json'.format(
2018-03-29 18:15:32 +00:00
pair=pair.replace("-", "_"),
interval=interval,
))
if os.path.isfile(filename):
with open(filename, "rt") as file:
data = json.load(file)
else:
data = []
2018-03-29 18:15:32 +00:00
logger.debug('Current Start: %s', data[1]['T'] if data else None)
logger.debug('Current End: %s', data[-1:][0]['T'] if data else None)
2018-03-29 18:15:32 +00:00
# Extend data with new ticker history
data.extend([
row for row in get_ticker_history(pair=pair, tick_interval=int(interval))
if row not in data
])
2018-03-29 18:15:32 +00:00
data = sorted(data, key=lambda _data: _data['T'])
logger.debug('New Start: %s', data[1]['T'])
logger.debug('New End: %s', data[-1:][0]['T'])
misc.file_dump_json(filename, data)