from freqtrade/develop
This commit is contained in:
Nullart
256
freqtrade/optimize/__init__.py
Normal file → Executable file
256
freqtrade/optimize/__init__.py
Normal file → Executable file
@@ -1,133 +1,229 @@
|
||||
# pragma pylint: disable=missing-docstring
|
||||
|
||||
import logging
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional, List, Dict
|
||||
from pandas import DataFrame
|
||||
from freqtrade.exchange import get_ticker_history
|
||||
from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf
|
||||
from freqtrade.analyze import populate_indicators, parse_ticker_dataframe
|
||||
from typing import Optional, List, Dict, Tuple, Any
|
||||
import arrow
|
||||
|
||||
from freqtrade import misc, constants, OperationalException
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.arguments import TimeRange
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_tickerdata_file(datadir, pair, ticker_interval):
|
||||
def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
|
||||
if not tickerlist:
|
||||
return tickerlist
|
||||
|
||||
start_index = 0
|
||||
stop_index = len(tickerlist)
|
||||
|
||||
if timerange.starttype == 'line':
|
||||
stop_index = timerange.startts
|
||||
if timerange.starttype == 'index':
|
||||
start_index = timerange.startts
|
||||
elif timerange.starttype == 'date':
|
||||
while (start_index < len(tickerlist) and
|
||||
tickerlist[start_index][0] < timerange.startts * 1000):
|
||||
start_index += 1
|
||||
|
||||
if timerange.stoptype == 'line':
|
||||
start_index = len(tickerlist) + timerange.stopts
|
||||
if timerange.stoptype == 'index':
|
||||
stop_index = timerange.stopts
|
||||
elif timerange.stoptype == 'date':
|
||||
while (stop_index > 0 and
|
||||
tickerlist[stop_index-1][0] > timerange.stopts * 1000):
|
||||
stop_index -= 1
|
||||
|
||||
if start_index > stop_index:
|
||||
raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect')
|
||||
|
||||
return tickerlist[start_index:stop_index]
|
||||
|
||||
|
||||
def load_tickerdata_file(
|
||||
datadir: str, pair: str,
|
||||
ticker_interval: str,
|
||||
timerange: Optional[TimeRange] = None) -> Optional[List[Dict]]:
|
||||
"""
|
||||
Load a pair from file,
|
||||
:return dict OR empty if unsuccesful
|
||||
"""
|
||||
path = make_testdata_path(datadir)
|
||||
file = '{abspath}/{pair}-{ticker_interval}.json'.format(
|
||||
abspath=path,
|
||||
pair=pair,
|
||||
ticker_interval=ticker_interval,
|
||||
)
|
||||
# The file does not exist we download it
|
||||
if not os.path.isfile(file):
|
||||
pair_s = pair.replace('/', '_')
|
||||
file = os.path.join(path, f'{pair_s}-{ticker_interval}.json')
|
||||
gzipfile = file + '.gz'
|
||||
|
||||
# If the file does not exist we download it when None is returned.
|
||||
# If file exists, read the file, load the json
|
||||
if os.path.isfile(gzipfile):
|
||||
logger.debug('Loading ticker data from file %s', gzipfile)
|
||||
with gzip.open(gzipfile) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
elif os.path.isfile(file):
|
||||
logger.debug('Loading ticker data from file %s', file)
|
||||
with open(file) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
else:
|
||||
return None
|
||||
|
||||
# Read the file, load the json
|
||||
with open(file) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
if timerange:
|
||||
pairdata = trim_tickerlist(pairdata, timerange)
|
||||
return pairdata
|
||||
|
||||
|
||||
def load_data(datadir: str, ticker_interval: int = 5, pairs: Optional[List[str]] = None,
|
||||
refresh_pairs: Optional[bool] = False) -> Dict[str, List]:
|
||||
def load_data(datadir: str,
|
||||
ticker_interval: str,
|
||||
pairs: List[str],
|
||||
refresh_pairs: Optional[bool] = False,
|
||||
exchange: Optional[Exchange] = None,
|
||||
timerange: TimeRange = TimeRange(None, None, 0, 0)) -> Dict[str, List]:
|
||||
"""
|
||||
Loads ticker history data for the given parameters
|
||||
:param ticker_interval: ticker interval in minutes
|
||||
:param pairs: list of pairs
|
||||
:return: dict
|
||||
"""
|
||||
result = {}
|
||||
|
||||
_pairs = pairs or hyperopt_optimize_conf()['exchange']['pair_whitelist']
|
||||
|
||||
# If the user force the refresh of pairs
|
||||
if refresh_pairs:
|
||||
logger.info('Download data for all pairs and store them in %s', datadir)
|
||||
download_pairs(datadir, _pairs)
|
||||
if not exchange:
|
||||
raise OperationalException("Exchange needs to be initialized when "
|
||||
"calling load_data with refresh_pairs=True")
|
||||
download_pairs(datadir, exchange, pairs, ticker_interval, timerange=timerange)
|
||||
|
||||
for pair in pairs:
|
||||
pairdata = load_tickerdata_file(datadir, pair, ticker_interval, timerange=timerange)
|
||||
if pairdata:
|
||||
result[pair] = pairdata
|
||||
else:
|
||||
logger.warning(
|
||||
'No data for pair: "%s", Interval: %s. '
|
||||
'Use --refresh-pairs-cached to download the data',
|
||||
pair,
|
||||
ticker_interval
|
||||
)
|
||||
|
||||
for pair in _pairs:
|
||||
pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
|
||||
if not pairdata:
|
||||
# download the tickerdata from exchange
|
||||
download_backtesting_testdata(datadir, pair=pair, interval=ticker_interval)
|
||||
# and retry reading the pair
|
||||
pairdata = load_tickerdata_file(datadir, pair, ticker_interval)
|
||||
result[pair] = pairdata
|
||||
return result
|
||||
|
||||
|
||||
def preprocess(tickerdata: Dict[str, List]) -> Dict[str, DataFrame]:
|
||||
"""Creates a dataframe and populates indicators for given ticker data"""
|
||||
return {pair: populate_indicators(parse_ticker_dataframe(pair_data))
|
||||
for pair, pair_data in tickerdata.items()}
|
||||
|
||||
|
||||
def make_testdata_path(datadir: str) -> str:
|
||||
"""Return the path where testdata files are stored"""
|
||||
return datadir or os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
'..', 'tests', 'testdata'))
|
||||
return datadir or os.path.abspath(
|
||||
os.path.join(
|
||||
os.path.dirname(__file__), '..', 'tests', 'testdata'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def download_pairs(datadir, pairs: List[str]) -> bool:
|
||||
"""For each pairs passed in parameters, download 1 and 5 ticker intervals"""
|
||||
def download_pairs(datadir, exchange: Exchange, pairs: List[str],
|
||||
ticker_interval: str,
|
||||
timerange: TimeRange = TimeRange(None, None, 0, 0)) -> bool:
|
||||
"""For each pairs passed in parameters, download the ticker intervals"""
|
||||
for pair in pairs:
|
||||
try:
|
||||
for interval in [1, 5]:
|
||||
download_backtesting_testdata(datadir, pair=pair, interval=interval)
|
||||
download_backtesting_testdata(datadir,
|
||||
exchange=exchange,
|
||||
pair=pair,
|
||||
tick_interval=ticker_interval,
|
||||
timerange=timerange)
|
||||
except BaseException:
|
||||
logger.info('Failed to download the pair: "{pair}", Interval: {interval} min'.format(
|
||||
pair=pair,
|
||||
interval=interval,
|
||||
))
|
||||
logger.info(
|
||||
'Failed to download the pair: "%s", Interval: %s',
|
||||
pair,
|
||||
ticker_interval
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def download_backtesting_testdata(datadir: str, pair: str, interval: int = 5) -> bool:
|
||||
def load_cached_data_for_updating(filename: str,
|
||||
tick_interval: str,
|
||||
timerange: Optional[TimeRange]) -> Tuple[
|
||||
List[Any],
|
||||
Optional[int]]:
|
||||
"""
|
||||
Download the latest 1 and 5 ticker intervals from Bittrex for the pairs passed in parameters
|
||||
Load cached data and choose what part of the data should be updated
|
||||
"""
|
||||
|
||||
since_ms = None
|
||||
|
||||
# user sets timerange, so find the start time
|
||||
if timerange:
|
||||
if timerange.starttype == 'date':
|
||||
since_ms = timerange.startts * 1000
|
||||
elif timerange.stoptype == 'line':
|
||||
num_minutes = timerange.stopts * constants.TICKER_INTERVAL_MINUTES[tick_interval]
|
||||
since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000
|
||||
|
||||
# read the cached file
|
||||
if os.path.isfile(filename):
|
||||
with open(filename, "rt") as file:
|
||||
data = json.load(file)
|
||||
# remove the last item, because we are not sure if it is correct
|
||||
# it could be fetched when the candle was incompleted
|
||||
if data:
|
||||
data.pop()
|
||||
else:
|
||||
data = []
|
||||
|
||||
if data:
|
||||
if since_ms and since_ms < data[0][0]:
|
||||
# the data is requested for earlier period than the cache has
|
||||
# so fully redownload all the data
|
||||
data = []
|
||||
else:
|
||||
# a part of the data was already downloaded, so
|
||||
# download unexist data only
|
||||
since_ms = data[-1][0] + 1
|
||||
|
||||
return (data, since_ms)
|
||||
|
||||
|
||||
def download_backtesting_testdata(datadir: str,
|
||||
exchange: Exchange,
|
||||
pair: str,
|
||||
tick_interval: str = '5m',
|
||||
timerange: Optional[TimeRange] = None) -> None:
|
||||
|
||||
"""
|
||||
Download the latest ticker intervals from the exchange for the pairs passed in parameters
|
||||
The data is downloaded starting from the last correct ticker interval data that
|
||||
esists in a cache. If timerange starts earlier than the data in the cache,
|
||||
the full data will be redownloaded
|
||||
|
||||
Based on @Rybolov work: https://github.com/rybolov/freqtrade-data
|
||||
:param pairs: list of pairs to download
|
||||
:return: bool
|
||||
:param tick_interval: ticker interval
|
||||
:param timerange: range of time to download
|
||||
:return: None
|
||||
|
||||
"""
|
||||
|
||||
path = make_testdata_path(datadir)
|
||||
logger.info('Download the pair: "{pair}", Interval: {interval} min'.format(
|
||||
pair=pair,
|
||||
interval=interval,
|
||||
))
|
||||
filepair = pair.replace("/", "_")
|
||||
filename = os.path.join(path, f'{filepair}-{tick_interval}.json')
|
||||
|
||||
filepair = pair.replace("-", "_")
|
||||
filename = os.path.join(path, '{pair}-{interval}.json'.format(
|
||||
pair=filepair,
|
||||
interval=interval,
|
||||
))
|
||||
filename = filename.replace('USDT_BTC', 'BTC_FAKEBULL')
|
||||
logger.info(
|
||||
'Download the pair: "%s", Interval: %s',
|
||||
pair,
|
||||
tick_interval
|
||||
)
|
||||
|
||||
if os.path.isfile(filename):
|
||||
with open(filename, "rt") as fp:
|
||||
data = json.load(fp)
|
||||
logger.debug("Current Start: {}".format(data[1]['T']))
|
||||
logger.debug("Current End: {}".format(data[-1:][0]['T']))
|
||||
else:
|
||||
data = []
|
||||
logger.debug("Current Start: None")
|
||||
logger.debug("Current End: None")
|
||||
data, since_ms = load_cached_data_for_updating(filename, tick_interval, timerange)
|
||||
|
||||
new_data = get_ticker_history(pair=pair, tick_interval=int(interval))
|
||||
for row in new_data:
|
||||
if row not in data:
|
||||
data.append(row)
|
||||
logger.debug("New Start: {}".format(data[1]['T']))
|
||||
logger.debug("New End: {}".format(data[-1:][0]['T']))
|
||||
data = sorted(data, key=lambda data: data['T'])
|
||||
logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None')
|
||||
logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None')
|
||||
|
||||
with open(filename, "wt") as fp:
|
||||
json.dump(data, fp)
|
||||
new_data = exchange.get_ticker_history(pair=pair, tick_interval=tick_interval,
|
||||
since_ms=since_ms)
|
||||
data.extend(new_data)
|
||||
|
||||
return True
|
||||
logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
|
||||
logger.debug("New End: %s", misc.format_ms_time(data[-1][0]))
|
||||
|
||||
misc.file_dump_json(filename, data)
|
||||
|
||||
490
freqtrade/optimize/backtesting.py
Normal file → Executable file
490
freqtrade/optimize/backtesting.py
Normal file → Executable file
@@ -1,197 +1,369 @@
|
||||
# pragma pylint: disable=missing-docstring,W0212
|
||||
# pragma pylint: disable=missing-docstring, W0212, too-many-arguments
|
||||
|
||||
"""
|
||||
This module contains the backtesting logic
|
||||
"""
|
||||
import logging
|
||||
from typing import Dict, Tuple
|
||||
import operator
|
||||
from argparse import Namespace
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
|
||||
|
||||
import arrow
|
||||
from pandas import DataFrame, Series
|
||||
from pandas import DataFrame
|
||||
from tabulate import tabulate
|
||||
|
||||
import freqtrade.misc as misc
|
||||
import freqtrade.optimize as optimize
|
||||
from freqtrade import exchange
|
||||
from freqtrade.analyze import populate_buy_trend, populate_sell_trend
|
||||
from freqtrade.exchange import Bittrex
|
||||
from freqtrade.main import min_roi_reached
|
||||
from freqtrade.optimize import preprocess
|
||||
from freqtrade import DependencyException, constants
|
||||
from freqtrade.analyze import Analyze
|
||||
from freqtrade.arguments import Arguments
|
||||
from freqtrade.configuration import Configuration
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.misc import file_dump_json
|
||||
from freqtrade.persistence import Trade
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
|
||||
class BacktestResult(NamedTuple):
|
||||
"""
|
||||
Get the maximum timeframe for the given backtest data
|
||||
:param data: dictionary with preprocessed backtesting data
|
||||
:return: tuple containing min_date, max_date
|
||||
NamedTuple Defining BacktestResults inputs.
|
||||
"""
|
||||
all_dates = Series([])
|
||||
for pair, pair_data in data.items():
|
||||
all_dates = all_dates.append(pair_data['date'])
|
||||
all_dates.sort_values(inplace=True)
|
||||
return arrow.get(all_dates.iloc[0]), arrow.get(all_dates.iloc[-1])
|
||||
pair: str
|
||||
profit_percent: float
|
||||
profit_abs: float
|
||||
open_time: datetime
|
||||
close_time: datetime
|
||||
open_index: int
|
||||
close_index: int
|
||||
trade_duration: float
|
||||
open_at_end: bool
|
||||
open_rate: float
|
||||
close_rate: float
|
||||
|
||||
|
||||
def generate_text_table(
|
||||
data: Dict[str, Dict], results: DataFrame, stake_currency, ticker_interval) -> str:
|
||||
class Backtesting(object):
|
||||
"""
|
||||
Generates and returns a text table for the given backtest data and the results dataframe
|
||||
:return: pretty printed table with tabulate as str
|
||||
Backtesting class, this class contains all the logic to run a backtest
|
||||
|
||||
To run a backtest:
|
||||
backtesting = Backtesting(config)
|
||||
backtesting.start()
|
||||
"""
|
||||
floatfmt = ('s', 'd', '.2f', '.8f', '.1f')
|
||||
tabular_data = []
|
||||
headers = ['pair', 'buy count', 'avg profit %',
|
||||
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
|
||||
for pair in data:
|
||||
result = results[results.currency == pair]
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
self.config = config
|
||||
self.analyze = Analyze(self.config)
|
||||
self.ticker_interval = self.analyze.strategy.ticker_interval
|
||||
self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe
|
||||
self.populate_buy_trend = self.analyze.populate_buy_trend
|
||||
self.populate_sell_trend = self.analyze.populate_sell_trend
|
||||
|
||||
# Reset keys for backtesting
|
||||
self.config['exchange']['key'] = ''
|
||||
self.config['exchange']['secret'] = ''
|
||||
self.config['exchange']['password'] = ''
|
||||
self.config['exchange']['uid'] = ''
|
||||
self.config['dry_run'] = True
|
||||
self.exchange = Exchange(self.config)
|
||||
self.fee = self.exchange.get_fee()
|
||||
|
||||
@staticmethod
|
||||
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
|
||||
"""
|
||||
Get the maximum timeframe for the given backtest data
|
||||
:param data: dictionary with preprocessed backtesting data
|
||||
:return: tuple containing min_date, max_date
|
||||
"""
|
||||
timeframe = [
|
||||
(arrow.get(min(frame.date)), arrow.get(max(frame.date)))
|
||||
for frame in data.values()
|
||||
]
|
||||
return min(timeframe, key=operator.itemgetter(0))[0], \
|
||||
max(timeframe, key=operator.itemgetter(1))[1]
|
||||
|
||||
def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str:
|
||||
"""
|
||||
Generates and returns a text table for the given backtest data and the results dataframe
|
||||
:return: pretty printed table with tabulate as str
|
||||
"""
|
||||
stake_currency = str(self.config.get('stake_currency'))
|
||||
|
||||
floatfmt = ('s', 'd', '.2f', '.8f', '.1f')
|
||||
tabular_data = []
|
||||
headers = ['pair', 'buy count', 'avg profit %',
|
||||
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
|
||||
for pair in data:
|
||||
result = results[results.pair == pair]
|
||||
tabular_data.append([
|
||||
pair,
|
||||
len(result.index),
|
||||
result.profit_percent.mean() * 100.0,
|
||||
result.profit_abs.sum(),
|
||||
result.trade_duration.mean(),
|
||||
len(result[result.profit_abs > 0]),
|
||||
len(result[result.profit_abs < 0])
|
||||
])
|
||||
|
||||
# Append Total
|
||||
tabular_data.append([
|
||||
pair,
|
||||
len(result.index),
|
||||
result.profit_percent.mean() * 100.0,
|
||||
result.profit_BTC.sum(),
|
||||
result.duration.mean() * ticker_interval,
|
||||
result.profit.sum(),
|
||||
result.loss.sum()
|
||||
'TOTAL',
|
||||
len(results.index),
|
||||
results.profit_percent.mean() * 100.0,
|
||||
results.profit_abs.sum(),
|
||||
results.trade_duration.mean(),
|
||||
len(results[results.profit_abs > 0]),
|
||||
len(results[results.profit_abs < 0])
|
||||
])
|
||||
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
|
||||
|
||||
# Append Total
|
||||
tabular_data.append([
|
||||
'TOTAL',
|
||||
len(results.index),
|
||||
results.profit_percent.mean() * 100.0,
|
||||
results.profit_BTC.sum(),
|
||||
results.duration.mean() * ticker_interval,
|
||||
results.profit.sum(),
|
||||
results.loss.sum()
|
||||
])
|
||||
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt)
|
||||
def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None:
|
||||
|
||||
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
|
||||
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
|
||||
t.open_rate, t.close_rate, t.open_at_end)
|
||||
for index, t in results.iterrows()]
|
||||
|
||||
def backtest(stake_amount: float, processed: Dict[str, DataFrame],
|
||||
max_open_trades: int = 0, realistic: bool = True, sell_profit_only: bool = False,
|
||||
stoploss: int = -1.00, use_sell_signal: bool = False) -> DataFrame:
|
||||
"""
|
||||
Implements backtesting functionality
|
||||
:param stake_amount: btc amount to use for each trade
|
||||
:param processed: a processed dictionary with format {pair, data}
|
||||
:param max_open_trades: maximum number of concurrent trades (default: 0, disabled)
|
||||
:param realistic: do we try to simulate realistic trades? (default: True)
|
||||
:return: DataFrame
|
||||
"""
|
||||
trades = []
|
||||
trade_count_lock: dict = {}
|
||||
exchange._API = Bittrex({'key': '', 'secret': ''})
|
||||
for pair, pair_data in processed.items():
|
||||
pair_data['buy'], pair_data['sell'] = 0, 0
|
||||
ticker = populate_sell_trend(populate_buy_trend(pair_data))
|
||||
# for each buy point
|
||||
lock_pair_until = None
|
||||
buy_subset = ticker[ticker.buy == 1][['buy', 'open', 'close', 'date', 'sell']]
|
||||
for row in buy_subset.itertuples(index=True):
|
||||
if realistic:
|
||||
if lock_pair_until is not None and row.Index <= lock_pair_until:
|
||||
continue
|
||||
if records:
|
||||
logger.info('Dumping backtest results to %s', recordfilename)
|
||||
file_dump_json(recordfilename, records)
|
||||
|
||||
def _get_sell_trade_entry(
|
||||
self, pair: str, buy_row: DataFrame,
|
||||
partial_ticker: List, trade_count_lock: Dict, args: Dict) -> Optional[BacktestResult]:
|
||||
|
||||
stake_amount = args['stake_amount']
|
||||
max_open_trades = args.get('max_open_trades', 0)
|
||||
trade = Trade(
|
||||
open_rate=buy_row.close,
|
||||
open_date=buy_row.date,
|
||||
stake_amount=stake_amount,
|
||||
amount=stake_amount / buy_row.open,
|
||||
fee_open=self.fee,
|
||||
fee_close=self.fee
|
||||
)
|
||||
|
||||
# calculate win/lose forwards from buy point
|
||||
for sell_row in partial_ticker:
|
||||
if max_open_trades > 0:
|
||||
# Check if max_open_trades has already been reached for the given date
|
||||
if not trade_count_lock.get(row.date, 0) < max_open_trades:
|
||||
continue
|
||||
# Increase trade_count_lock for every iteration
|
||||
trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1
|
||||
|
||||
if max_open_trades > 0:
|
||||
# Increase lock
|
||||
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
|
||||
buy_signal = sell_row.buy
|
||||
if self.analyze.should_sell(trade, sell_row.close, sell_row.date, buy_signal,
|
||||
sell_row.sell):
|
||||
|
||||
trade = Trade(
|
||||
open_rate=row.close,
|
||||
open_date=row.date,
|
||||
stake_amount=stake_amount,
|
||||
amount=stake_amount / row.open,
|
||||
fee=exchange.get_fee()
|
||||
return BacktestResult(pair=pair,
|
||||
profit_percent=trade.calc_profit_percent(rate=sell_row.close),
|
||||
profit_abs=trade.calc_profit(rate=sell_row.close),
|
||||
open_time=buy_row.date,
|
||||
close_time=sell_row.date,
|
||||
trade_duration=(sell_row.date - buy_row.date).seconds // 60,
|
||||
open_index=buy_row.Index,
|
||||
close_index=sell_row.Index,
|
||||
open_at_end=False,
|
||||
open_rate=buy_row.close,
|
||||
close_rate=sell_row.close
|
||||
)
|
||||
if partial_ticker:
|
||||
# no sell condition found - trade stil open at end of backtest period
|
||||
sell_row = partial_ticker[-1]
|
||||
btr = BacktestResult(pair=pair,
|
||||
profit_percent=trade.calc_profit_percent(rate=sell_row.close),
|
||||
profit_abs=trade.calc_profit(rate=sell_row.close),
|
||||
open_time=buy_row.date,
|
||||
close_time=sell_row.date,
|
||||
trade_duration=(sell_row.date - buy_row.date).seconds // 60,
|
||||
open_index=buy_row.Index,
|
||||
close_index=sell_row.Index,
|
||||
open_at_end=True,
|
||||
open_rate=buy_row.close,
|
||||
close_rate=sell_row.close
|
||||
)
|
||||
logger.debug('Force_selling still open trade %s with %s perc - %s', btr.pair,
|
||||
btr.profit_percent, btr.profit_abs)
|
||||
return btr
|
||||
return None
|
||||
|
||||
def backtest(self, args: Dict) -> DataFrame:
|
||||
"""
|
||||
Implements backtesting functionality
|
||||
|
||||
NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
|
||||
Of course try to not have ugly code. By some accessor are sometime slower than functions.
|
||||
Avoid, logging on this method
|
||||
|
||||
:param args: a dict containing:
|
||||
stake_amount: btc amount to use for each trade
|
||||
processed: a processed dictionary with format {pair, data}
|
||||
max_open_trades: maximum number of concurrent trades (default: 0, disabled)
|
||||
realistic: do we try to simulate realistic trades? (default: True)
|
||||
:return: DataFrame
|
||||
"""
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell']
|
||||
processed = args['processed']
|
||||
max_open_trades = args.get('max_open_trades', 0)
|
||||
realistic = args.get('realistic', False)
|
||||
trades = []
|
||||
trade_count_lock: Dict = {}
|
||||
for pair, pair_data in processed.items():
|
||||
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
|
||||
|
||||
ticker_data = self.populate_sell_trend(
|
||||
self.populate_buy_trend(pair_data))[headers].copy()
|
||||
|
||||
# to avoid using data from future, we buy/sell with signal from previous candle
|
||||
ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
|
||||
ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
|
||||
|
||||
ticker_data.drop(ticker_data.head(1).index, inplace=True)
|
||||
|
||||
# Convert from Pandas to list for performance reasons
|
||||
# (Looping Pandas is slow.)
|
||||
ticker = [x for x in ticker_data.itertuples()]
|
||||
|
||||
lock_pair_until = None
|
||||
for index, row in enumerate(ticker):
|
||||
if row.buy == 0 or row.sell == 1:
|
||||
continue # skip rows where no buy signal or that would immediately sell off
|
||||
|
||||
if realistic:
|
||||
if lock_pair_until is not None and row.date <= lock_pair_until:
|
||||
continue
|
||||
if max_open_trades > 0:
|
||||
# Check if max_open_trades has already been reached for the given date
|
||||
if not trade_count_lock.get(row.date, 0) < max_open_trades:
|
||||
continue
|
||||
|
||||
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
|
||||
|
||||
trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
|
||||
trade_count_lock, args)
|
||||
|
||||
if trade_entry:
|
||||
lock_pair_until = trade_entry.close_time
|
||||
trades.append(trade_entry)
|
||||
else:
|
||||
# Set lock_pair_until to end of testing period if trade could not be closed
|
||||
# This happens only if the buy-signal was with the last candle
|
||||
lock_pair_until = ticker_data.iloc[-1].date
|
||||
|
||||
return DataFrame.from_records(trades, columns=BacktestResult._fields)
|
||||
|
||||
def start(self) -> None:
|
||||
"""
|
||||
Run a backtesting end-to-end
|
||||
:return: None
|
||||
"""
|
||||
data = {}
|
||||
pairs = self.config['exchange']['pair_whitelist']
|
||||
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
|
||||
logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
|
||||
|
||||
if self.config.get('live'):
|
||||
logger.info('Downloading data for all pairs in whitelist ...')
|
||||
for pair in pairs:
|
||||
data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval)
|
||||
else:
|
||||
logger.info('Using local backtesting data (using whitelist in given config) ...')
|
||||
|
||||
timerange = Arguments.parse_timerange(None if self.config.get(
|
||||
'timerange') is None else str(self.config.get('timerange')))
|
||||
data = optimize.load_data(
|
||||
self.config['datadir'],
|
||||
pairs=pairs,
|
||||
ticker_interval=self.ticker_interval,
|
||||
refresh_pairs=self.config.get('refresh_pairs', False),
|
||||
exchange=self.exchange,
|
||||
timerange=timerange
|
||||
)
|
||||
|
||||
# calculate win/lose forwards from buy point
|
||||
sell_subset = ticker[row.Index + 1:][['close', 'date', 'sell']]
|
||||
for row2 in sell_subset.itertuples(index=True):
|
||||
if max_open_trades > 0:
|
||||
# Increase trade_count_lock for every iteration
|
||||
trade_count_lock[row2.date] = trade_count_lock.get(row2.date, 0) + 1
|
||||
if not data:
|
||||
logger.critical("No data found. Terminating.")
|
||||
return
|
||||
# Ignore max_open_trades in backtesting, except realistic flag was passed
|
||||
if self.config.get('realistic_simulation', False):
|
||||
max_open_trades = self.config['max_open_trades']
|
||||
else:
|
||||
logger.info('Ignoring max_open_trades (realistic_simulation not set) ...')
|
||||
max_open_trades = 0
|
||||
|
||||
current_profit_percent = trade.calc_profit_percent(rate=row2.close)
|
||||
if (sell_profit_only and current_profit_percent < 0):
|
||||
continue
|
||||
if min_roi_reached(trade, row2.close, row2.date) or \
|
||||
(row2.sell == 1 and use_sell_signal) or \
|
||||
current_profit_percent <= stoploss:
|
||||
current_profit_btc = trade.calc_profit(rate=row2.close)
|
||||
lock_pair_until = row2.Index
|
||||
preprocessed = self.tickerdata_to_dataframe(data)
|
||||
|
||||
trades.append(
|
||||
(
|
||||
pair,
|
||||
current_profit_percent,
|
||||
current_profit_btc,
|
||||
row2.Index - row.Index,
|
||||
current_profit_btc > 0,
|
||||
current_profit_btc < 0
|
||||
)
|
||||
)
|
||||
break
|
||||
labels = ['currency', 'profit_percent', 'profit_BTC', 'duration', 'profit', 'loss']
|
||||
return DataFrame.from_records(trades, columns=labels)
|
||||
# Print timeframe
|
||||
min_date, max_date = self.get_timeframe(preprocessed)
|
||||
logger.info(
|
||||
'Measuring data from %s up to %s (%s days)..',
|
||||
min_date.isoformat(),
|
||||
max_date.isoformat(),
|
||||
(max_date - min_date).days
|
||||
)
|
||||
|
||||
# Execute backtest and print results
|
||||
results = self.backtest(
|
||||
{
|
||||
'stake_amount': self.config.get('stake_amount'),
|
||||
'processed': preprocessed,
|
||||
'max_open_trades': max_open_trades,
|
||||
'realistic': self.config.get('realistic_simulation', False),
|
||||
}
|
||||
)
|
||||
|
||||
if self.config.get('export', False):
|
||||
self._store_backtest_result(self.config.get('exportfilename'), results)
|
||||
|
||||
logger.info(
|
||||
'\n======================================== '
|
||||
'BACKTESTING REPORT'
|
||||
' =========================================\n'
|
||||
'%s',
|
||||
self._generate_text_table(
|
||||
data,
|
||||
results
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
'\n====================================== '
|
||||
'LEFT OPEN TRADES REPORT'
|
||||
' ======================================\n'
|
||||
'%s',
|
||||
self._generate_text_table(
|
||||
data,
|
||||
results.loc[results.open_at_end]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def start(args):
|
||||
# Initialize logger
|
||||
logging.basicConfig(
|
||||
level=args.loglevel,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
)
|
||||
def setup_configuration(args: Namespace) -> Dict[str, Any]:
|
||||
"""
|
||||
Prepare the configuration for the backtesting
|
||||
:param args: Cli args from Arguments()
|
||||
:return: Configuration
|
||||
"""
|
||||
configuration = Configuration(args)
|
||||
config = configuration.get_config()
|
||||
|
||||
exchange._API = Bittrex({'key': '', 'secret': ''})
|
||||
# Ensure we do not use Exchange credentials
|
||||
config['exchange']['key'] = ''
|
||||
config['exchange']['secret'] = ''
|
||||
|
||||
logger.info('Using config: %s ...', args.config)
|
||||
config = misc.load_config(args.config)
|
||||
if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
|
||||
raise DependencyException('stake amount could not be "%s" for backtesting' %
|
||||
constants.UNLIMITED_STAKE_AMOUNT)
|
||||
|
||||
logger.info('Using ticker_interval: %s ...', args.ticker_interval)
|
||||
return config
|
||||
|
||||
data = {}
|
||||
pairs = config['exchange']['pair_whitelist']
|
||||
if args.live:
|
||||
logger.info('Downloading data for all pairs in whitelist ...')
|
||||
for pair in pairs:
|
||||
data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
|
||||
else:
|
||||
logger.info('Using local backtesting data (using whitelist in given config) ...')
|
||||
data = optimize.load_data(args.datadir, pairs=pairs, ticker_interval=args.ticker_interval,
|
||||
refresh_pairs=args.refresh_pairs)
|
||||
|
||||
logger.info('Using stake_currency: %s ...', config['stake_currency'])
|
||||
logger.info('Using stake_amount: %s ...', config['stake_amount'])
|
||||
def start(args: Namespace) -> None:
|
||||
"""
|
||||
Start Backtesting script
|
||||
:param args: Cli args from Arguments()
|
||||
:return: None
|
||||
"""
|
||||
# Initialize configuration
|
||||
config = setup_configuration(args)
|
||||
logger.info('Starting freqtrade in Backtesting mode')
|
||||
|
||||
max_open_trades = 0
|
||||
if args.realistic_simulation:
|
||||
logger.info('Using max_open_trades: %s ...', config['max_open_trades'])
|
||||
max_open_trades = config['max_open_trades']
|
||||
|
||||
# Monkey patch config
|
||||
from freqtrade import main
|
||||
main._CONF = config
|
||||
|
||||
preprocessed = preprocess(data)
|
||||
# Print timeframe
|
||||
min_date, max_date = get_timeframe(preprocessed)
|
||||
logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat())
|
||||
|
||||
# Execute backtest and print results
|
||||
results = backtest(
|
||||
stake_amount=config['stake_amount'],
|
||||
processed=preprocessed,
|
||||
max_open_trades=max_open_trades,
|
||||
realistic=args.realistic_simulation,
|
||||
sell_profit_only=config.get('experimental', {}).get('sell_profit_only', False),
|
||||
stoploss=config.get('stoploss'),
|
||||
use_sell_signal=config.get('experimental', {}).get('use_sell_signal', False)
|
||||
)
|
||||
logger.info(
|
||||
'\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa
|
||||
generate_text_table(data, results, config['stake_currency'], args.ticker_interval)
|
||||
)
|
||||
# Initialize backtesting object
|
||||
backtesting = Backtesting(config)
|
||||
backtesting.start()
|
||||
|
||||
639
freqtrade/optimize/hyperopt.py
Normal file → Executable file
639
freqtrade/optimize/hyperopt.py
Normal file → Executable file
@@ -1,318 +1,407 @@
|
||||
# pragma pylint: disable=missing-docstring,W0212,W0603
|
||||
# pragma pylint: disable=too-many-instance-attributes, pointless-string-statement
|
||||
|
||||
"""
|
||||
This module contains the hyperopt logic
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import pickle
|
||||
import signal
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
from argparse import Namespace
|
||||
from functools import reduce
|
||||
from math import exp
|
||||
from operator import itemgetter
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe
|
||||
from hyperopt.mongoexp import MongoTrials
|
||||
import talib.abstract as ta
|
||||
from pandas import DataFrame
|
||||
from sklearn.externals.joblib import Parallel, delayed, dump, load
|
||||
from skopt import Optimizer
|
||||
from skopt.space import Categorical, Dimension, Integer, Real
|
||||
|
||||
from freqtrade import main # noqa
|
||||
from freqtrade import exchange, optimize
|
||||
from freqtrade.exchange import Bittrex
|
||||
from freqtrade.misc import load_config
|
||||
from freqtrade.optimize.backtesting import backtest
|
||||
from freqtrade.optimize.hyperopt_conf import hyperopt_optimize_conf
|
||||
from freqtrade.vendor.qtpylib.indicators import crossed_above
|
||||
|
||||
# Remove noisy log messages
|
||||
logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING)
|
||||
logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
|
||||
import freqtrade.vendor.qtpylib.indicators as qtpylib
|
||||
from freqtrade.arguments import Arguments
|
||||
from freqtrade.configuration import Configuration
|
||||
from freqtrade.optimize import load_data
|
||||
from freqtrade.optimize.backtesting import Backtesting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
|
||||
TARGET_TRADES = 1100
|
||||
TOTAL_TRIES = 0
|
||||
_CURRENT_TRIES = 0
|
||||
CURRENT_BEST_LOSS = 100
|
||||
|
||||
# max average trade duration in minutes
|
||||
# if eval ends with higher value, we consider it a failed eval
|
||||
MAX_ACCEPTED_TRADE_DURATION = 240
|
||||
|
||||
# this is expexted avg profit * expected trade count
|
||||
# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85
|
||||
EXPECTED_MAX_PROFIT = 3.85
|
||||
|
||||
# Configuration and data used by hyperopt
|
||||
PROCESSED = None # optimize.preprocess(optimize.load_data())
|
||||
OPTIMIZE_CONFIG = hyperopt_optimize_conf()
|
||||
|
||||
# Hyperopt Trials
|
||||
TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle')
|
||||
TRIALS = Trials()
|
||||
|
||||
# Monkey patch config
|
||||
from freqtrade import main # noqa
|
||||
main._CONF = OPTIMIZE_CONFIG
|
||||
MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization
|
||||
TICKERDATA_PICKLE = os.path.join('user_data', 'hyperopt_tickerdata.pkl')
|
||||
|
||||
|
||||
SPACE = {
|
||||
'mfi': hp.choice('mfi', [
|
||||
{'enabled': False},
|
||||
{'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)}
|
||||
]),
|
||||
'fastd': hp.choice('fastd', [
|
||||
{'enabled': False},
|
||||
{'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)}
|
||||
]),
|
||||
'adx': hp.choice('adx', [
|
||||
{'enabled': False},
|
||||
{'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)}
|
||||
]),
|
||||
'rsi': hp.choice('rsi', [
|
||||
{'enabled': False},
|
||||
{'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)}
|
||||
]),
|
||||
'uptrend_long_ema': hp.choice('uptrend_long_ema', [
|
||||
{'enabled': False},
|
||||
{'enabled': True}
|
||||
]),
|
||||
'uptrend_short_ema': hp.choice('uptrend_short_ema', [
|
||||
{'enabled': False},
|
||||
{'enabled': True}
|
||||
]),
|
||||
'over_sar': hp.choice('over_sar', [
|
||||
{'enabled': False},
|
||||
{'enabled': True}
|
||||
]),
|
||||
'green_candle': hp.choice('green_candle', [
|
||||
{'enabled': False},
|
||||
{'enabled': True}
|
||||
]),
|
||||
'uptrend_sma': hp.choice('uptrend_sma', [
|
||||
{'enabled': False},
|
||||
{'enabled': True}
|
||||
]),
|
||||
'trigger': hp.choice('trigger', [
|
||||
{'type': 'lower_bb'},
|
||||
{'type': 'faststoch10'},
|
||||
{'type': 'ao_cross_zero'},
|
||||
{'type': 'ema5_cross_ema10'},
|
||||
{'type': 'macd_cross_signal'},
|
||||
{'type': 'sar_reversal'},
|
||||
{'type': 'stochf_cross'},
|
||||
{'type': 'ht_sine'},
|
||||
]),
|
||||
'stoploss': hp.uniform('stoploss', -0.5, -0.02),
|
||||
}
|
||||
class Hyperopt(Backtesting):
|
||||
"""
|
||||
Hyperopt class, this class contains all the logic to run a hyperopt simulation
|
||||
|
||||
To run a backtest:
|
||||
hyperopt = Hyperopt(config)
|
||||
hyperopt.start()
|
||||
"""
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
super().__init__(config)
|
||||
# set TARGET_TRADES to suit your number concurrent trades so its realistic
|
||||
# to the number of days
|
||||
self.target_trades = 600
|
||||
self.total_tries = config.get('epochs', 0)
|
||||
self.current_best_loss = 100
|
||||
|
||||
def save_trials(trials, trials_path=TRIALS_FILE):
|
||||
"""Save hyperopt trials to file"""
|
||||
logger.info('Saving Trials to \'{}\''.format(trials_path))
|
||||
pickle.dump(trials, open(trials_path, 'wb'))
|
||||
# max average trade duration in minutes
|
||||
# if eval ends with higher value, we consider it a failed eval
|
||||
self.max_accepted_trade_duration = 300
|
||||
|
||||
# this is expexted avg profit * expected trade count
|
||||
# for example 3.5%, 1100 trades, self.expected_max_profit = 3.85
|
||||
# check that the reported Σ% values do not exceed this!
|
||||
self.expected_max_profit = 3.0
|
||||
|
||||
def read_trials(trials_path=TRIALS_FILE):
|
||||
"""Read hyperopt trials file"""
|
||||
logger.info('Reading Trials from \'{}\''.format(trials_path))
|
||||
trials = pickle.load(open(trials_path, 'rb'))
|
||||
os.remove(trials_path)
|
||||
return trials
|
||||
# Previous evaluations
|
||||
self.trials_file = os.path.join('user_data', 'hyperopt_results.pickle')
|
||||
self.trials: List = []
|
||||
|
||||
def get_args(self, params):
|
||||
dimensions = self.hyperopt_space()
|
||||
# Ensure the number of dimensions match
|
||||
# the number of parameters in the list x.
|
||||
if len(params) != len(dimensions):
|
||||
raise ValueError('Mismatch in number of search-space dimensions. '
|
||||
f'len(dimensions)=={len(dimensions)} and len(x)=={len(params)}')
|
||||
|
||||
def log_trials_result(trials):
|
||||
vals = json.dumps(trials.best_trial['misc']['vals'], indent=4)
|
||||
results = trials.best_trial['result']['result']
|
||||
logger.info('Best result:\n%s\nwith values:\n%s', results, vals)
|
||||
# Create a dict where the keys are the names of the dimensions
|
||||
# and the values are taken from the list of parameters x.
|
||||
arg_dict = {dim.name: value for dim, value in zip(dimensions, params)}
|
||||
return arg_dict
|
||||
|
||||
|
||||
def log_results(results):
|
||||
""" log results if it is better than any previous evaluation """
|
||||
global CURRENT_BEST_LOSS
|
||||
|
||||
if results['loss'] < CURRENT_BEST_LOSS:
|
||||
CURRENT_BEST_LOSS = results['loss']
|
||||
logger.info('{:5d}/{}: {}'.format(
|
||||
results['current_tries'],
|
||||
results['total_tries'],
|
||||
results['result']))
|
||||
else:
|
||||
print('.', end='')
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def calculate_loss(total_profit: float, trade_count: int, trade_duration: float):
|
||||
""" objective function, returns smaller number for more optimal results """
|
||||
trade_loss = 1 - 0.35 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.2)
|
||||
profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT)
|
||||
duration_loss = min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1)
|
||||
return trade_loss + profit_loss + duration_loss
|
||||
|
||||
|
||||
def optimizer(params):
|
||||
global _CURRENT_TRIES
|
||||
|
||||
from freqtrade.optimize import backtesting
|
||||
backtesting.populate_buy_trend = buy_strategy_generator(params)
|
||||
|
||||
results = backtest(OPTIMIZE_CONFIG['stake_amount'], PROCESSED, stoploss=params['stoploss'])
|
||||
result_explanation = format_results(results)
|
||||
|
||||
total_profit = results.profit_percent.sum()
|
||||
trade_count = len(results.index)
|
||||
trade_duration = results.duration.mean() * 5
|
||||
|
||||
if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION:
|
||||
print('.', end='')
|
||||
return {
|
||||
'status': STATUS_FAIL,
|
||||
'loss': float('inf')
|
||||
}
|
||||
|
||||
loss = calculate_loss(total_profit, trade_count, trade_duration)
|
||||
|
||||
_CURRENT_TRIES += 1
|
||||
|
||||
log_results({
|
||||
'loss': loss,
|
||||
'current_tries': _CURRENT_TRIES,
|
||||
'total_tries': TOTAL_TRIES,
|
||||
'result': result_explanation,
|
||||
})
|
||||
|
||||
return {
|
||||
'loss': loss,
|
||||
'status': STATUS_OK,
|
||||
'result': result_explanation,
|
||||
}
|
||||
|
||||
|
||||
def format_results(results: DataFrame):
|
||||
return ('{:6d} trades. Avg profit {: 5.2f}%. '
|
||||
'Total profit {: 11.8f} BTC. Avg duration {:5.1f} mins.').format(
|
||||
len(results.index),
|
||||
results.profit_percent.mean() * 100.0,
|
||||
results.profit_BTC.sum(),
|
||||
results.duration.mean() * 5,
|
||||
)
|
||||
|
||||
|
||||
def buy_strategy_generator(params):
|
||||
def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
|
||||
conditions = []
|
||||
# GUARDS AND TRENDS
|
||||
if params['uptrend_long_ema']['enabled']:
|
||||
conditions.append(dataframe['ema50'] > dataframe['ema100'])
|
||||
if params['uptrend_short_ema']['enabled']:
|
||||
conditions.append(dataframe['ema5'] > dataframe['ema10'])
|
||||
if params['mfi']['enabled']:
|
||||
conditions.append(dataframe['mfi'] < params['mfi']['value'])
|
||||
if params['fastd']['enabled']:
|
||||
conditions.append(dataframe['fastd'] < params['fastd']['value'])
|
||||
if params['adx']['enabled']:
|
||||
conditions.append(dataframe['adx'] > params['adx']['value'])
|
||||
if params['rsi']['enabled']:
|
||||
conditions.append(dataframe['rsi'] < params['rsi']['value'])
|
||||
if params['over_sar']['enabled']:
|
||||
conditions.append(dataframe['close'] > dataframe['sar'])
|
||||
if params['green_candle']['enabled']:
|
||||
conditions.append(dataframe['close'] > dataframe['open'])
|
||||
if params['uptrend_sma']['enabled']:
|
||||
prevsma = dataframe['sma'].shift(1)
|
||||
conditions.append(dataframe['sma'] > prevsma)
|
||||
|
||||
# TRIGGERS
|
||||
triggers = {
|
||||
'lower_bb': dataframe['tema'] <= dataframe['blower'],
|
||||
'faststoch10': (crossed_above(dataframe['fastd'], 10.0)),
|
||||
'ao_cross_zero': (crossed_above(dataframe['ao'], 0.0)),
|
||||
'ema5_cross_ema10': (crossed_above(dataframe['ema5'], dataframe['ema10'])),
|
||||
'macd_cross_signal': (crossed_above(dataframe['macd'], dataframe['macdsignal'])),
|
||||
'sar_reversal': (crossed_above(dataframe['close'], dataframe['sar'])),
|
||||
'stochf_cross': (crossed_above(dataframe['fastk'], dataframe['fastd'])),
|
||||
'ht_sine': (crossed_above(dataframe['htleadsine'], dataframe['htsine'])),
|
||||
}
|
||||
conditions.append(triggers.get(params['trigger']['type']))
|
||||
|
||||
dataframe.loc[
|
||||
reduce(lambda x, y: x & y, conditions),
|
||||
'buy'] = 1
|
||||
@staticmethod
|
||||
def populate_indicators(dataframe: DataFrame) -> DataFrame:
|
||||
dataframe['adx'] = ta.ADX(dataframe)
|
||||
macd = ta.MACD(dataframe)
|
||||
dataframe['macd'] = macd['macd']
|
||||
dataframe['macdsignal'] = macd['macdsignal']
|
||||
dataframe['mfi'] = ta.MFI(dataframe)
|
||||
dataframe['rsi'] = ta.RSI(dataframe)
|
||||
stoch_fast = ta.STOCHF(dataframe)
|
||||
dataframe['fastd'] = stoch_fast['fastd']
|
||||
dataframe['minus_di'] = ta.MINUS_DI(dataframe)
|
||||
# Bollinger bands
|
||||
bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
|
||||
dataframe['bb_lowerband'] = bollinger['lower']
|
||||
dataframe['sar'] = ta.SAR(dataframe)
|
||||
|
||||
return dataframe
|
||||
return populate_buy_trend
|
||||
|
||||
def save_trials(self) -> None:
|
||||
"""
|
||||
Save hyperopt trials to file
|
||||
"""
|
||||
if self.trials:
|
||||
logger.info('Saving %d evaluations to \'%s\'', len(self.trials), self.trials_file)
|
||||
dump(self.trials, self.trials_file)
|
||||
|
||||
def start(args):
|
||||
global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES
|
||||
def read_trials(self) -> List:
|
||||
"""
|
||||
Read hyperopt trials file
|
||||
"""
|
||||
logger.info('Reading Trials from \'%s\'', self.trials_file)
|
||||
trials = load(self.trials_file)
|
||||
os.remove(self.trials_file)
|
||||
return trials
|
||||
|
||||
TOTAL_TRIES = args.epochs
|
||||
def log_trials_result(self) -> None:
|
||||
"""
|
||||
Display Best hyperopt result
|
||||
"""
|
||||
results = sorted(self.trials, key=itemgetter('loss'))
|
||||
best_result = results[0]
|
||||
logger.info(
|
||||
'Best result:\n%s\nwith values:\n%s',
|
||||
best_result['result'],
|
||||
best_result['params']
|
||||
)
|
||||
if 'roi_t1' in best_result['params']:
|
||||
logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params']))
|
||||
|
||||
exchange._API = Bittrex({'key': '', 'secret': ''})
|
||||
def log_results(self, results) -> None:
|
||||
"""
|
||||
Log results if it is better than any previous evaluation
|
||||
"""
|
||||
if results['loss'] < self.current_best_loss:
|
||||
current = results['current_tries']
|
||||
total = results['total_tries']
|
||||
res = results['result']
|
||||
loss = results['loss']
|
||||
self.current_best_loss = results['loss']
|
||||
log_msg = f'\n{current:5d}/{total}: {res}. Loss {loss:.5f}'
|
||||
print(log_msg)
|
||||
else:
|
||||
print('.', end='')
|
||||
sys.stdout.flush()
|
||||
|
||||
# Initialize logger
|
||||
logging.basicConfig(
|
||||
level=args.loglevel,
|
||||
format='\n%(message)s',
|
||||
)
|
||||
def calculate_loss(self, total_profit: float, trade_count: int, trade_duration: float) -> float:
|
||||
"""
|
||||
Objective function, returns smaller number for more optimal results
|
||||
"""
|
||||
trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8)
|
||||
profit_loss = max(0, 1 - total_profit / self.expected_max_profit)
|
||||
duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1)
|
||||
result = trade_loss + profit_loss + duration_loss
|
||||
return result
|
||||
|
||||
logger.info('Using config: %s ...', args.config)
|
||||
config = load_config(args.config)
|
||||
pairs = config['exchange']['pair_whitelist']
|
||||
PROCESSED = optimize.preprocess(optimize.load_data(
|
||||
args.datadir, pairs=pairs, ticker_interval=args.ticker_interval))
|
||||
@staticmethod
|
||||
def generate_roi_table(params: Dict) -> Dict[int, float]:
|
||||
"""
|
||||
Generate the ROI table thqt will be used by Hyperopt
|
||||
"""
|
||||
roi_table = {}
|
||||
roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
|
||||
roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
|
||||
roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
|
||||
roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
|
||||
|
||||
if args.mongodb:
|
||||
logger.info('Using mongodb ...')
|
||||
logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')
|
||||
return roi_table
|
||||
|
||||
db_name = 'freqtrade_hyperopt'
|
||||
TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
|
||||
else:
|
||||
logger.info('Preparing Trials..')
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
# read trials file if we have one
|
||||
if os.path.exists(TRIALS_FILE):
|
||||
TRIALS = read_trials()
|
||||
@staticmethod
|
||||
def roi_space() -> List[Dimension]:
|
||||
"""
|
||||
Values to search for each ROI steps
|
||||
"""
|
||||
return [
|
||||
Integer(10, 120, name='roi_t1'),
|
||||
Integer(10, 60, name='roi_t2'),
|
||||
Integer(10, 40, name='roi_t3'),
|
||||
Real(0.01, 0.04, name='roi_p1'),
|
||||
Real(0.01, 0.07, name='roi_p2'),
|
||||
Real(0.01, 0.20, name='roi_p3'),
|
||||
]
|
||||
|
||||
_CURRENT_TRIES = len(TRIALS.results)
|
||||
TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES
|
||||
logger.info(
|
||||
'Continuing with trials. Current: {}, Total: {}'
|
||||
.format(_CURRENT_TRIES, TOTAL_TRIES))
|
||||
@staticmethod
|
||||
def stoploss_space() -> List[Dimension]:
|
||||
"""
|
||||
Stoploss search space
|
||||
"""
|
||||
return [
|
||||
Real(-0.5, -0.02, name='stoploss'),
|
||||
]
|
||||
|
||||
try:
|
||||
best_parameters = fmin(
|
||||
fn=optimizer,
|
||||
space=SPACE,
|
||||
algo=tpe.suggest,
|
||||
max_evals=TOTAL_TRIES,
|
||||
trials=TRIALS
|
||||
@staticmethod
|
||||
def indicator_space() -> List[Dimension]:
|
||||
"""
|
||||
Define your Hyperopt space for searching strategy parameters
|
||||
"""
|
||||
return [
|
||||
Integer(10, 25, name='mfi-value'),
|
||||
Integer(15, 45, name='fastd-value'),
|
||||
Integer(20, 50, name='adx-value'),
|
||||
Integer(20, 40, name='rsi-value'),
|
||||
Categorical([True, False], name='mfi-enabled'),
|
||||
Categorical([True, False], name='fastd-enabled'),
|
||||
Categorical([True, False], name='adx-enabled'),
|
||||
Categorical([True, False], name='rsi-enabled'),
|
||||
Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger')
|
||||
]
|
||||
|
||||
def has_space(self, space: str) -> bool:
|
||||
"""
|
||||
Tell if a space value is contained in the configuration
|
||||
"""
|
||||
if space in self.config['spaces'] or 'all' in self.config['spaces']:
|
||||
return True
|
||||
return False
|
||||
|
||||
def hyperopt_space(self) -> List[Dimension]:
|
||||
"""
|
||||
Return the space to use during Hyperopt
|
||||
"""
|
||||
spaces: List[Dimension] = []
|
||||
if self.has_space('buy'):
|
||||
spaces += Hyperopt.indicator_space()
|
||||
if self.has_space('roi'):
|
||||
spaces += Hyperopt.roi_space()
|
||||
if self.has_space('stoploss'):
|
||||
spaces += Hyperopt.stoploss_space()
|
||||
return spaces
|
||||
|
||||
@staticmethod
|
||||
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
|
||||
"""
|
||||
Define the buy strategy parameters to be used by hyperopt
|
||||
"""
|
||||
def populate_buy_trend(dataframe: DataFrame) -> DataFrame:
|
||||
"""
|
||||
Buy strategy Hyperopt will build and use
|
||||
"""
|
||||
conditions = []
|
||||
# GUARDS AND TRENDS
|
||||
if 'mfi-enabled' in params and params['mfi-enabled']:
|
||||
conditions.append(dataframe['mfi'] < params['mfi-value'])
|
||||
if 'fastd-enabled' in params and params['fastd-enabled']:
|
||||
conditions.append(dataframe['fastd'] < params['fastd-value'])
|
||||
if 'adx-enabled' in params and params['adx-enabled']:
|
||||
conditions.append(dataframe['adx'] > params['adx-value'])
|
||||
if 'rsi-enabled' in params and params['rsi-enabled']:
|
||||
conditions.append(dataframe['rsi'] < params['rsi-value'])
|
||||
|
||||
# TRIGGERS
|
||||
if params['trigger'] == 'bb_lower':
|
||||
conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
|
||||
if params['trigger'] == 'macd_cross_signal':
|
||||
conditions.append(qtpylib.crossed_above(
|
||||
dataframe['macd'], dataframe['macdsignal']
|
||||
))
|
||||
if params['trigger'] == 'sar_reversal':
|
||||
conditions.append(qtpylib.crossed_above(
|
||||
dataframe['close'], dataframe['sar']
|
||||
))
|
||||
|
||||
dataframe.loc[
|
||||
reduce(lambda x, y: x & y, conditions),
|
||||
'buy'] = 1
|
||||
|
||||
return dataframe
|
||||
|
||||
return populate_buy_trend
|
||||
|
||||
def generate_optimizer(self, _params) -> Dict:
|
||||
params = self.get_args(_params)
|
||||
|
||||
if self.has_space('roi'):
|
||||
self.analyze.strategy.minimal_roi = self.generate_roi_table(params)
|
||||
|
||||
if self.has_space('buy'):
|
||||
self.populate_buy_trend = self.buy_strategy_generator(params)
|
||||
|
||||
if self.has_space('stoploss'):
|
||||
self.analyze.strategy.stoploss = params['stoploss']
|
||||
|
||||
processed = load(TICKERDATA_PICKLE)
|
||||
results = self.backtest(
|
||||
{
|
||||
'stake_amount': self.config['stake_amount'],
|
||||
'processed': processed,
|
||||
'realistic': self.config.get('realistic_simulation', False),
|
||||
}
|
||||
)
|
||||
result_explanation = self.format_results(results)
|
||||
|
||||
total_profit = results.profit_percent.sum()
|
||||
trade_count = len(results.index)
|
||||
trade_duration = results.trade_duration.mean()
|
||||
|
||||
if trade_count == 0:
|
||||
return {
|
||||
'loss': MAX_LOSS,
|
||||
'params': params,
|
||||
'result': result_explanation,
|
||||
}
|
||||
|
||||
loss = self.calculate_loss(total_profit, trade_count, trade_duration)
|
||||
|
||||
return {
|
||||
'loss': loss,
|
||||
'params': params,
|
||||
'result': result_explanation,
|
||||
}
|
||||
|
||||
def format_results(self, results: DataFrame) -> str:
|
||||
"""
|
||||
Return the format result in a string
|
||||
"""
|
||||
trades = len(results.index)
|
||||
avg_profit = results.profit_percent.mean() * 100.0
|
||||
total_profit = results.profit_abs.sum()
|
||||
stake_cur = self.config['stake_currency']
|
||||
profit = results.profit_percent.sum()
|
||||
duration = results.trade_duration.mean()
|
||||
|
||||
return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. '
|
||||
f'Total profit {total_profit: 11.8f} {stake_cur} '
|
||||
f'({profit:.4f}Σ%). Avg duration {duration:5.1f} mins.')
|
||||
|
||||
def get_optimizer(self, cpu_count) -> Optimizer:
|
||||
return Optimizer(
|
||||
self.hyperopt_space(),
|
||||
base_estimator="ET",
|
||||
acq_optimizer="auto",
|
||||
n_initial_points=30,
|
||||
acq_optimizer_kwargs={'n_jobs': cpu_count}
|
||||
)
|
||||
|
||||
results = sorted(TRIALS.results, key=itemgetter('loss'))
|
||||
best_result = results[0]['result']
|
||||
def run_optimizer_parallel(self, parallel, asked) -> List:
|
||||
return parallel(delayed(self.generate_optimizer)(v) for v in asked)
|
||||
|
||||
except ValueError:
|
||||
best_parameters = {}
|
||||
best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \
|
||||
'try with more epochs (param: -e).'
|
||||
def load_previous_results(self):
|
||||
""" read trials file if we have one """
|
||||
if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0:
|
||||
self.trials = self.read_trials()
|
||||
logger.info(
|
||||
'Loaded %d previous evaluations from disk.',
|
||||
len(self.trials)
|
||||
)
|
||||
|
||||
# Improve best parameter logging display
|
||||
if best_parameters:
|
||||
best_parameters = space_eval(SPACE, best_parameters)
|
||||
def start(self) -> None:
|
||||
timerange = Arguments.parse_timerange(None if self.config.get(
|
||||
'timerange') is None else str(self.config.get('timerange')))
|
||||
data = load_data(
|
||||
datadir=str(self.config.get('datadir')),
|
||||
pairs=self.config['exchange']['pair_whitelist'],
|
||||
ticker_interval=self.ticker_interval,
|
||||
timerange=timerange
|
||||
)
|
||||
|
||||
logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4))
|
||||
logger.info('Best Result:\n%s', best_result)
|
||||
if self.has_space('buy'):
|
||||
self.analyze.populate_indicators = Hyperopt.populate_indicators # type: ignore
|
||||
dump(self.tickerdata_to_dataframe(data), TICKERDATA_PICKLE)
|
||||
self.exchange = None # type: ignore
|
||||
self.load_previous_results()
|
||||
|
||||
# Store trials result to file to resume next time
|
||||
save_trials(TRIALS)
|
||||
cpus = multiprocessing.cpu_count()
|
||||
logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!')
|
||||
|
||||
opt = self.get_optimizer(cpus)
|
||||
EVALS = max(self.total_tries//cpus, 1)
|
||||
try:
|
||||
with Parallel(n_jobs=cpus) as parallel:
|
||||
for i in range(EVALS):
|
||||
asked = opt.ask(n_points=cpus)
|
||||
f_val = self.run_optimizer_parallel(parallel, asked)
|
||||
opt.tell(asked, [i['loss'] for i in f_val])
|
||||
|
||||
self.trials += f_val
|
||||
for j in range(cpus):
|
||||
self.log_results({
|
||||
'loss': f_val[j]['loss'],
|
||||
'current_tries': i * cpus + j,
|
||||
'total_tries': self.total_tries,
|
||||
'result': f_val[j]['result'],
|
||||
})
|
||||
except KeyboardInterrupt:
|
||||
print('User interrupted..')
|
||||
|
||||
self.save_trials()
|
||||
self.log_trials_result()
|
||||
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
"""Hyperopt SIGINT handler"""
|
||||
logger.info('Hyperopt received {}'.format(signal.Signals(sig).name))
|
||||
def start(args: Namespace) -> None:
|
||||
"""
|
||||
Start Backtesting script
|
||||
:param args: Cli args from Arguments()
|
||||
:return: None
|
||||
"""
|
||||
|
||||
save_trials(TRIALS)
|
||||
log_trials_result(TRIALS)
|
||||
sys.exit(0)
|
||||
# Remove noisy log messages
|
||||
logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
|
||||
|
||||
# Initialize configuration
|
||||
# Monkey patch the configuration with hyperopt_conf.py
|
||||
configuration = Configuration(args)
|
||||
logger.info('Starting freqtrade in Hyperopt mode')
|
||||
config = configuration.load_config()
|
||||
|
||||
config['exchange']['key'] = ''
|
||||
config['exchange']['secret'] = ''
|
||||
|
||||
# Initialize backtesting object
|
||||
hyperopt = Hyperopt(config)
|
||||
hyperopt.start()
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
"""
|
||||
File that contains the configuration for Hyperopt
|
||||
"""
|
||||
|
||||
|
||||
def hyperopt_optimize_conf() -> dict:
|
||||
"""
|
||||
This function is used to define which parameters Hyperopt must used.
|
||||
The "pair_whitelist" is only used is your are using Hyperopt with MongoDB,
|
||||
without MongoDB, Hyperopt will use the pair your have set in your config file.
|
||||
:return:
|
||||
"""
|
||||
return {
|
||||
'max_open_trades': 3,
|
||||
'stake_currency': 'BTC',
|
||||
'stake_amount': 0.01,
|
||||
"minimal_roi": {
|
||||
'40': 0.0,
|
||||
'30': 0.01,
|
||||
'20': 0.02,
|
||||
'0': 0.04,
|
||||
},
|
||||
'stoploss': -0.10,
|
||||
"bid_strategy": {
|
||||
"ask_last_balance": 0.0
|
||||
},
|
||||
"exchange": {
|
||||
"pair_whitelist": [
|
||||
"BTC_ETH",
|
||||
"BTC_LTC",
|
||||
"BTC_ETC",
|
||||
"BTC_DASH",
|
||||
"BTC_ZEC",
|
||||
"BTC_XLM",
|
||||
"BTC_NXT",
|
||||
"BTC_POWR",
|
||||
"BTC_ADA",
|
||||
"BTC_XMR"
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user