285 lines
9.2 KiB
Python
285 lines
9.2 KiB
Python
# pragma pylint: disable=missing-docstring,W0212,W0603
|
|
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import pickle
|
|
import signal
|
|
import sys
|
|
from math import exp
|
|
from operator import itemgetter
|
|
from typing import Dict, Any
|
|
|
|
from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, space_eval, tpe
|
|
from hyperopt.mongoexp import MongoTrials
|
|
from pandas import DataFrame
|
|
|
|
# Monkey patch config
|
|
from freqtrade import main # noqa; noqa
|
|
from freqtrade import exchange, misc, optimize
|
|
from freqtrade.exchange import Bittrex
|
|
from freqtrade.misc import load_config
|
|
from freqtrade.optimize import backtesting
|
|
from freqtrade.optimize.backtesting import backtest
|
|
from freqtrade.strategy.strategy import Strategy
|
|
from user_data.hyperopt_conf import hyperopt_optimize_conf
|
|
|
|
# Remove noisy log messages
|
|
logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING)
|
|
logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days
|
|
TARGET_TRADES = 600
|
|
TOTAL_TRIES = 0
|
|
_CURRENT_TRIES = 0
|
|
CURRENT_BEST_LOSS = 100
|
|
|
|
# max average trade duration in minutes
|
|
# if eval ends with higher value, we consider it a failed eval
|
|
MAX_ACCEPTED_TRADE_DURATION = 300
|
|
|
|
# this is expexted avg profit * expected trade count
|
|
# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85
|
|
# check that the reported Σ% values do not exceed this!
|
|
EXPECTED_MAX_PROFIT = 3.0
|
|
|
|
# Configuration and data used by hyperopt
|
|
PROCESSED = None # optimize.preprocess(optimize.load_data())
|
|
OPTIMIZE_CONFIG = hyperopt_optimize_conf()
|
|
|
|
# Hyperopt Trials
|
|
TRIALS_FILE = os.path.join('user_data', 'hyperopt_trials.pickle')
|
|
TRIALS = Trials()
|
|
|
|
main._CONF = OPTIMIZE_CONFIG
|
|
|
|
|
|
def save_trials(trials, trials_path=TRIALS_FILE):
|
|
"""Save hyperopt trials to file"""
|
|
logger.info('Saving Trials to \'{}\''.format(trials_path))
|
|
pickle.dump(trials, open(trials_path, 'wb'))
|
|
|
|
|
|
def read_trials(trials_path=TRIALS_FILE):
|
|
"""Read hyperopt trials file"""
|
|
logger.info('Reading Trials from \'{}\''.format(trials_path))
|
|
trials = pickle.load(open(trials_path, 'rb'))
|
|
os.remove(trials_path)
|
|
return trials
|
|
|
|
|
|
def log_trials_result(trials):
|
|
vals = json.dumps(trials.best_trial['misc']['vals'], indent=4)
|
|
results = trials.best_trial['result']['result']
|
|
logger.info('Best result:\n%s\nwith values:\n%s', results, vals)
|
|
|
|
|
|
def log_results(results):
|
|
""" log results if it is better than any previous evaluation """
|
|
global CURRENT_BEST_LOSS
|
|
|
|
if results['loss'] < CURRENT_BEST_LOSS:
|
|
CURRENT_BEST_LOSS = results['loss']
|
|
logger.info('{:5d}/{}: {}. Loss {:.5f}'.format(
|
|
results['current_tries'],
|
|
results['total_tries'],
|
|
results['result'],
|
|
results['loss']))
|
|
else:
|
|
print('.', end='')
|
|
sys.stdout.flush()
|
|
|
|
|
|
def calculate_loss(total_profit: float, trade_count: int, trade_duration: float):
|
|
""" objective function, returns smaller number for more optimal results """
|
|
trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8)
|
|
profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT)
|
|
duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1)
|
|
return trade_loss + profit_loss + duration_loss
|
|
|
|
|
|
def generate_roi_table(params) -> Dict[int, float]:
|
|
roi_table = {}
|
|
roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
|
|
roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
|
|
roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
|
|
roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0
|
|
|
|
return roi_table
|
|
|
|
|
|
def has_space(spaces, space):
|
|
if space in spaces or 'all' in spaces:
|
|
return True
|
|
return False
|
|
|
|
|
|
def hyperopt_space(selected_spaces: str, strategy) -> Dict[str, Any]:
|
|
spaces = {}
|
|
if has_space(selected_spaces, 'buy'):
|
|
spaces = {**spaces, **strategy.indicator_space()}
|
|
if has_space(selected_spaces, 'roi'):
|
|
spaces = {**spaces, **strategy.roi_space()}
|
|
if has_space(selected_spaces, 'stoploss'):
|
|
spaces = {**spaces, **strategy.stoploss_space()}
|
|
return spaces
|
|
|
|
|
|
def generate_optimizer(args, strategy):
|
|
def optimizer(params):
|
|
global _CURRENT_TRIES
|
|
|
|
if has_space(args.spaces, 'roi'):
|
|
strategy.minimal_roi = generate_roi_table(params)
|
|
|
|
if has_space(args.spaces, 'buy'):
|
|
backtesting.populate_buy_trend = strategy.buy_strategy_generator(params)
|
|
|
|
if has_space(args.spaces, 'stoploss'):
|
|
strategy.stoploss = params['stoploss']
|
|
results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'],
|
|
'processed': PROCESSED,
|
|
'realistic': args.realistic_simulation,
|
|
})
|
|
result_explanation = format_results(results)
|
|
|
|
total_profit = results.profit_percent.sum()
|
|
trade_count = len(results.index)
|
|
trade_duration = results.duration.mean()
|
|
|
|
if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION:
|
|
print('.', end='')
|
|
return {
|
|
'status': STATUS_FAIL,
|
|
'loss': float('inf')
|
|
}
|
|
|
|
loss = calculate_loss(total_profit, trade_count, trade_duration)
|
|
|
|
_CURRENT_TRIES += 1
|
|
|
|
log_results({
|
|
'loss': loss,
|
|
'current_tries': _CURRENT_TRIES,
|
|
'total_tries': TOTAL_TRIES,
|
|
'result': result_explanation,
|
|
})
|
|
|
|
return {
|
|
'loss': loss,
|
|
'status': STATUS_OK,
|
|
'result': result_explanation,
|
|
}
|
|
|
|
return optimizer
|
|
|
|
|
|
def format_results(results: DataFrame):
|
|
return ('{:6d} trades. Avg profit {: 5.2f}%. '
|
|
'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format(
|
|
len(results.index),
|
|
results.profit_percent.mean() * 100.0,
|
|
results.profit_BTC.sum(),
|
|
results.profit_percent.sum(),
|
|
results.duration.mean(),
|
|
)
|
|
|
|
|
|
def start(args):
|
|
global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES
|
|
|
|
TOTAL_TRIES = args.epochs
|
|
|
|
exchange._API = Bittrex({'key': '', 'secret': ''})
|
|
|
|
# Initialize logger
|
|
logging.basicConfig(
|
|
level=args.loglevel,
|
|
format='\n%(message)s',
|
|
)
|
|
|
|
logger.info('Using config: %s ...', args.config)
|
|
config = load_config(args.config)
|
|
pairs = config['exchange']['pair_whitelist']
|
|
|
|
# If -i/--ticker-interval is use we override the configuration parameter
|
|
# (that will override the strategy configuration)
|
|
if args.ticker_interval:
|
|
config.update({'ticker_interval': args.ticker_interval})
|
|
|
|
# init the strategy to use
|
|
config.update({'strategy': args.strategy})
|
|
strategy = Strategy()
|
|
strategy.init(config)
|
|
|
|
timerange = misc.parse_timerange(args.timerange)
|
|
data = optimize.load_data(args.datadir, pairs=pairs,
|
|
ticker_interval=strategy.ticker_interval,
|
|
timerange=timerange)
|
|
if has_space(args.spaces, 'buy'):
|
|
optimize.populate_indicators = strategy.populate_indicators
|
|
PROCESSED = optimize.tickerdata_to_dataframe(data)
|
|
|
|
if args.mongodb:
|
|
logger.info('Using mongodb ...')
|
|
logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')
|
|
|
|
db_name = 'freqtrade_hyperopt'
|
|
TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
|
|
else:
|
|
logger.info('Preparing Trials..')
|
|
signal.signal(signal.SIGINT, signal_handler)
|
|
# read trials file if we have one
|
|
if os.path.exists(TRIALS_FILE):
|
|
TRIALS = read_trials()
|
|
|
|
_CURRENT_TRIES = len(TRIALS.results)
|
|
TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES
|
|
logger.info(
|
|
'Continuing with trials. Current: {}, Total: {}'
|
|
.format(_CURRENT_TRIES, TOTAL_TRIES))
|
|
|
|
try:
|
|
best_parameters = fmin(
|
|
fn=generate_optimizer(args, strategy),
|
|
space=hyperopt_space(args.spaces, strategy),
|
|
algo=tpe.suggest,
|
|
max_evals=TOTAL_TRIES,
|
|
trials=TRIALS
|
|
)
|
|
|
|
results = sorted(TRIALS.results, key=itemgetter('loss'))
|
|
best_result = results[0]['result']
|
|
|
|
except ValueError:
|
|
best_parameters = {}
|
|
best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \
|
|
'try with more epochs (param: -e).'
|
|
|
|
# Improve best parameter logging display
|
|
if best_parameters:
|
|
best_parameters = space_eval(
|
|
hyperopt_space(args.spaces, strategy),
|
|
best_parameters
|
|
)
|
|
|
|
logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4))
|
|
if 'roi_t1' in best_parameters:
|
|
logger.info('ROI table:\n%s', generate_roi_table(best_parameters))
|
|
logger.info('Best Result:\n%s', best_result)
|
|
|
|
# Store trials result to file to resume next time
|
|
save_trials(TRIALS)
|
|
|
|
|
|
def signal_handler(sig, frame):
|
|
"""Hyperopt SIGINT handler"""
|
|
logger.info('Hyperopt received {}'.format(signal.Signals(sig).name))
|
|
|
|
save_trials(TRIALS)
|
|
log_trials_result(TRIALS)
|
|
sys.exit(0)
|