stable/freqtrade/optimize/hyperopt.py

# pragma pylint: disable=missing-docstring,W0212,W0603


import json
import logging
import os
import pickle
import signal
import sys
from math import exp
from operator import itemgetter
from typing import Dict, Any

from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, space_eval, tpe
from hyperopt.mongoexp import MongoTrials
from pandas import DataFrame

# Monkey patch config
from freqtrade import main  # noqa; noqa
from freqtrade import exchange, misc, optimize
from freqtrade.exchange import Bittrex
from freqtrade.misc import load_config
from freqtrade.optimize import backtesting
from freqtrade.optimize.backtesting import backtest
from freqtrade.strategy.strategy import Strategy
from user_data.hyperopt_conf import hyperopt_optimize_conf

# Remove noisy log messages
logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING)
logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING)

logger = logging.getLogger(__name__)

# set TARGET_TRADES to suit your number concurrent trades so its realistic to the number of days
TARGET_TRADES = 600
TOTAL_TRIES = 0
_CURRENT_TRIES = 0
CURRENT_BEST_LOSS = 100

# max average trade duration in minutes
# if eval ends with higher value, we consider it a failed eval
MAX_ACCEPTED_TRADE_DURATION = 300

# this is expexted avg profit * expected trade count
# for example 3.5%, 1100 trades, EXPECTED_MAX_PROFIT = 3.85
# check that the reported Σ% values do not exceed this!
EXPECTED_MAX_PROFIT = 3.0

# Configuration and data used by hyperopt
PROCESSED = None  # optimize.preprocess(optimize.load_data())
OPTIMIZE_CONFIG = hyperopt_optimize_conf()

# Hyperopt Trials
TRIALS_FILE = os.path.join('user_data', 'hyperopt_trials.pickle')
TRIALS = Trials()

main._CONF = OPTIMIZE_CONFIG


def save_trials(trials, trials_path=TRIALS_FILE):
    """Save hyperopt trials to file"""
    logger.info('Saving Trials to \'{}\''.format(trials_path))
    pickle.dump(trials, open(trials_path, 'wb'))


def read_trials(trials_path=TRIALS_FILE):
    """Read hyperopt trials file"""
    logger.info('Reading Trials from \'{}\''.format(trials_path))
    trials = pickle.load(open(trials_path, 'rb'))
    os.remove(trials_path)
    return trials


def log_trials_result(trials):
    vals = json.dumps(trials.best_trial['misc']['vals'], indent=4)
    results = trials.best_trial['result']['result']
    logger.info('Best result:\n%s\nwith values:\n%s', results, vals)


def log_results(results):
    """ log results if it is better than any previous evaluation """
    global CURRENT_BEST_LOSS

    if results['loss'] < CURRENT_BEST_LOSS:
        CURRENT_BEST_LOSS = results['loss']
        logger.info('{:5d}/{}: {}. Loss {:.5f}'.format(
            results['current_tries'],
            results['total_tries'],
            results['result'],
            results['loss']))
    else:
        print('.', end='')
        sys.stdout.flush()


def calculate_loss(total_profit: float, trade_count: int, trade_duration: float):
    """ objective function, returns smaller number for more optimal results """
    trade_loss = 1 - 0.25 * exp(-(trade_count - TARGET_TRADES) ** 2 / 10 ** 5.8)
    profit_loss = max(0, 1 - total_profit / EXPECTED_MAX_PROFIT)
    duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1)
    return trade_loss + profit_loss + duration_loss


def generate_roi_table(params) -> Dict[int, float]:
    roi_table = {}
    roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3']
    roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2']
    roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1']
    roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0

    return roi_table


def has_space(spaces, space):
    if space in spaces or 'all' in spaces:
        return True
    return False


def hyperopt_space(selected_spaces: str, strategy) -> Dict[str, Any]:
    spaces = {}
    if has_space(selected_spaces, 'buy'):
        spaces = {**spaces, **strategy.indicator_space()}
    if has_space(selected_spaces, 'roi'):
        spaces = {**spaces, **strategy.roi_space()}
    if has_space(selected_spaces, 'stoploss'):
        spaces = {**spaces, **strategy.stoploss_space()}
    return spaces


def generate_optimizer(args, strategy):
    def optimizer(params):
        global _CURRENT_TRIES

        if has_space(args.spaces, 'roi'):
            strategy.minimal_roi = generate_roi_table(params)

        if has_space(args.spaces, 'buy'):
            backtesting.populate_buy_trend = strategy.buy_strategy_generator(params)

        if has_space(args.spaces, 'stoploss'):
            strategy.stoploss = params['stoploss']
        results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'],
                            'processed': PROCESSED,
                            'realistic': args.realistic_simulation,
                            })
        result_explanation = format_results(results)

        total_profit = results.profit_percent.sum()
        trade_count = len(results.index)
        trade_duration = results.duration.mean()

        if trade_count == 0 or trade_duration > MAX_ACCEPTED_TRADE_DURATION:
            print('.', end='')
            return {
                'status': STATUS_FAIL,
                'loss': float('inf')
            }

        loss = calculate_loss(total_profit, trade_count, trade_duration)

        _CURRENT_TRIES += 1

        log_results({
            'loss': loss,
            'current_tries': _CURRENT_TRIES,
            'total_tries': TOTAL_TRIES,
            'result': result_explanation,
        })

        return {
            'loss': loss,
            'status': STATUS_OK,
            'result': result_explanation,
        }

    return optimizer


def format_results(results: DataFrame):
    return ('{:6d} trades. Avg profit {: 5.2f}%. '
            'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format(
                len(results.index),
                results.profit_percent.mean() * 100.0,
                results.profit_BTC.sum(),
                results.profit_percent.sum(),
                results.duration.mean(),
            )


def start(args):
    global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES

    TOTAL_TRIES = args.epochs

    exchange._API = Bittrex({'key': '', 'secret': ''})

    # Initialize logger
    logging.basicConfig(
        level=args.loglevel,
        format='\n%(message)s',
    )

    logger.info('Using config: %s ...', args.config)
    config = load_config(args.config)
    pairs = config['exchange']['pair_whitelist']

    # If -i/--ticker-interval is use we override the configuration parameter
    # (that will override the strategy configuration)
    if args.ticker_interval:
        config.update({'ticker_interval': args.ticker_interval})

    # init the strategy to use
    config.update({'strategy': args.strategy})
    strategy = Strategy()
    strategy.init(config)

    timerange = misc.parse_timerange(args.timerange)
    data = optimize.load_data(args.datadir, pairs=pairs,
                              ticker_interval=strategy.ticker_interval,
                              timerange=timerange)
    if has_space(args.spaces, 'buy'):
        optimize.populate_indicators = strategy.populate_indicators
    PROCESSED = optimize.tickerdata_to_dataframe(data)

    if args.mongodb:
        logger.info('Using mongodb ...')
        logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')

        db_name = 'freqtrade_hyperopt'
        TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
    else:
        logger.info('Preparing Trials..')
        signal.signal(signal.SIGINT, signal_handler)
        # read trials file if we have one
        if os.path.exists(TRIALS_FILE):
            TRIALS = read_trials()

            _CURRENT_TRIES = len(TRIALS.results)
            TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES
            logger.info(
                'Continuing with trials. Current: {}, Total: {}'
                .format(_CURRENT_TRIES, TOTAL_TRIES))

    try:
        best_parameters = fmin(
            fn=generate_optimizer(args, strategy),
            space=hyperopt_space(args.spaces, strategy),
            algo=tpe.suggest,
            max_evals=TOTAL_TRIES,
            trials=TRIALS
        )

        results = sorted(TRIALS.results, key=itemgetter('loss'))
        best_result = results[0]['result']

    except ValueError:
        best_parameters = {}
        best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \
                      'try with more epochs (param: -e).'

    # Improve best parameter logging display
    if best_parameters:
        best_parameters = space_eval(
            hyperopt_space(args.spaces, strategy),
            best_parameters
        )

    logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4))
    if 'roi_t1' in best_parameters:
        logger.info('ROI table:\n%s', generate_roi_table(best_parameters))
    logger.info('Best Result:\n%s', best_result)

    # Store trials result to file to resume next time
    save_trials(TRIALS)


def signal_handler(sig, frame):
    """Hyperopt SIGINT handler"""
    logger.info('Hyperopt received {}'.format(signal.Signals(sig).name))

    save_trials(TRIALS)
    log_trials_result(TRIALS)
    sys.exit(0)