diff --git a/.gitignore b/.gitignore index 672dd1f6d..c81b55222 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ target/ .venv .idea .vscode + +hyperopt_trials.pickle diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cf46b96ad..5d342b831 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,6 +4,9 @@ import json import logging import sys +import pickle +import signal +import os from functools import reduce from math import exp from operator import itemgetter @@ -27,7 +30,7 @@ logger = logging.getLogger(__name__) # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 -TOTAL_TRIES = None +TOTAL_TRIES = 0 _CURRENT_TRIES = 0 CURRENT_BEST_LOSS = 100 @@ -43,6 +46,10 @@ EXPECTED_MAX_PROFIT = 3.85 PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() +# Hyperopt Trials +TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle') +TRIALS = Trials() + # Monkey patch config from freqtrade import main # noqa main._CONF = OPTIMIZE_CONFIG @@ -99,6 +106,26 @@ SPACE = { } +def save_trials(trials, trials_path=TRIALS_FILE): + """Save hyperopt trials to file""" + logger.info('Saving Trials to \'{}\''.format(trials_path)) + pickle.dump(trials, open(trials_path, 'wb')) + + +def read_trials(trials_path=TRIALS_FILE): + """Read hyperopt trials file""" + logger.info('Reading Trials from \'{}\''.format(trials_path)) + trials = pickle.load(open(trials_path, 'rb')) + os.remove(trials_path) + return trials + + +def log_trials_result(trials): + vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) + results = trials.best_trial['result']['result'] + logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -216,7 +243,8 @@ def buy_strategy_generator(params): def start(args): - global TOTAL_TRIES, PROCESSED, SPACE + global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES + TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -238,9 +266,19 @@ def start(args): logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') db_name = 'freqtrade_hyperopt' - trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') + TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') else: - trials = Trials() + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, signal_handler) + # read trials file if we have one + if os.path.exists(TRIALS_FILE): + TRIALS = read_trials() + + _CURRENT_TRIES = len(TRIALS.results) + TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES + logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(_CURRENT_TRIES, TOTAL_TRIES)) try: best_parameters = fmin( @@ -248,10 +286,10 @@ def start(args): space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, - trials=trials + trials=TRIALS ) - results = sorted(trials.results, key=itemgetter('loss')) + results = sorted(TRIALS.results, key=itemgetter('loss')) best_result = results[0]['result'] except ValueError: @@ -265,3 +303,15 @@ def start(args): logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + save_trials(TRIALS) + + +def signal_handler(sig, frame): + """Hyperopt SIGINT handler""" + logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + save_trials(TRIALS) + log_trials_result(TRIALS) + sys.exit(0) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a309af7fe..3e03d26c0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,7 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 - from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results + log_results, save_trials, read_trials def test_loss_calculation_prefer_correct_trade_count(): @@ -27,16 +26,37 @@ def test_loss_calculation_has_limited_profit(): def create_trials(mocker): + """ + When creating trials, mock the hyperopt Trials so that *by default* + - we don't create any pickle'd files in the filesystem + - we might have a pickle'd file so make sure that we return + false when looking for it + """ + mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='freqtrade/tests/optimize/ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', + return_value=True) return mocker.Mock( results=[{ 'loss': 1, - 'result': 'foo' - }] + 'result': 'foo', + 'status': 'ok' + }], + best_trial={'misc': {'vals': {'adx': 999}}} ) def test_start_calls_fmin(mocker): - mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) mocker.patch('freqtrade.optimize.preprocess') mocker.patch('freqtrade.optimize.load_data') mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) @@ -141,3 +161,63 @@ def test_fmin_throw_value_error(mocker, caplog): for line in exists: assert line in caplog.text + + +def test_resuming_previous_hyperopt_results_succeeds(mocker): + import freqtrade.optimize.hyperopt as hyperopt + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', + return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', + return_value=len(trials.results)) + mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=trials) + mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', + return_value={}) + args = mocker.Mock(epochs=1, + config='config.json.example', + mongodb=False) + + start(args) + + mock_read.assert_called_once() + mock_save.assert_called_once() + + current_tries = hyperopt._CURRENT_TRIES + total_tries = hyperopt.TOTAL_TRIES + + assert current_tries == len(trials.results) + assert total_tries == (current_tries + len(trials.results)) + + +def test_save_trials_saves_trials(mocker): + trials = create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', + return_value=None) + trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=trials_path) + save_trials(trials, trials_path) + + mock_dump.assert_called_once_with(trials, trials_path) + + +def test_read_trials_returns_trials_file(mocker): + trials = create_trials(mocker) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', + return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=mock_load) + + assert read_trials() == trials + mock_open.assert_called_once() + mock_load.assert_called_once()