From ca8cab0ce9e59abd4dc3aa9b02a640a035e01f19 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 11:44:41 +0200 Subject: [PATCH 1/7] Hyperopt to handle SIGINT by saving/reading the trials file --- .gitignore | 2 ++ freqtrade/optimize/hyperopt.py | 47 +++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 672dd1f6d..c81b55222 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ target/ .venv .idea .vscode + +hyperopt_trials.pickle diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index cf46b96ad..febaa196d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,6 +4,9 @@ import json import logging import sys +import pickle +import signal +import os from functools import reduce from math import exp from operator import itemgetter @@ -43,6 +46,10 @@ EXPECTED_MAX_PROFIT = 3.85 PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() +# Hyperopt Trials +TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle' +TRIALS = Trials() + # Monkey patch config from freqtrade import main # noqa main._CONF = OPTIMIZE_CONFIG @@ -99,6 +106,19 @@ SPACE = { } +def save_trials(trials, trials_path=TRIALS_FILE): + "Save hyperopt trials to file" + logger.info('Saving Trials to \'{}\''.format(trials_path)) + pickle.dump(trials, open(trials_path, 'wb')) + + +def read_trials(trials_path=TRIALS_FILE): + "Read hyperopt trials file" + logger.info('Reading Trials from \'{}\''.format(trials_path)) + trials = pickle.load(open(trials_path, 'rb')) + # os.remove(trials_path) + return trials + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -216,7 +236,12 @@ def buy_strategy_generator(params): def start(args): +<<<<<<< HEAD global TOTAL_TRIES, PROCESSED, SPACE +======= + global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + +>>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -238,9 +263,14 @@ def start(args): logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!') db_name = 'freqtrade_hyperopt' - trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') + TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') else: - trials = Trials() + logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, signal_handler) + # read trials file if we have one + if os.path.exists(TRIALS_FILE): + TRIALS = read_trials() + _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok']) try: best_parameters = fmin( @@ -248,7 +278,7 @@ def start(args): space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, - trials=trials + trials=TRIALS ) results = sorted(trials.results, key=itemgetter('loss')) @@ -265,3 +295,14 @@ def start(args): logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + save_trials(TRIALS) + + +def signal_handler(sig, frame): + "Hyperopt SIGINT handler" + logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) + + save_trials(TRIALS) + sys.exit(0) From a48840509b4d168e44750e96bf6277dbbe53bb5d Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 15:34:21 +0200 Subject: [PATCH 2/7] Hyperopt: use results from previous runs --- freqtrade/optimize/hyperopt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index febaa196d..c520ed81d 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -119,6 +119,7 @@ def read_trials(trials_path=TRIALS_FILE): # os.remove(trials_path) return trials + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -270,7 +271,11 @@ def start(args): # read trials file if we have one if os.path.exists(TRIALS_FILE): TRIALS = read_trials() - _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok']) + _CURRENT_TRIES = len(TRIALS.results) + TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES + logger.info( + 'Continuing with trials. Current: {}, Total: {}' + .format(_CURRENT_TRIES, TOTAL_TRIES)) try: best_parameters = fmin( From b35fa4c9f6ba29c4c2333251df77276301cf449c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Sat, 6 Jan 2018 15:53:22 +0200 Subject: [PATCH 3/7] hyperopt: show the best results so far --- freqtrade/optimize/hyperopt.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c520ed81d..dc611fcbc 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -120,6 +120,11 @@ def read_trials(trials_path=TRIALS_FILE): return trials +def log_trials_result(trials): + vals = trials.best_trial['misc']['vals'] + results = trials.best_trial['result']['result'] + logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -310,4 +315,5 @@ def signal_handler(sig, frame): logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) save_trials(TRIALS) + log_trials_result(TRIALS) sys.exit(0) From 1647e7a0c1357fc663d52386381563687f45554c Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 11:37:27 +0200 Subject: [PATCH 4/7] update fix failing tests, unitest that resume hyperopt functionality works --- freqtrade/optimize/hyperopt.py | 12 ++--- freqtrade/tests/optimize/test_hyperopt.py | 66 +++++++++++++++++++++-- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index dc611fcbc..4162e0758 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 -TOTAL_TRIES = None +TOTAL_TRIES = 0 _CURRENT_TRIES = 0 CURRENT_BEST_LOSS = 100 @@ -116,7 +116,7 @@ def read_trials(trials_path=TRIALS_FILE): "Read hyperopt trials file" logger.info('Reading Trials from \'{}\''.format(trials_path)) trials = pickle.load(open(trials_path, 'rb')) - # os.remove(trials_path) + os.remove(trials_path) return trials @@ -125,6 +125,7 @@ def log_trials_result(trials): results = trials.best_trial['result']['result'] logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + def log_results(results): """ log results if it is better than any previous evaluation """ global CURRENT_BEST_LOSS @@ -242,12 +243,8 @@ def buy_strategy_generator(params): def start(args): -<<<<<<< HEAD - global TOTAL_TRIES, PROCESSED, SPACE -======= - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES ->>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file TOTAL_TRIES = args.epochs exchange._API = Bittrex({'key': '', 'secret': ''}) @@ -276,6 +273,7 @@ def start(args): # read trials file if we have one if os.path.exists(TRIALS_FILE): TRIALS = read_trials() + _CURRENT_TRIES = len(TRIALS.results) TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES logger.info( diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index a309af7fe..99a1f4a3c 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,5 +1,8 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 - +import pickle +import os +import pytest +import freqtrade.optimize.hyperopt from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ log_results @@ -27,16 +30,35 @@ def test_loss_calculation_has_limited_profit(): def create_trials(mocker): + """ + When creating trials, mock the hyperopt Trials so that *by default* + - we don't create any pickle'd files in the filesystem + - we might have a pickle'd file so make sure that we return + false when looking for it + """ + mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='freqtrade/tests/optimize/ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=False) + mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=None) return mocker.Mock( results=[{ 'loss': 1, - 'result': 'foo' - }] + 'result': 'foo', + 'status': 'ok' + }], + best_trial={'misc': {'vals': {'adx': 999}}} ) def test_start_calls_fmin(mocker): - mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker)) + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) mocker.patch('freqtrade.optimize.preprocess') mocker.patch('freqtrade.optimize.load_data') mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={}) @@ -141,3 +163,39 @@ def test_fmin_throw_value_error(mocker, caplog): for line in exists: assert line in caplog.text + + +def test_resuming_previous_hyperopt_results_succeeds(mocker): + import freqtrade.optimize.hyperopt as hyperopt + trials = create_trials(mocker) + mocker.patch('freqtrade.optimize.hyperopt.TRIALS', + return_value=trials) + mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', + return_value=True) + mocker.patch('freqtrade.optimize.hyperopt.len', + return_value=len(trials.results)) + mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials', + return_value=trials) + mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials', + return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.sorted', + return_value=trials.results) + mocker.patch('freqtrade.optimize.preprocess') + mocker.patch('freqtrade.optimize.load_data') + mocker.patch('freqtrade.optimize.hyperopt.fmin', + return_value={}) + args = mocker.Mock(epochs=1, + config='config.json.example', + mongodb=False) + + start(args) + + mock_read.assert_called_once() + mock_save.assert_called_once() + + current_tries = hyperopt._CURRENT_TRIES + total_tries = hyperopt.TOTAL_TRIES + + assert current_tries == len(trials.results) + assert total_tries == (current_tries + len(trials.results)) + From fe2b0c28621c53ded0ebe85ae1c4eb59c2661236 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 12:19:44 +0200 Subject: [PATCH 5/7] add unittest to save and read trials file --- freqtrade/tests/optimize/test_hyperopt.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 99a1f4a3c..3e03d26c0 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -1,10 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import pickle -import os -import pytest -import freqtrade.optimize.hyperopt from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \ - log_results + log_results, save_trials, read_trials def test_loss_calculation_prefer_correct_trade_count(): @@ -44,6 +40,8 @@ def create_trials(mocker): return_value=None) mocker.patch('freqtrade.optimize.hyperopt.read_trials', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.os.remove', + return_value=True) return mocker.Mock( results=[{ 'loss': 1, @@ -199,3 +197,27 @@ def test_resuming_previous_hyperopt_results_succeeds(mocker): assert current_tries == len(trials.results) assert total_tries == (current_tries + len(trials.results)) + +def test_save_trials_saves_trials(mocker): + trials = create_trials(mocker) + mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', + return_value=None) + trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE', + return_value='ut_trials.pickle') + mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=trials_path) + save_trials(trials, trials_path) + + mock_dump.assert_called_once_with(trials, trials_path) + + +def test_read_trials_returns_trials_file(mocker): + trials = create_trials(mocker) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', + return_value=trials) + mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', + return_value=mock_load) + + assert read_trials() == trials + mock_open.assert_called_once() + mock_load.assert_called_once() From ffae0b2cd54eab2d6eea2eae026194ca9d3d9657 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Tue, 9 Jan 2018 12:37:56 +0200 Subject: [PATCH 6/7] hyperopt: prettyfie best values when receiving SIGINT, use the global TRIALS --- freqtrade/optimize/hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 4162e0758..03f833c4a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -121,7 +121,7 @@ def read_trials(trials_path=TRIALS_FILE): def log_trials_result(trials): - vals = trials.best_trial['misc']['vals'] + vals = json.dumps(trials.best_trial['misc']['vals'], indent=4) results = trials.best_trial['result']['result'] logger.info('Best result:\n%s\nwith values:\n%s', results, vals) @@ -289,7 +289,7 @@ def start(args): trials=TRIALS ) - results = sorted(trials.results, key=itemgetter('loss')) + results = sorted(TRIALS.results, key=itemgetter('loss')) best_result = results[0]['result'] except ValueError: From e67c652988cf55b85cb06771dc513222c6eb0ea8 Mon Sep 17 00:00:00 2001 From: Samuel Husso Date: Wed, 10 Jan 2018 11:50:00 +0200 Subject: [PATCH 7/7] use os.path.join, fix docstrings --- freqtrade/optimize/hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 03f833c4a..5d342b831 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -47,7 +47,7 @@ PROCESSED = None # optimize.preprocess(optimize.load_data()) OPTIMIZE_CONFIG = hyperopt_optimize_conf() # Hyperopt Trials -TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle' +TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle') TRIALS = Trials() # Monkey patch config @@ -107,13 +107,13 @@ SPACE = { def save_trials(trials, trials_path=TRIALS_FILE): - "Save hyperopt trials to file" + """Save hyperopt trials to file""" logger.info('Saving Trials to \'{}\''.format(trials_path)) pickle.dump(trials, open(trials_path, 'wb')) def read_trials(trials_path=TRIALS_FILE): - "Read hyperopt trials file" + """Read hyperopt trials file""" logger.info('Reading Trials from \'{}\''.format(trials_path)) trials = pickle.load(open(trials_path, 'rb')) os.remove(trials_path) @@ -309,7 +309,7 @@ def start(args): def signal_handler(sig, frame): - "Hyperopt SIGINT handler" + """Hyperopt SIGINT handler""" logger.info('Hyperopt received {}'.format(signal.Signals(sig).name)) save_trials(TRIALS)