Merge pull request #351 from gcarq/feat/hyperopt-resume

resume hyperopt run

.gitignore

@@ -85,3 +85,5 @@ target/
 .venv
 .idea
 .vscode
+
+hyperopt_trials.pickle

freqtrade/optimize/hyperopt.py

@@ -4,6 +4,9 @@
 import json
 import logging
 import sys
+import pickle
+import signal
+import os
 from functools import reduce
 from math import exp
 from operator import itemgetter
@@ -27,7 +30,7 @@ logger = logging.getLogger(__name__)
 
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
-TOTAL_TRIES = None
+TOTAL_TRIES = 0
 _CURRENT_TRIES = 0
 CURRENT_BEST_LOSS = 100
 
@@ -43,6 +46,10 @@ EXPECTED_MAX_PROFIT = 3.85
 PROCESSED = None  # optimize.preprocess(optimize.load_data())
 OPTIMIZE_CONFIG = hyperopt_optimize_conf()
 
+# Hyperopt Trials
+TRIALS_FILE = os.path.join('freqtrade', 'optimize', 'hyperopt_trials.pickle')
+TRIALS = Trials()
+
 # Monkey patch config
 from freqtrade import main  # noqa
 main._CONF = OPTIMIZE_CONFIG
@@ -99,6 +106,26 @@ SPACE = {
 }
 
 
+def save_trials(trials, trials_path=TRIALS_FILE):
+    """Save hyperopt trials to file"""
+    logger.info('Saving Trials to \'{}\''.format(trials_path))
+    pickle.dump(trials, open(trials_path, 'wb'))
+
+
+def read_trials(trials_path=TRIALS_FILE):
+    """Read hyperopt trials file"""
+    logger.info('Reading Trials from \'{}\''.format(trials_path))
+    trials = pickle.load(open(trials_path, 'rb'))
+    os.remove(trials_path)
+    return trials
+
+
+def log_trials_result(trials):
+    vals = json.dumps(trials.best_trial['misc']['vals'], indent=4)
+    results = trials.best_trial['result']['result']
+    logger.info('Best result:\n%s\nwith values:\n%s', results, vals)
+
+
 def log_results(results):
     """ log results if it is better than any previous evaluation """
     global CURRENT_BEST_LOSS
@@ -216,7 +243,8 @@ def buy_strategy_generator(params):
 
 
 def start(args):
-    global TOTAL_TRIES, PROCESSED, SPACE
+    global TOTAL_TRIES, PROCESSED, SPACE, TRIALS, _CURRENT_TRIES
+
     TOTAL_TRIES = args.epochs
 
     exchange._API = Bittrex({'key': '', 'secret': ''})
@@ -238,9 +266,19 @@ def start(args):
         logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')
 
         db_name = 'freqtrade_hyperopt'
-        trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
+        TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
     else:
-        trials = Trials()
+        logger.info('Preparing Trials..')
+        signal.signal(signal.SIGINT, signal_handler)
+        # read trials file if we have one
+        if os.path.exists(TRIALS_FILE):
+            TRIALS = read_trials()
+
+            _CURRENT_TRIES = len(TRIALS.results)
+            TOTAL_TRIES = TOTAL_TRIES + _CURRENT_TRIES
+            logger.info(
+                'Continuing with trials. Current: {}, Total: {}'
+                .format(_CURRENT_TRIES, TOTAL_TRIES))
 
     try:
         best_parameters = fmin(
@@ -248,10 +286,10 @@ def start(args):
             space=SPACE,
             algo=tpe.suggest,
             max_evals=TOTAL_TRIES,
-            trials=trials
+            trials=TRIALS
         )
 
-        results = sorted(trials.results, key=itemgetter('loss'))
+        results = sorted(TRIALS.results, key=itemgetter('loss'))
         best_result = results[0]['result']
 
     except ValueError:
@@ -265,3 +303,15 @@ def start(args):
 
     logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4))
     logger.info('Best Result:\n%s', best_result)
+
+    # Store trials result to file to resume next time
+    save_trials(TRIALS)
+
+
+def signal_handler(sig, frame):
+    """Hyperopt SIGINT handler"""
+    logger.info('Hyperopt received {}'.format(signal.Signals(sig).name))
+
+    save_trials(TRIALS)
+    log_trials_result(TRIALS)
+    sys.exit(0)

freqtrade/tests/optimize/test_hyperopt.py

@@ -1,7 +1,6 @@
 # pragma pylint: disable=missing-docstring,W0212,C0103
-
 from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \
-    log_results
+    log_results, save_trials, read_trials
 
 
 def test_loss_calculation_prefer_correct_trade_count():
@@ -27,16 +26,37 @@ def test_loss_calculation_has_limited_profit():
 
 
 def create_trials(mocker):
+    """
+    When creating trials, mock the hyperopt Trials so that *by default*
+      - we don't create any pickle'd files in the filesystem
+      - we might have a pickle'd file so make sure that we return
+        false when looking for it
+    """
+    mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE',
+                 return_value='freqtrade/tests/optimize/ut_trials.pickle')
+    mocker.patch('freqtrade.optimize.hyperopt.os.path.exists',
+                 return_value=False)
+    mocker.patch('freqtrade.optimize.hyperopt.save_trials',
+                 return_value=None)
+    mocker.patch('freqtrade.optimize.hyperopt.read_trials',
+                 return_value=None)
+    mocker.patch('freqtrade.optimize.hyperopt.os.remove',
+                 return_value=True)
     return mocker.Mock(
         results=[{
             'loss': 1,
-            'result': 'foo'
+            'result': 'foo',
-        }]
+            'status': 'ok'
+        }],
+        best_trial={'misc': {'vals': {'adx': 999}}}
     )
 
 
 def test_start_calls_fmin(mocker):
-    mocker.patch('freqtrade.optimize.hyperopt.Trials', return_value=create_trials(mocker))
+    trials = create_trials(mocker)
+    mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials)
+    mocker.patch('freqtrade.optimize.hyperopt.sorted',
+                 return_value=trials.results)
     mocker.patch('freqtrade.optimize.preprocess')
     mocker.patch('freqtrade.optimize.load_data')
     mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
@@ -141,3 +161,63 @@ def test_fmin_throw_value_error(mocker, caplog):
 
     for line in exists:
         assert line in caplog.text
+
+
+def test_resuming_previous_hyperopt_results_succeeds(mocker):
+    import freqtrade.optimize.hyperopt as hyperopt
+    trials = create_trials(mocker)
+    mocker.patch('freqtrade.optimize.hyperopt.TRIALS',
+                 return_value=trials)
+    mocker.patch('freqtrade.optimize.hyperopt.os.path.exists',
+                 return_value=True)
+    mocker.patch('freqtrade.optimize.hyperopt.len',
+                 return_value=len(trials.results))
+    mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials',
+                             return_value=trials)
+    mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials',
+                             return_value=None)
+    mocker.patch('freqtrade.optimize.hyperopt.sorted',
+                 return_value=trials.results)
+    mocker.patch('freqtrade.optimize.preprocess')
+    mocker.patch('freqtrade.optimize.load_data')
+    mocker.patch('freqtrade.optimize.hyperopt.fmin',
+                 return_value={})
+    args = mocker.Mock(epochs=1,
+                       config='config.json.example',
+                       mongodb=False)
+
+    start(args)
+
+    mock_read.assert_called_once()
+    mock_save.assert_called_once()
+
+    current_tries = hyperopt._CURRENT_TRIES
+    total_tries = hyperopt.TOTAL_TRIES
+
+    assert current_tries == len(trials.results)
+    assert total_tries == (current_tries + len(trials.results))
+
+
+def test_save_trials_saves_trials(mocker):
+    trials = create_trials(mocker)
+    mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump',
+                             return_value=None)
+    trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE',
+                               return_value='ut_trials.pickle')
+    mocker.patch('freqtrade.optimize.hyperopt.open',
+                 return_value=trials_path)
+    save_trials(trials, trials_path)
+
+    mock_dump.assert_called_once_with(trials, trials_path)
+
+
+def test_read_trials_returns_trials_file(mocker):
+    trials = create_trials(mocker)
+    mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load',
+                             return_value=trials)
+    mock_open = mocker.patch('freqtrade.optimize.hyperopt.open',
+                             return_value=mock_load)
+
+    assert read_trials() == trials
+    mock_open.assert_called_once()
+    mock_load.assert_called_once()