Hyperopt to handle SIGINT by saving/reading the trials file

.gitignore

@@ -85,3 +85,5 @@ target/
 .venv
 .idea
 .vscode
+
+hyperopt_trials.pickle

freqtrade/optimize/hyperopt.py

@@ -4,6 +4,9 @@
 import json
 import logging
 import sys
+import pickle
+import signal
+import os
 from functools import reduce
 from math import exp
 from operator import itemgetter
@@ -43,6 +46,10 @@ EXPECTED_MAX_PROFIT = 3.85
 PROCESSED = None  # optimize.preprocess(optimize.load_data())
 OPTIMIZE_CONFIG = hyperopt_optimize_conf()
 
+# Hyperopt Trials
+TRIALS_FILE = 'freqtrade/optimize/hyperopt_trials.pickle'
+TRIALS = Trials()
+
 # Monkey patch config
 from freqtrade import main  # noqa
 main._CONF = OPTIMIZE_CONFIG
@@ -99,6 +106,19 @@ SPACE = {
 }
 
 
+def save_trials(trials, trials_path=TRIALS_FILE):
+    "Save hyperopt trials to file"
+    logger.info('Saving Trials to \'{}\''.format(trials_path))
+    pickle.dump(trials, open(trials_path, 'wb'))
+
+
+def read_trials(trials_path=TRIALS_FILE):
+    "Read hyperopt trials file"
+    logger.info('Reading Trials from \'{}\''.format(trials_path))
+    trials = pickle.load(open(trials_path, 'rb'))
+    # os.remove(trials_path)
+    return trials
+
 def log_results(results):
     """ log results if it is better than any previous evaluation """
     global CURRENT_BEST_LOSS
@@ -216,7 +236,12 @@ def buy_strategy_generator(params):
 
 
 def start(args):
+<<<<<<< HEAD
     global TOTAL_TRIES, PROCESSED, SPACE
+=======
+    global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES
+
+>>>>>>> Hyperopt to handle SIGINT by saving/reading the trials file
     TOTAL_TRIES = args.epochs
 
     exchange._API = Bittrex({'key': '', 'secret': ''})
@@ -238,9 +263,14 @@ def start(args):
         logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!')
 
         db_name = 'freqtrade_hyperopt'
-        trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
+        TRIALS = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
     else:
-        trials = Trials()
+        logger.info('Preparing Trials..')
+        signal.signal(signal.SIGINT, signal_handler)
+        # read trials file if we have one
+        if os.path.exists(TRIALS_FILE):
+            TRIALS = read_trials()
+            _CURRENT_TRIES = len([result for result in TRIALS.results if result['status'] == 'ok'])
 
     try:
         best_parameters = fmin(
@@ -248,7 +278,7 @@ def start(args):
             space=SPACE,
             algo=tpe.suggest,
             max_evals=TOTAL_TRIES,
-            trials=trials
+            trials=TRIALS
         )
 
         results = sorted(trials.results, key=itemgetter('loss'))
@@ -265,3 +295,14 @@ def start(args):
 
     logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4))
     logger.info('Best Result:\n%s', best_result)
+
+    # Store trials result to file to resume next time
+    save_trials(TRIALS)
+
+
+def signal_handler(sig, frame):
+    "Hyperopt SIGINT handler"
+    logger.info('Hyperopt received {}'.format(signal.Signals(sig).name))
+
+    save_trials(TRIALS)
+    sys.exit(0)