add mongodb support for hyperopt parallelization

.gitignore

@@ -76,6 +76,7 @@ target/
 config.json
 preprocessor.py
 *.sqlite
+.mongodb
 
 .env
 .venv

freqtrade/misc.py

@@ -167,6 +167,13 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None:
         type=int,
         metavar='INT',
     )
+    hyperopt_cmd.add_argument(
+        '--use-mongodb',
+        help='parallelize evaluations with mongodb (requires mongod in PATH)',
+        dest='mongodb',
+        action='store_true',
+    )
+
 
 # Required json-schema for user specified config
 CONF_SCHEMA = {

freqtrade/optimize/backtesting.py

@@ -118,36 +118,38 @@ def backtest(config: Dict, processed: Dict[str, DataFrame],
 
 
 def start(args):
-    print('')
+    # Initialize logger
+    logging.basicConfig(
+        level=args.loglevel,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    )
+
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    print('Using config: {} ...'.format(args.config))
+    logger.info('Using config: %s ...', args.config)
     config = load_config(args.config)
 
-    print('Using ticker_interval: {} ...'.format(args.ticker_interval))
+    logger.info('Using ticker_interval: %s ...', args.ticker_interval)
 
     data = {}
     if args.live:
-        print('Downloading data for all pairs in whitelist ...')
+        logger.info('Downloading data for all pairs in whitelist ...')
         for pair in config['exchange']['pair_whitelist']:
             data[pair] = exchange.get_ticker_history(pair, args.ticker_interval)
     else:
-        print('Using local backtesting data (ignoring whitelist in given config)...')
+        logger.info('Using local backtesting data (ignoring whitelist in given config) ...')
         data = load_data(args.ticker_interval)
 
-    print('Using stake_currency: {} ...\nUsing stake_amount: {} ...'.format(
+        logger.info('Using stake_currency: %s ...', config['stake_currency'])
-        config['stake_currency'], config['stake_amount']
+        logger.info('Using stake_amount: %s ...', config['stake_amount'])
-    ))
 
     # Print timeframe
     min_date, max_date = get_timeframe(data)
-    print('Measuring data from {} up to {} ...'.format(
+    logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat())
-        min_date.isoformat(), max_date.isoformat()
-    ))
 
     max_open_trades = 0
     if args.realistic_simulation:
-        print('Using max_open_trades: {} ...'.format(config['max_open_trades']))
+        logger.info('Using max_open_trades: %s ...', config['max_open_trades'])
         max_open_trades = config['max_open_trades']
 
     # Monkey patch config
@@ -158,5 +160,7 @@ def start(args):
     results = backtest(
         config, preprocess(data), max_open_trades, args.realistic_simulation
     )
-    print('====================== BACKTESTING REPORT ======================================\n\n')
+    logger.info(
-    print(generate_text_table(data, results, config['stake_currency']))
+        '\n====================== BACKTESTING REPORT ======================================\n%s',
+        generate_text_table(data, results, config['stake_currency'])
+    )

freqtrade/optimize/hyperopt.py

@@ -2,11 +2,13 @@
 
 
 import json
+import logging
 from functools import reduce
 from math import exp
 from operator import itemgetter
 
 from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
+from hyperopt.mongoexp import MongoTrials
 from pandas import DataFrame
 
 from freqtrade import exchange, optimize
@@ -14,6 +16,9 @@ from freqtrade.exchange import Bittrex
 from freqtrade.optimize.backtesting import backtest
 from freqtrade.vendor.qtpylib.indicators import crossed_above
 
+logger = logging.getLogger(__name__)
+
+
 # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data
 TARGET_TRADES = 1100
 TOTAL_TRIES = 4
@@ -34,6 +39,11 @@ OPTIMIZE_CONFIG = {
     'stoploss': -0.10,
 }
 
+# Monkey patch config
+from freqtrade import main
+main._CONF = OPTIMIZE_CONFIG
+
+
 SPACE = {
     'mfi': hp.choice('mfi', [
         {'enabled': False},
@@ -101,7 +111,7 @@ def optimizer(params):
     profit_loss = max(0, 1 - total_profit / 10000)  # max profit 10000
 
     _CURRENT_TRIES += 1
-    print('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
+    logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result))
 
     return {
         'loss': trade_loss + profit_loss,
@@ -169,15 +179,27 @@ def start(args):
     global TOTAL_TRIES
     TOTAL_TRIES = args.epochs
 
-    # Monkey patch config
-    from freqtrade import main
-    main._CONF = OPTIMIZE_CONFIG
-
     exchange._API = Bittrex({'key': '', 'secret': ''})
 
-    trials = Trials()
+    # Initialize logger
+    logging.basicConfig(
+        level=args.loglevel,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    )
+
+    if args.mongodb:
+        logger.info('Using mongodb.')
+        logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually')
+
+        db_name = 'freqtrade_hyperopt'
+        trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1')
+    else:
+        trials = Trials()
+
     best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials)
-    print('\n==================== HYPEROPT BACKTESTING REPORT ==============================\n')
+    logger.info(
-    print('Best parameters: {}'.format(json.dumps(best, indent=4)))
+        '\n==================== HYPEROPT BACKTESTING REPORT ==============================\n'
+    )
+    logger.info('Best parameters:\n%s', json.dumps(best, indent=4))
     results = sorted(trials.results, key=itemgetter('loss'))
-    print('Best Result: {}\n'.format(results[0]['result']))
+    logger.info('Best Result:\n%s', results[0]['result'])

scripts/start-hyperopt-worker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+DB_NAME=freqtrade_hyperopt
+
+hyperopt-mongo-worker --mongo=127.0.0.1:1234/${DB_NAME} --poll-interval=0.1

scripts/start-mongodb.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DB_PATH="${DIR}/../.mongodb"
+
+mkdir -p ${DB_PATH}
+mongod --dbpath ${DB_PATH} \
+    --bind_ip 127.0.0.1 \
+    --port 1234 \
+    --directoryperdb \
+    --journal \
+    --nohttpinterface