From e27a6a7a91772f73fc935c05ad77c477aa1ef1c6 Mon Sep 17 00:00:00 2001 From: gcarq Date: Sat, 25 Nov 2017 02:04:37 +0100 Subject: [PATCH] add mongodb support for hyperopt parallelization --- .gitignore | 1 + freqtrade/misc.py | 7 ++++++ freqtrade/optimize/backtesting.py | 32 ++++++++++++++----------- freqtrade/optimize/hyperopt.py | 40 ++++++++++++++++++++++++------- scripts/start-hyperopt-worker.sh | 5 ++++ scripts/start-mongodb.sh | 12 ++++++++++ 6 files changed, 74 insertions(+), 23 deletions(-) create mode 100755 scripts/start-hyperopt-worker.sh create mode 100755 scripts/start-mongodb.sh diff --git a/.gitignore b/.gitignore index 2d4af4dac..00f0e4651 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ target/ config.json preprocessor.py *.sqlite +.mongodb .env .venv diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 4f09db4c4..dcae19ac4 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -167,6 +167,13 @@ def build_subcommands(parser: argparse.ArgumentParser) -> None: type=int, metavar='INT', ) + hyperopt_cmd.add_argument( + '--use-mongodb', + help='parallelize evaluations with mongodb (requires mongod in PATH)', + dest='mongodb', + action='store_true', + ) + # Required json-schema for user specified config CONF_SCHEMA = { diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6c81fc75e..49f7e79a8 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -118,36 +118,38 @@ def backtest(config: Dict, processed: Dict[str, DataFrame], def start(args): - print('') + # Initialize logger + logging.basicConfig( + level=args.loglevel, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + exchange._API = Bittrex({'key': '', 'secret': ''}) - print('Using config: {} ...'.format(args.config)) + logger.info('Using config: %s ...', args.config) config = load_config(args.config) - print('Using ticker_interval: {} ...'.format(args.ticker_interval)) + logger.info('Using ticker_interval: %s ...', args.ticker_interval) data = {} if args.live: - print('Downloading data for all pairs in whitelist ...') + logger.info('Downloading data for all pairs in whitelist ...') for pair in config['exchange']['pair_whitelist']: data[pair] = exchange.get_ticker_history(pair, args.ticker_interval) else: - print('Using local backtesting data (ignoring whitelist in given config)...') + logger.info('Using local backtesting data (ignoring whitelist in given config) ...') data = load_data(args.ticker_interval) - print('Using stake_currency: {} ...\nUsing stake_amount: {} ...'.format( - config['stake_currency'], config['stake_amount'] - )) + logger.info('Using stake_currency: %s ...', config['stake_currency']) + logger.info('Using stake_amount: %s ...', config['stake_amount']) # Print timeframe min_date, max_date = get_timeframe(data) - print('Measuring data from {} up to {} ...'.format( - min_date.isoformat(), max_date.isoformat() - )) + logger.info('Measuring data from %s up to %s ...', min_date.isoformat(), max_date.isoformat()) max_open_trades = 0 if args.realistic_simulation: - print('Using max_open_trades: {} ...'.format(config['max_open_trades'])) + logger.info('Using max_open_trades: %s ...', config['max_open_trades']) max_open_trades = config['max_open_trades'] # Monkey patch config @@ -158,5 +160,7 @@ def start(args): results = backtest( config, preprocess(data), max_open_trades, args.realistic_simulation ) - print('====================== BACKTESTING REPORT ======================================\n\n') - print(generate_text_table(data, results, config['stake_currency'])) + logger.info( + '\n====================== BACKTESTING REPORT ======================================\n%s', + generate_text_table(data, results, config['stake_currency']) + ) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b243c0981..1b7e9a225 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -2,11 +2,13 @@ import json +import logging from functools import reduce from math import exp from operator import itemgetter from hyperopt import fmin, tpe, hp, Trials, STATUS_OK +from hyperopt.mongoexp import MongoTrials from pandas import DataFrame from freqtrade import exchange, optimize @@ -14,6 +16,9 @@ from freqtrade.exchange import Bittrex from freqtrade.optimize.backtesting import backtest from freqtrade.vendor.qtpylib.indicators import crossed_above +logger = logging.getLogger(__name__) + + # set TARGET_TRADES to suit your number concurrent trades so its realistic to 20days of data TARGET_TRADES = 1100 TOTAL_TRIES = 4 @@ -34,6 +39,11 @@ OPTIMIZE_CONFIG = { 'stoploss': -0.10, } +# Monkey patch config +from freqtrade import main +main._CONF = OPTIMIZE_CONFIG + + SPACE = { 'mfi': hp.choice('mfi', [ {'enabled': False}, @@ -101,7 +111,7 @@ def optimizer(params): profit_loss = max(0, 1 - total_profit / 10000) # max profit 10000 _CURRENT_TRIES += 1 - print('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result)) + logger.info('{:5d}/{}: {}'.format(_CURRENT_TRIES, TOTAL_TRIES, result)) return { 'loss': trade_loss + profit_loss, @@ -169,15 +179,27 @@ def start(args): global TOTAL_TRIES TOTAL_TRIES = args.epochs - # Monkey patch config - from freqtrade import main - main._CONF = OPTIMIZE_CONFIG - exchange._API = Bittrex({'key': '', 'secret': ''}) - trials = Trials() + # Initialize logger + logging.basicConfig( + level=args.loglevel, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + ) + + if args.mongodb: + logger.info('Using mongodb.') + logger.info('Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually') + + db_name = 'freqtrade_hyperopt' + trials = MongoTrials('mongo://127.0.0.1:1234/{}/jobs'.format(db_name), exp_key='exp1') + else: + trials = Trials() + best = fmin(fn=optimizer, space=SPACE, algo=tpe.suggest, max_evals=TOTAL_TRIES, trials=trials) - print('\n==================== HYPEROPT BACKTESTING REPORT ==============================\n') - print('Best parameters: {}'.format(json.dumps(best, indent=4))) + logger.info( + '\n==================== HYPEROPT BACKTESTING REPORT ==============================\n' + ) + logger.info('Best parameters:\n%s', json.dumps(best, indent=4)) results = sorted(trials.results, key=itemgetter('loss')) - print('Best Result: {}\n'.format(results[0]['result'])) + logger.info('Best Result:\n%s', results[0]['result']) diff --git a/scripts/start-hyperopt-worker.sh b/scripts/start-hyperopt-worker.sh new file mode 100755 index 000000000..ffe4753cb --- /dev/null +++ b/scripts/start-hyperopt-worker.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +DB_NAME=freqtrade_hyperopt + +hyperopt-mongo-worker --mongo=127.0.0.1:1234/${DB_NAME} --poll-interval=0.1 diff --git a/scripts/start-mongodb.sh b/scripts/start-mongodb.sh new file mode 100755 index 000000000..5a1758d04 --- /dev/null +++ b/scripts/start-mongodb.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DB_PATH="${DIR}/../.mongodb" + +mkdir -p ${DB_PATH} +mongod --dbpath ${DB_PATH} \ + --bind_ip 127.0.0.1 \ + --port 1234 \ + --directoryperdb \ + --journal \ + --nohttpinterface