diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 87ce67fc6..f5cd573ea 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -216,7 +216,7 @@ usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE] [--stake_amount STAKE_AMOUNT] [-r] [--customhyperopt NAME] [--eps] [--dmmp] [-e INT] [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] - [--print-all] + [--print-all] [-j JOBS] optional arguments: -h, --help show this help message and exit @@ -247,6 +247,12 @@ optional arguments: Specify which parameters to hyperopt. Space separate list. Default: all. --print-all Print all results, not only the best ones. + -j JOBS, --job-workers JOBS + The number of concurrently running jobs for + hyperoptimization (hyperopt worker processes). If -1 + (default), all CPUs are used, for -2, all CPUs but one + are used, etc. If 1 is given, no parallel computing + code is used at all. ``` ## Edge commands diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 8ae3922ca..930342e5a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -306,6 +306,17 @@ class Arguments(object): dest='print_all', default=False ) + parser.add_argument( + '-j', '--job-workers', + help='The number of concurrently running jobs for hyperoptimization ' + '(hyperopt worker processes). ' + 'If -1 (default), all CPUs are used, for -2, all CPUs but one are used, etc. ' + 'If 1 is given, no parallel computing code is used at all.', + dest='hyperopt_jobs', + default=-1, + type=int, + metavar='JOBS', + ) parser.add_argument( '--random-state', help='Set random state to some positive integer for reproducible hyperopt results.', diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index ca3b81279..db9f0c98e 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -309,6 +309,10 @@ class Configuration(object): config.update({'print_all': self.args.print_all}) logger.info('Parameter --print-all detected: %s', config.get('print_all')) + if 'hyperopt_jobs' in self.args and self.args.hyperopt_jobs: + config.update({'hyperopt_jobs': self.args.hyperopt_jobs}) + logger.info('Parameter -j/--job-workers detected: %s', config.get('hyperopt_jobs')) + if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2f819773f..de2ab8a8a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,7 +5,6 @@ This module contains the hyperopt logic """ import logging -import multiprocessing import os import sys from argparse import Namespace @@ -15,7 +14,7 @@ from pathlib import Path from pprint import pprint from typing import Any, Dict, List -from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects +from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension @@ -275,28 +274,32 @@ class Hyperopt(Backtesting): self.load_previous_results() - cpus = multiprocessing.cpu_count() + cpus = cpu_count() logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') + config_jobs = self.config.get('hyperopt_jobs', -1) + logger.info(f'Number of parallel jobs set as: {config_jobs}') - opt = self.get_optimizer(cpus) - EVALS = max(self.total_tries // cpus, 1) + opt = self.get_optimizer(config_jobs) try: - with Parallel(n_jobs=cpus) as parallel: + with Parallel(n_jobs=config_jobs) as parallel: + jobs = parallel._effective_n_jobs() + logger.info(f'Effective number of parallel workers used: {jobs}') + EVALS = max(self.total_tries // jobs, 1) for i in range(EVALS): - asked = opt.ask(n_points=cpus) + asked = opt.ask(n_points=jobs) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val - for j in range(cpus): + for j in range(jobs): self.log_results({ 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, + 'current_tries': i * jobs + j, 'total_tries': self.total_tries, 'result': f_val[j]['result'], }) logger.debug(f"Optimizer params: {f_val[j]['params']}") - for j in range(cpus): + for j in range(jobs): logger.debug(f"Opimizer state: Xi: {opt.Xi[-j-1]}, yi: {opt.yi[-j-1]}") except KeyboardInterrupt: print('User interrupted..') diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 7151935b2..063d0e791 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -314,7 +314,6 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) @@ -325,6 +324,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: default_conf.update({'epochs': 1}) default_conf.update({'timerange': None}) default_conf.update({'spaces': 'all'}) + default_conf.update({'hyperopt_jobs': 1}) hyperopt = Hyperopt(default_conf) hyperopt.strategy.tickerdata_to_dataframe = MagicMock()