From 7c8e26c717fd61550d8ffe3077b2c4d3d0a7dcb4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 23 Apr 2019 00:30:09 +0300 Subject: [PATCH 1/6] -j/--job-workers option added for controlling the number of joblib parallel worker processes used in hyperopt docs refreshed --- docs/bot-usage.md | 19 ++++++++++++++++--- freqtrade/arguments.py | 11 +++++++++++ freqtrade/configuration.py | 18 ++++-------------- freqtrade/optimize/hyperopt.py | 16 ++++++++++------ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 55988985a..9261294e1 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -206,8 +206,11 @@ to find optimal parameter values for your stategy. ``` usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE] - [--customhyperopt NAME] [--eps] [--dmmp] [-e INT] - [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] + [--max_open_trades MAX_OPEN_TRADES] + [--stake_amount STAKE_AMOUNT] [--customhyperopt NAME] + [--eps] [--dmmp] [-e INT] + [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] + [--print-all] [-j JOBS] optional arguments: -h, --help show this help message and exit @@ -215,6 +218,10 @@ optional arguments: Specify ticker interval (1m, 5m, 30m, 1h, 1d). --timerange TIMERANGE Specify what timerange of data to use. + --max_open_trades MAX_OPEN_TRADES + Specify max_open_trades to use. + --stake_amount STAKE_AMOUNT + Specify stake_amount. --customhyperopt NAME Specify hyperopt class name (default: DefaultHyperOpts). @@ -229,7 +236,13 @@ optional arguments: -s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...], --spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate list. Default: all. - + --print-all Print all results, not only the best ones. + -j JOBS, --job-workers JOBS + The number of concurrently running jobs for + hyperoptimization (hyperopt worker processes). If -1 + (default), all CPUs are used, for -2, all CPUs but one + are used, etc. If 1 is given, no parallel computing + code is used at all. ``` ## Edge commands diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 96f080bd2..3631e6615 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -315,6 +315,17 @@ class Arguments(object): dest='print_all', default=False ) + parser.add_argument( + '-j', '--job-workers', + help='The number of concurrently running jobs for hyperoptimization ' + '(hyperopt worker processes). ' + 'If -1 (default), all CPUs are used, for -2, all CPUs but one are used, etc. ' + 'If 1 is given, no parallel computing code is used at all.', + dest='hyperopt_jobs', + default=-1, + type=int, + metavar='JOBS', + ) def _build_subcommands(self) -> None: """ diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index 65a8d644e..a13c24f6a 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -208,17 +208,14 @@ class Configuration(object): logger.info('Parameter -i/--ticker-interval detected ...') logger.info('Using ticker_interval: %s ...', config.get('ticker_interval')) - # If -l/--live is used we add it to the configuration if 'live' in self.args and self.args.live: config.update({'live': True}) logger.info('Parameter -l/--live detected ...') - # If --enable-position-stacking is used we add it to the configuration if 'position_stacking' in self.args and self.args.position_stacking: config.update({'position_stacking': True}) logger.info('Parameter --enable-position-stacking detected ...') - # If --disable-max-market-positions or --max_open_trades is used we update configuration if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions: config.update({'use_max_market_positions': False}) logger.info('Parameter --disable-max-market-positions detected ...') @@ -230,25 +227,21 @@ class Configuration(object): else: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) - # If --stake_amount is used we update configuration if 'stake_amount' in self.args and self.args.stake_amount: config.update({'stake_amount': self.args.stake_amount}) logger.info('Parameter --stake_amount detected, overriding stake_amount to: %s ...', config.get('stake_amount')) - # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: config.update({'timerange': self.args.timerange}) logger.info('Parameter --timerange detected: %s ...', self.args.timerange) - # If --datadir is used we add it to the configuration if 'datadir' in self.args and self.args.datadir: config.update({'datadir': self._create_datadir(config, self.args.datadir)}) else: config.update({'datadir': self._create_datadir(config, None)}) logger.info('Using data folder: %s ...', config.get('datadir')) - # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') @@ -261,12 +254,10 @@ class Configuration(object): config.update({'ticker_interval': self.args.ticker_interval}) logger.info('Overriding ticker interval with Command line argument') - # If --export is used we add it to the configuration if 'export' in self.args and self.args.export: config.update({'export': self.args.export}) logger.info('Parameter --export detected: %s ...', self.args.export) - # If --export-filename is used we add it to the configuration if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename: config.update({'exportfilename': self.args.exportfilename}) logger.info('Storing backtest results to %s ...', self.args.exportfilename) @@ -279,12 +270,10 @@ class Configuration(object): :return: configuration as dictionary """ - # If --timerange is used we add it to the configuration if 'timerange' in self.args and self.args.timerange: config.update({'timerange': self.args.timerange}) logger.info('Parameter --timerange detected: %s ...', self.args.timerange) - # If --timerange is used we add it to the configuration if 'stoploss_range' in self.args and self.args.stoploss_range: txt_range = eval(self.args.stoploss_range) config['edge'].update({'stoploss_range_min': txt_range[0]}) @@ -292,7 +281,6 @@ class Configuration(object): config['edge'].update({'stoploss_range_step': txt_range[2]}) logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range) - # If -r/--refresh-pairs-cached is used we add it to the configuration if 'refresh_pairs' in self.args and self.args.refresh_pairs: config.update({'refresh_pairs': True}) logger.info('Parameter -r/--refresh-pairs-cached detected ...') @@ -309,13 +297,11 @@ class Configuration(object): # Add the hyperopt file to use config.update({'hyperopt': self.args.hyperopt}) - # If --epochs is used we add it to the configuration if 'epochs' in self.args and self.args.epochs: config.update({'epochs': self.args.epochs}) logger.info('Parameter --epochs detected ...') logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs')) - # If --spaces is used we add it to the configuration if 'spaces' in self.args and self.args.spaces: config.update({'spaces': self.args.spaces}) logger.info('Parameter -s/--spaces detected: %s', config.get('spaces')) @@ -324,6 +310,10 @@ class Configuration(object): config.update({'print_all': self.args.print_all}) logger.info('Parameter --print-all detected: %s', config.get('print_all')) + if 'hyperopt_jobs' in self.args and self.args.hyperopt_jobs: + config.update({'hyperopt_jobs': self.args.hyperopt_jobs}) + logger.info('Parameter -j/--job-workers detected: %s', config.get('hyperopt_jobs')) + return config def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b37027244..2bcfdd499 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -270,21 +270,25 @@ class Hyperopt(Backtesting): cpus = multiprocessing.cpu_count() logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') + config_jobs = self.config.get('hyperopt_jobs', -1) + logger.info(f'Number of parallel jobs set as: {config_jobs}') - opt = self.get_optimizer(cpus) - EVALS = max(self.total_tries // cpus, 1) + opt = self.get_optimizer(config_jobs) try: - with Parallel(n_jobs=cpus) as parallel: + with Parallel(n_jobs=config_jobs) as parallel: + jobs = parallel._effective_n_jobs() + logger.info(f'Effective number of parallel workers used: {jobs}') + EVALS = max(self.total_tries // jobs, 1) for i in range(EVALS): - asked = opt.ask(n_points=cpus) + asked = opt.ask(n_points=jobs) f_val = self.run_optimizer_parallel(parallel, asked) opt.tell(asked, [i['loss'] for i in f_val]) self.trials += f_val - for j in range(cpus): + for j in range(jobs): self.log_results({ 'loss': f_val[j]['loss'], - 'current_tries': i * cpus + j, + 'current_tries': i * jobs + j, 'total_tries': self.total_tries, 'result': f_val[j]['result'], }) From cc9f899cd619e599a12eb235c0feaa0ef56b2e62 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Tue, 23 Apr 2019 21:25:36 +0300 Subject: [PATCH 2/6] removed explicit dependency on multiprocessing module --- freqtrade/optimize/hyperopt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2bcfdd499..312bdba98 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,7 +5,6 @@ This module contains the hyperopt logic """ import logging -import multiprocessing import os import sys from argparse import Namespace @@ -15,7 +14,7 @@ from pathlib import Path from pprint import pprint from typing import Any, Dict, List -from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects +from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects, cpu_count from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension @@ -268,7 +267,7 @@ class Hyperopt(Backtesting): self.exchange = None # type: ignore self.load_previous_results() - cpus = multiprocessing.cpu_count() + cpus = cpu_count() logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!') config_jobs = self.config.get('hyperopt_jobs', -1) logger.info(f'Number of parallel jobs set as: {config_jobs}') From 2f0ad0d28c2bf1b0ee075f2282cd0d23dab3d937 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Tue, 23 Apr 2019 22:03:41 +0300 Subject: [PATCH 3/6] test adjusted --- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 7151935b2..08107265d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -314,7 +314,7 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.multiprocessing.cpu_count', MagicMock(return_value=1)) + mocker.patch('freqtrade.optimize.hyperopt.joblib.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) From 6a0f527e0e8cdfee8db94a641ab69adb2965527c Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 24 Apr 2019 10:35:04 +0300 Subject: [PATCH 4/6] merge --job-workers and commit printing debug log messages with the opt state --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 58bcebf3f..de2ab8a8a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -299,7 +299,7 @@ class Hyperopt(Backtesting): 'result': f_val[j]['result'], }) logger.debug(f"Optimizer params: {f_val[j]['params']}") - for j in range(cpus): + for j in range(jobs): logger.debug(f"Opimizer state: Xi: {opt.Xi[-j-1]}, yi: {opt.yi[-j-1]}") except KeyboardInterrupt: print('User interrupted..') From 95ebd07735df241dda21465ba14d2e5f0da70b09 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 24 Apr 2019 10:38:50 +0300 Subject: [PATCH 5/6] an attempt to fix mocking --- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 08107265d..aaaa5c27d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -314,7 +314,7 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.joblib.cpu_count', MagicMock(return_value=1)) + mocker.patch('freqtrade.optimize.hyperopt.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) From a8e787fda887dac389a4bd290a57b60784c05cea Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Wed, 24 Apr 2019 11:25:15 +0300 Subject: [PATCH 6/6] test adjusted --- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index aaaa5c27d..063d0e791 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -314,7 +314,6 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock()) - mocker.patch('freqtrade.optimize.hyperopt.cpu_count', MagicMock(return_value=1)) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}]) @@ -325,6 +324,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: default_conf.update({'epochs': 1}) default_conf.update({'timerange': None}) default_conf.update({'spaces': 'all'}) + default_conf.update({'hyperopt_jobs': 1}) hyperopt = Hyperopt(default_conf) hyperopt.strategy.tickerdata_to_dataframe = MagicMock()