-j/--job-workers option added for controlling the number of joblib parallel worker processes used in hyperopt

docs refreshed

docs/bot-usage.md

@@ -206,8 +206,11 @@ to find optimal parameter values for your stategy.
 
 ```
 usage: freqtrade hyperopt [-h] [-i TICKER_INTERVAL] [--timerange TIMERANGE]
-                          [--customhyperopt NAME] [--eps] [--dmmp] [-e INT]
+                        [--max_open_trades MAX_OPEN_TRADES]
+                        [--stake_amount STAKE_AMOUNT] [--customhyperopt NAME]
+                        [--eps] [--dmmp] [-e INT]
                         [-s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]]
+                        [--print-all] [-j JOBS]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -215,6 +218,10 @@ optional arguments:
                         Specify ticker interval (1m, 5m, 30m, 1h, 1d).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
+  --max_open_trades MAX_OPEN_TRADES
+                        Specify max_open_trades to use.
+  --stake_amount STAKE_AMOUNT
+                        Specify stake_amount.
   --customhyperopt NAME
                         Specify hyperopt class name (default:
                         DefaultHyperOpts).
@@ -229,7 +236,13 @@ optional arguments:
   -s {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...], --spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]
                         Specify which parameters to hyperopt. Space separate
                         list. Default: all.
-
+  --print-all           Print all results, not only the best ones.
+  -j JOBS, --job-workers JOBS
+                        The number of concurrently running jobs for
+                        hyperoptimization (hyperopt worker processes). If -1
+                        (default), all CPUs are used, for -2, all CPUs but one
+                        are used, etc. If 1 is given, no parallel computing
+                        code is used at all.
 ```
 
 ## Edge commands

freqtrade/arguments.py

@@ -315,6 +315,17 @@ class Arguments(object):
             dest='print_all',
             default=False
         )
+        parser.add_argument(
+            '-j', '--job-workers',
+            help='The number of concurrently running jobs for hyperoptimization '
+                 '(hyperopt worker processes). '
+                 'If -1 (default), all CPUs are used, for -2, all CPUs but one are used, etc. '
+                 'If 1 is given, no parallel computing code is used at all.',
+            dest='hyperopt_jobs',
+            default=-1,
+            type=int,
+            metavar='JOBS',
+        )
 
     def _build_subcommands(self) -> None:
         """

freqtrade/configuration.py

@@ -208,17 +208,14 @@ class Configuration(object):
             logger.info('Parameter -i/--ticker-interval detected ...')
             logger.info('Using ticker_interval: %s ...', config.get('ticker_interval'))
 
-        # If -l/--live is used we add it to the configuration
         if 'live' in self.args and self.args.live:
             config.update({'live': True})
             logger.info('Parameter -l/--live detected ...')
 
-        # If --enable-position-stacking is used we add it to the configuration
         if 'position_stacking' in self.args and self.args.position_stacking:
             config.update({'position_stacking': True})
             logger.info('Parameter --enable-position-stacking detected ...')
 
-        # If --disable-max-market-positions or --max_open_trades is used we update configuration
         if 'use_max_market_positions' in self.args and not self.args.use_max_market_positions:
             config.update({'use_max_market_positions': False})
             logger.info('Parameter --disable-max-market-positions detected ...')
@@ -230,25 +227,21 @@ class Configuration(object):
         else:
             logger.info('Using max_open_trades: %s ...', config.get('max_open_trades'))
 
-        # If --stake_amount is used we update configuration
         if 'stake_amount' in self.args and self.args.stake_amount:
             config.update({'stake_amount': self.args.stake_amount})
             logger.info('Parameter --stake_amount detected, overriding stake_amount to: %s ...',
                         config.get('stake_amount'))
 
-        # If --timerange is used we add it to the configuration
         if 'timerange' in self.args and self.args.timerange:
             config.update({'timerange': self.args.timerange})
             logger.info('Parameter --timerange detected: %s ...', self.args.timerange)
 
-        # If --datadir is used we add it to the configuration
         if 'datadir' in self.args and self.args.datadir:
             config.update({'datadir': self._create_datadir(config, self.args.datadir)})
         else:
             config.update({'datadir': self._create_datadir(config, None)})
         logger.info('Using data folder: %s ...', config.get('datadir'))
 
-        # If -r/--refresh-pairs-cached is used we add it to the configuration
         if 'refresh_pairs' in self.args and self.args.refresh_pairs:
             config.update({'refresh_pairs': True})
             logger.info('Parameter -r/--refresh-pairs-cached detected ...')
@@ -261,12 +254,10 @@ class Configuration(object):
             config.update({'ticker_interval': self.args.ticker_interval})
             logger.info('Overriding ticker interval with Command line argument')
 
-        # If --export is used we add it to the configuration
         if 'export' in self.args and self.args.export:
             config.update({'export': self.args.export})
             logger.info('Parameter --export detected: %s ...', self.args.export)
 
-        # If --export-filename is used we add it to the configuration
         if 'export' in config and 'exportfilename' in self.args and self.args.exportfilename:
             config.update({'exportfilename': self.args.exportfilename})
             logger.info('Storing backtest results to %s ...', self.args.exportfilename)
@@ -279,12 +270,10 @@ class Configuration(object):
         :return: configuration as dictionary
         """
 
-        # If --timerange is used we add it to the configuration
         if 'timerange' in self.args and self.args.timerange:
             config.update({'timerange': self.args.timerange})
             logger.info('Parameter --timerange detected: %s ...', self.args.timerange)
 
-        # If --timerange is used we add it to the configuration
         if 'stoploss_range' in self.args and self.args.stoploss_range:
             txt_range = eval(self.args.stoploss_range)
             config['edge'].update({'stoploss_range_min': txt_range[0]})
@@ -292,7 +281,6 @@ class Configuration(object):
             config['edge'].update({'stoploss_range_step': txt_range[2]})
             logger.info('Parameter --stoplosses detected: %s ...', self.args.stoploss_range)
 
-        # If -r/--refresh-pairs-cached is used we add it to the configuration
         if 'refresh_pairs' in self.args and self.args.refresh_pairs:
             config.update({'refresh_pairs': True})
             logger.info('Parameter -r/--refresh-pairs-cached detected ...')
@@ -309,13 +297,11 @@ class Configuration(object):
             # Add the hyperopt file to use
             config.update({'hyperopt': self.args.hyperopt})
 
-        # If --epochs is used we add it to the configuration
         if 'epochs' in self.args and self.args.epochs:
             config.update({'epochs': self.args.epochs})
             logger.info('Parameter --epochs detected ...')
             logger.info('Will run Hyperopt with for %s epochs ...', config.get('epochs'))
 
-        # If --spaces is used we add it to the configuration
         if 'spaces' in self.args and self.args.spaces:
             config.update({'spaces': self.args.spaces})
             logger.info('Parameter -s/--spaces detected: %s', config.get('spaces'))
@@ -324,6 +310,10 @@ class Configuration(object):
             config.update({'print_all': self.args.print_all})
             logger.info('Parameter --print-all detected: %s', config.get('print_all'))
 
+        if 'hyperopt_jobs' in self.args and self.args.hyperopt_jobs:
+            config.update({'hyperopt_jobs': self.args.hyperopt_jobs})
+            logger.info('Parameter -j/--job-workers detected: %s', config.get('hyperopt_jobs'))
+
         return config
 
     def _validate_config_schema(self, conf: Dict[str, Any]) -> Dict[str, Any]:

freqtrade/optimize/hyperopt.py

@@ -270,21 +270,25 @@ class Hyperopt(Backtesting):
 
         cpus = multiprocessing.cpu_count()
         logger.info(f'Found {cpus} CPU cores. Let\'s make them scream!')
+        config_jobs = self.config.get('hyperopt_jobs', -1)
+        logger.info(f'Number of parallel jobs set as: {config_jobs}')
 
-        opt = self.get_optimizer(cpus)
+        opt = self.get_optimizer(config_jobs)
-        EVALS = max(self.total_tries // cpus, 1)
         try:
-            with Parallel(n_jobs=cpus) as parallel:
+            with Parallel(n_jobs=config_jobs) as parallel:
+                jobs = parallel._effective_n_jobs()
+                logger.info(f'Effective number of parallel workers used: {jobs}')
+                EVALS = max(self.total_tries // jobs, 1)
                 for i in range(EVALS):
-                    asked = opt.ask(n_points=cpus)
+                    asked = opt.ask(n_points=jobs)
                     f_val = self.run_optimizer_parallel(parallel, asked)
                     opt.tell(asked, [i['loss'] for i in f_val])
 
                     self.trials += f_val
-                    for j in range(cpus):
+                    for j in range(jobs):
                         self.log_results({
                             'loss': f_val[j]['loss'],
-                            'current_tries': i * cpus + j,
+                            'current_tries': i * jobs + j,
                             'total_tries': self.total_tries,
                             'result': f_val[j]['result'],
                         })