diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 4a87def88..c104c5d8d 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -18,15 +18,16 @@ ARGS_TRADE = ["db_url", "sd_notify", "dry_run"] ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "max_open_trades", "stake_amount", "fee"] -ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "strategy_list", "export", "exportfilename"] +ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + [ + "position_stacking", "use_max_market_positions", "strategy_list", "export", "exportfilename" +] -ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", - "position_stacking", "epochs", "spaces", - "use_max_market_positions", "print_all", - "print_colorized", "print_json", "hyperopt_jobs", - "hyperopt_random_state", "hyperopt_min_trades", - "hyperopt_continue", "hyperopt_loss"] +ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + [ + "hyperopt", "hyperopt_path", "position_stacking", "epochs", "spaces", + "use_max_market_positions", "print_all", "print_colorized", "print_json", "hyperopt_jobs", + "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_continue", "hyperopt_loss", "effort", + "mode", "n_points", "lie_strat" +] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 8eb5c3ce8..8b92852f9 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -195,6 +195,36 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=constants.HYPEROPT_EPOCH, ), + "effort": Arg( + '--effort', + help=('The higher the number, the longer will be the search if' + 'no epochs are defined (default: %(default)d).'), + type=float, + metavar='FLOAT', + default=constants.HYPEROPT_EFFORT, + ), + "mode": Arg( + '--mode', + help='Switches hyperopt to use one optimizer per job, use it' + 'when backtesting iterations are cheap (default: %(default)s).', + metavar='NAME', + default=constants.HYPEROPT_MODE), + "n_points": Arg( + '--n-points', + help='Controls how many points to ask to the optimizer ' + 'increase if cpu usage of each core ' + 'appears low (default: %(default)d).', + type=int, + metavar='INT', + default=constants.HYPEROPT_N_POINTS + ), + "lie_strat": Arg( + '--lie-strat', + help='Sets the strategy that the optimizer uses to lie ' + 'when asking for more than one point, ' + 'no effect if n_point is one (default: %(default)s).', + default=constants.HYPEROPT_LIE_STRAT + ), "spaces": Arg( '--spaces', help='Specify which parameters to hyperopt. Space-separated list.', diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 2fc605926..a15bcf4e2 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -59,6 +59,7 @@ def start_hyperopt(args: Dict[str, Any]) -> None: try: from filelock import FileLock, Timeout from freqtrade.optimize.hyperopt import Hyperopt + from freqtrade.optimize import hyperopt_backend as backend except ImportError as e: raise OperationalException( f"{e}. Please ensure that the hyperopt dependencies are installed.") from e @@ -77,8 +78,8 @@ def start_hyperopt(args: Dict[str, Any]) -> None: logging.getLogger('filelock').setLevel(logging.WARNING) # Initialize backtesting object - hyperopt = Hyperopt(config) - hyperopt.start() + backend.hyperopt = Hyperopt(config) + backend.hyperopt.start() except Timeout: logger.info("Another running instance of freqtrade Hyperopt detected.") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 08a600176..82a6be3cf 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -265,10 +265,22 @@ class Configuration: self._args_to_config(config, argname='epochs', logstring='Parameter --epochs detected ... ' - 'Will run Hyperopt with for {} epochs ...' - ) - - self._args_to_config(config, argname='spaces', + 'Will run Hyperopt with for {} epochs ...') + self._args_to_config(config, + argname='effort', + logstring='Parameter --effort detected ... ' + 'Parameter --effort detected: {}') + self._args_to_config(config, + argname='mode', + logstring='Hyperopt will run in {} mode ...') + self._args_to_config(config, + argname='explore', + logstring='Acquisition strategy set to random {}...') + self._args_to_config(config, + argname='n_points', + logstring='Optimizers will be asked for {} points...') + self._args_to_config(config, + argname='spaces', logstring='Parameter -s/--spaces detected: {}') self._args_to_config(config, argname='print_all', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1dadc6e16..12f50ffc1 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -8,9 +8,13 @@ from typing import List, Tuple DEFAULT_CONFIG = 'config.json' DEFAULT_EXCHANGE = 'bittrex' -PROCESS_THROTTLE_SECS = 5 # sec -HYPEROPT_EPOCH = 100 # epochs -RETRY_TIMEOUT = 30 # sec +PROCESS_THROTTLE_SECS = 5 # sec +HYPEROPT_EPOCH = 0 # epochs +HYPEROPT_EFFORT = 0. # tune max epoch count +HYPEROPT_N_POINTS = 1 # tune iterations between estimations +HYPEROPT_MODE = 'single' +HYPEROPT_LIE_STRAT = 'default' +RETRY_TIMEOUT = 30 # sec DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6d11e543b..9e1daf1c2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -1,38 +1,45 @@ # pragma pylint: disable=too-many-instance-attributes, pointless-string-statement - """ This module contains the hyperopt logic """ +import io import locale import logging +import os import random +import sys import warnings -from math import ceil -from collections import OrderedDict +from collections import OrderedDict, deque +from math import factorial, log +from multiprocessing import Manager from operator import itemgetter +from os import path from pathlib import Path from pprint import pformat -from typing import Any, Dict, List, Optional +from queue import Queue +from typing import Any, Dict, List, Optional, Set, Tuple -import rapidjson -from colorama import Fore, Style -from joblib import (Parallel, cpu_count, delayed, dump, load, - wrap_non_picklable_objects) -from pandas import DataFrame, json_normalize, isna import progressbar +import rapidjson import tabulate -from os import path -import io +from colorama import Fore, Style +from colorama import init as colorama_init +from joblib import (Parallel, cpu_count, delayed, dump, load, parallel_backend, + wrap_non_picklable_objects) +from numpy import iinfo, int32 +from pandas import DataFrame, isna, json_normalize +# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules +import freqtrade.optimize.hyperopt_backend as backend from freqtrade.data.converter import trim_dataframe from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException from freqtrade.misc import plural, round_dict from freqtrade.optimize.backtesting import Backtesting -# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 -from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 +from freqtrade.optimize.hyperopt_loss_interface import \ + IHyperOptLoss # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, HyperOptResolver) @@ -41,19 +48,26 @@ with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FutureWarning) from skopt import Optimizer from skopt.space import Dimension +# Additional regressors already pluggable into the optimizer +# from sklearn.linear_model import ARDRegression, BayesianRidge +# possibly interesting regressors that need predict method override +# from sklearn.ensemble import HistGradientBoostingRegressor +# from xgboost import XGBoostRegressor + progressbar.streams.wrap_stderr() progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) +# supported strategies when asking for multiple points to the optimizer +LIE_STRATS = ["cl_min", "cl_mean", "cl_max"] +LIE_STRATS_N = len(LIE_STRATS) -INITIAL_POINTS = 30 +# supported estimators +ESTIMATORS = ["GBRT", "ET", "RF"] +ESTIMATORS_N = len(ESTIMATORS) -# Keep no more than SKOPT_MODEL_QUEUE_SIZE models -# in the skopt model queue, to optimize memory consumption -SKOPT_MODEL_QUEUE_SIZE = 10 - -MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization +VOID_LOSS = iinfo(int32).max # just a big enough number to be bad result in loss optimization class Hyperopt: @@ -64,8 +78,8 @@ class Hyperopt: hyperopt = Hyperopt(config) hyperopt.start() """ - def __init__(self, config: Dict[str, Any]) -> None: + self.config = config self.backtesting = Backtesting(self.config) @@ -75,13 +89,34 @@ class Hyperopt: self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - self.results_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - self.data_pickle_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_tickerdata.pkl') - self.total_epochs = config.get('epochs', 0) + self.results_file = (self.config['user_data_dir'] / 'hyperopt_results' / + 'hyperopt_results.pickle') + self.opts_file = (self.config['user_data_dir'] / 'hyperopt_results' / + 'hyperopt_optimizers.pickle') + self.data_pickle_file = (self.config['user_data_dir'] / 'hyperopt_results' / + 'hyperopt_tickerdata.pkl') - self.current_best_loss = 100 + self.n_jobs = self.config.get('hyperopt_jobs', -1) + if self.n_jobs < 0: + self.n_jobs = cpu_count() // 2 or 1 + self.effort = max(0.01, + self.config['effort'] if 'effort' in self.config else 1 + ) + self.total_epochs = self.config['epochs'] if 'epochs' in self.config else 0 + self.max_epoch = 0 + self.max_epoch_reached = False + self.min_epochs = 0 + self.epochs_limit = lambda: self.total_epochs or self.max_epoch + + # a guessed number extracted by the space dimensions + self.search_space_size = 0 + # total number of candles being backtested + self.n_candles = 0 + + self.current_best_loss = VOID_LOSS + self.current_best_epoch = 0 + self.epochs_since_last_best: List = [] + self.avg_best_occurrence = 0 if not self.config.get('hyperopt_continue'): self.clean_hyperopt() @@ -90,8 +125,11 @@ class Hyperopt: self.num_epochs_saved = 0 - # Previous evaluations - self.epochs: List = [] + # evaluations + self.trials: List = [] + + # configure multi mode + self.setup_multi() # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): @@ -123,6 +161,67 @@ class Hyperopt: self.print_colorized = self.config.get('print_colorized', False) self.print_json = self.config.get('print_json', False) + def setup_multi(self): + # optimizers + self.opts: List[Optimizer] = [] + self.opt: Optimizer = None + self.Xi: Dict = {} + self.yi: Dict = {} + + backend.manager = Manager() + self.mode = self.config.get('mode', 'single') + self.shared = False + # in multi opt one model is enough + self.n_models = 1 + if self.mode in ('multi', 'shared'): + self.multi = True + if self.mode == 'shared': + self.shared = True + self.opt_base_estimator = lambda: 'GBRT' + else: + self.opt_base_estimator = self.estimators + self.opt_acq_optimizer = 'sampling' + backend.optimizers = backend.manager.Queue() + backend.results_batch = backend.manager.Queue() + else: + self.multi = False + backend.results_list = backend.manager.list([]) + # this is where opt_ask_and_tell stores the results after points are + # used for fit and predict, to avoid additional pickling + self.batch_results = [] + # self.opt_base_estimator = lambda: BayesianRidge(n_iter=100, normalize=True) + self.opt_acq_optimizer = 'sampling' + self.opt_base_estimator = lambda: 'ET' + # The GaussianProcessRegressor is heavy, which makes it not a good default + # however longer backtests might make it a better tradeoff + # self.opt_base_estimator = lambda: 'GP' + # self.opt_acq_optimizer = 'lbfgs' + + # in single opt assume runs are expensive so default to 1 point per ask + self.n_points = self.config.get('n_points', 1) + # if 0 n_points are given, don't use any base estimator (akin to random search) + if self.n_points < 1: + self.n_points = 1 + self.opt_base_estimator = lambda: "DUMMY" + self.opt_acq_optimizer = "sampling" + if self.n_points < 2: + # ask_points is what is used in the ask call + # because when n_points is None, it doesn't + # waste time generating new points + self.ask_points = None + else: + self.ask_points = self.n_points + # var used in epochs and batches calculation + self.opt_points = self.n_jobs * (self.n_points or 1) + # lie strategy + lie_strat = self.config.get('lie_strat', 'default') + if lie_strat == 'default': + self.lie_strat = lambda: 'cl_min' + elif lie_strat == 'random': + self.lie_strat = self.lie_strategy + else: + self.lie_strat = lambda: lie_strat + @staticmethod def get_lock_filename(config: Dict[str, Any]) -> str: @@ -132,7 +231,7 @@ class Hyperopt: """ Remove hyperopt pickle files to restart hyperopt. """ - for f in [self.data_pickle_file, self.results_file]: + for f in [self.data_pickle_file, self.results_file, self.opts_file]: p = Path(f) if p.is_file(): logger.info(f"Removing `{p}`.") @@ -160,9 +259,39 @@ class Hyperopt: logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.") dump(self.epochs, self.results_file) self.num_epochs_saved = num_epochs + self.save_opts() logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " f"saved to '{self.results_file}'.") + def save_opts(self) -> None: + """ + Save optimizers state to disk. The minimum required state could also be constructed + from the attributes [ models, space, rng ] with Xi, yi loaded from trials. + All we really care about are [rng, Xi, yi] since models are never passed over queues + and space is dependent on dimensions matching with hyperopt config + """ + # synchronize with saved trials + opts = [] + n_opts = 0 + if self.multi: + while not backend.optimizers.empty(): + opt = backend.optimizers.get() + opt = Hyperopt.opt_clear(opt) + opts.append(opt) + n_opts = len(opts) + for opt in opts: + backend.optimizers.put(opt) + else: + # when we clear the object for saving we have to make a copy to preserve state + opt = Hyperopt.opt_rand(self.opt, seed=False) + if self.opt: + n_opts = 1 + opts = [Hyperopt.opt_clear(self.opt)] + # (the optimizer copy function also fits a new model with the known points) + self.opt = opt + logger.debug(f"Saving {n_opts} {plural(n_opts, 'optimizer')}.") + dump(opts, self.opts_file) + @staticmethod def _read_results(results_file: Path) -> List: """ @@ -294,7 +423,7 @@ class Hyperopt: @staticmethod def _format_explanation_string(results, total_epochs) -> str: - return (("*" if results['is_initial_point'] else " ") + + return (("*" if 'is_initial_point' in results and results['is_initial_point'] else " ") + f"{results['current_epoch']:5d}/{total_epochs}: " + f"{results['results_explanation']} " + f"Objective: {results['loss']:.5f}") @@ -490,7 +619,7 @@ class Hyperopt: return spaces - def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: + def backtest_params(self, raw_params: List[Any], iteration=None) -> Dict: """ Used Optimize function. Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! @@ -534,11 +663,11 @@ class Hyperopt: max_open_trades=self.max_open_trades, position_stacking=self.position_stacking, ) - return self._get_results_dict(backtesting_results, min_date, max_date, - params_dict, params_details) + return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, + params_details) - def _get_results_dict(self, backtesting_results, min_date, max_date, - params_dict, params_details): + def _get_results_dict(self, backtesting_results, min_date, max_date, params_dict, + params_details): results_metrics = self._calculate_results_metrics(backtesting_results) results_explanation = self._format_results_explanation_string(results_metrics) @@ -549,7 +678,7 @@ class Hyperopt: # interesting -- consider it as 'bad' (assigned max. loss value) # in order to cast this hyperspace point away from optimization # path. We do not want to optimize 'hodl' strategies. - loss: float = MAX_LOSS + loss: float = VOID_LOSS if trade_count >= self.config['hyperopt_min_trades']: loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count, min_date=min_date.datetime, max_date=max_date.datetime) @@ -594,20 +723,332 @@ class Hyperopt: f"Avg duration {results_metrics['duration']:5.1f} min." ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8') - def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: + @staticmethod + def filter_void_losses(vals: List, opt: Optimizer) -> List: + """ remove out of bound losses from the results """ + if opt.void_loss == VOID_LOSS and len(opt.yi) < 1: + # only exclude results at the beginning when void loss is yet to be set + void_filtered = list(filter(lambda v: v["loss"] != VOID_LOSS, vals)) + else: + if opt.void_loss == VOID_LOSS: # set void loss once + opt.void_loss = max(opt.yi) + void_filtered = [] + # default bad losses to set void_loss + for k, v in enumerate(vals): + if v["loss"] == VOID_LOSS: + vals[k]["loss"] = opt.void_loss + void_filtered = vals + return void_filtered + + def lie_strategy(self): + """ Choose a strategy randomly among the supported ones, used in multi opt mode + to increase the diversion of the searches of each optimizer """ + return LIE_STRATS[random.randrange(0, LIE_STRATS_N)] + + def estimators(self): + return ESTIMATORS[random.randrange(0, ESTIMATORS_N)] + + def get_optimizer(self, random_state: int = None) -> Optimizer: + " Construct an optimizer object " + # https://github.com/scikit-learn/scikit-learn/issues/14265 + # lbfgs uses joblib threading backend so n_jobs has to be reduced + # to avoid oversubscription + if self.opt_acq_optimizer == 'lbfgs': + n_jobs = 1 + else: + n_jobs = self.n_jobs return Optimizer( - dimensions, - base_estimator="ET", - acq_optimizer="auto", - n_initial_points=INITIAL_POINTS, - acq_optimizer_kwargs={'n_jobs': cpu_count}, - random_state=self.random_state, - model_queue_size=SKOPT_MODEL_QUEUE_SIZE, + self.dimensions, + base_estimator=self.opt_base_estimator(), + acq_optimizer=self.opt_acq_optimizer, + n_initial_points=self.opt_n_initial_points, + acq_optimizer_kwargs={'n_jobs': n_jobs}, + model_queue_size=self.n_models, + random_state=random_state or self.random_state, ) - def run_optimizer_parallel(self, parallel, asked, i) -> List: - return parallel(delayed( - wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) + def run_backtest_parallel(self, parallel: Parallel, tries: int, first_try: int, + jobs: int): + """ launch parallel in single opt mode, return the evaluated epochs """ + parallel( + delayed(wrap_non_picklable_objects(self.parallel_objective)) + (asked, backend.results_list, i) + for asked, i in zip(self.opt_ask_and_tell(jobs, tries), + range(first_try, first_try + tries))) + + def run_multi_backtest_parallel(self, parallel: Parallel, tries: int, first_try: int, + jobs: int): + """ launch parallel in multi opt mode, return the evaluated epochs""" + parallel( + delayed(wrap_non_picklable_objects(self.parallel_opt_objective))( + i, backend.optimizers, jobs, backend.results_shared, backend.results_batch) + for i in range(first_try, first_try + tries)) + + def opt_ask_and_tell(self, jobs: int, tries: int): + """ + loop to manage optimizer state in single optimizer mode, everytime a job is + dispatched, we check the optimizer for points, to ask and to tell if any, + but only fit a new model every n_points, because if we fit at every result previous + points become invalid. + """ + vals = [] + fit = False + to_ask: deque = deque() + evald: Set[Tuple] = set() + opt = self.opt + + # this is needed because when we ask None points, the optimizer doesn't return a list + if self.ask_points: + def point(): + if to_ask: + return tuple(to_ask.popleft()) + else: + to_ask.extend(opt.ask(n_points=self.ask_points, strategy=self.lie_strat())) + return tuple(to_ask.popleft()) + else: + def point(): + return tuple(opt.ask(strategy=self.lie_strat())) + + for r in range(tries): + fit = (len(to_ask) < 1) + if len(backend.results_list) > 0: + vals.extend(backend.results_list) + del backend.results_list[:] + if vals: + # filter losses + void_filtered = Hyperopt.filter_void_losses(vals, opt) + if void_filtered: # again if all are filtered + opt.tell([Hyperopt.params_Xi(v) for v in void_filtered], + [v['loss'] for v in void_filtered], + fit=fit) # only fit when out of points + self.batch_results.extend(void_filtered) + del vals[:], void_filtered[:] + + a = point() + # this usually happens at the start when trying to fit before the initial points + if a in evald: + logger.debug("this point was evaluated before...") + opt.update_next() + a = point() + if a in evald: + break + evald.add(a) + yield a + + @staticmethod + def opt_get_past_points(is_shared: bool, asked: dict, results_shared: Dict) -> Tuple[dict, int]: + """ fetch shared results between optimizers """ + # a result is (y, counter) + for a in asked: + if a in results_shared: + y, counter = results_shared[a] + asked[a] = y + counter -= 1 + if counter < 1: + del results_shared[a] + return asked, len(results_shared) + + @staticmethod + def opt_rand(opt: Optimizer, rand: int = None, seed: bool = True) -> Optimizer: + """ return a new instance of the optimizer with modified rng """ + if seed: + if not rand: + rand = opt.rng.randint(0, VOID_LOSS) + opt.rng.seed(rand) + opt, opt.void_loss, opt.void, opt.rs = ( + opt.copy(random_state=opt.rng), opt.void_loss, opt.void, opt.rs + ) + return opt + + @staticmethod + def opt_state(shared: bool, optimizers: Queue) -> Optimizer: + """ fetch an optimizer in multi opt mode """ + # get an optimizer instance + opt = optimizers.get() + if shared: + # get a random number before putting it back to avoid + # replication with other workers and keep reproducibility + rand = opt.rng.randint(0, VOID_LOSS) + optimizers.put(opt) + # switch the seed to get a different point + opt = Hyperopt.opt_rand(opt, rand) + return opt + + @staticmethod + def opt_clear(opt: Optimizer): + """ clear state from an optimizer object """ + del opt.models[:], opt.Xi[:], opt.yi[:] + return opt + + @staticmethod + def opt_results(opt: Optimizer, void_filtered: list, jobs: int, is_shared: bool, + results_shared: Dict, results_batch: Queue, optimizers: Queue): + """ + update the board used to skip already computed points, + set the initial point status + """ + # add points of the current dispatch if any + if opt.void_loss != VOID_LOSS or len(void_filtered) > 0: + void = False + else: + void = True + # send back the updated optimizer only in non shared mode + if not is_shared: + opt = Hyperopt.opt_clear(opt) + # is not a replica in shared mode + optimizers.put(opt) + # NOTE: some results at the beginning won't be published + # because they are removed by filter_void_losses + rs = opt.rs + if not void: + # the tuple keys are used to avoid computation of done points by any optimizer + results_shared.update({tuple(Hyperopt.params_Xi(v)): (v["loss"], jobs - 1) + for v in void_filtered}) + # in multi opt mode (non shared) also track results for each optimizer (using rs as ID) + # this keys should be cleared after each batch + Xi, yi = results_shared[rs] + Xi = Xi + tuple((Hyperopt.params_Xi(v)) for v in void_filtered) + yi = yi + tuple(v["loss"] for v in void_filtered) + results_shared[rs] = (Xi, yi) + # this is the counter used by the optimizer internally to track the initial + # points evaluated so far.. + initial_points = opt._n_initial_points + # set initial point flag and optimizer random state + for n, v in enumerate(void_filtered): + v['is_initial_point'] = initial_points - n > 0 + v['random_state'] = rs + results_batch.put(void_filtered) + + def parallel_opt_objective(self, n: int, optimizers: Queue, jobs: int, + results_shared: Dict, results_batch: Queue): + """ + objective run in multi opt mode, optimizers share the results as soon as they are completed + """ + self.log_results_immediate(n) + is_shared = self.shared + opt = self.opt_state(is_shared, optimizers) + sss = self.search_space_size + asked: Dict[Tuple, Any] = {tuple([]): None} + asked_d: Dict[Tuple, Any] = {} + + # fit a model with the known points, (the optimizer has no points here since + # it was just fetched from the queue) + rs = opt.rs + Xi, yi = self.Xi[rs], self.yi[rs] + # add the points discovered within this batch + bXi, byi = results_shared[rs] + Xi.extend(list(bXi)) + yi.extend(list(byi)) + if Xi: + opt.tell(Xi, yi) + told = 0 # told + Xi_d = [] # done + yi_d = [] + Xi_t = [] # to do + # if opt.void == -1 the optimizer failed to give a new point (between dispatches), stop + # if asked == asked_d the points returned are the same, stop + # if opt.Xi > sss the optimizer has more points than the estimated search space size, stop + while opt.void != -1 and asked != asked_d and len(opt.Xi) < sss: + asked_d = asked + asked = opt.ask(n_points=self.ask_points, strategy=self.lie_strat()) + if not self.ask_points: + asked = {tuple(asked): None} + else: + asked = {tuple(a): None for a in asked} + # check if some points have been evaluated by other optimizers + p_asked, _ = Hyperopt.opt_get_past_points(is_shared, asked, results_shared) + for a in p_asked: + if p_asked[a] is not None: + if a not in Xi_d: + Xi_d.append(a) + yi_d.append(p_asked[a]) + else: + Xi_t.append(a) + # no points to do? + if len(Xi_t) < self.n_points: + len_Xi_d = len(Xi_d) + # did other workers backtest some points? + if len_Xi_d > told: + # if yes fit a new model with the new points + opt.tell(Xi_d[told:], yi_d[told:]) + told = len_Xi_d + else: # or get new points from a different random state + opt = Hyperopt.opt_rand(opt) + else: + break + # return early if there is nothing to backtest + if len(Xi_t) < 1: + if is_shared: + opt = optimizers.get() + opt.void = -1 + opt = Hyperopt.opt_clear(opt) + optimizers.put(opt) + return [] + # run the backtest for each point to do (Xi_t) + results = [self.backtest_params(a) for a in Xi_t] + # filter losses + void_filtered = Hyperopt.filter_void_losses(results, opt) + + Hyperopt.opt_results(opt, void_filtered, jobs, is_shared, + results_shared, results_batch, optimizers) + + def parallel_objective(self, asked, results_list: List = [], n=0): + """ objective run in single opt mode, run the backtest, store the results into a queue """ + self.log_results_immediate(n) + v = self.backtest_params(asked) + + v['is_initial_point'] = n < self.opt_n_initial_points + v['random_state'] = self.random_state + results_list.append(v) + + def log_results_immediate(self, n) -> None: + """ Signals that a new job has been scheduled""" + print('.', end='') + sys.stdout.flush() + + def log_results(self, batch_results, frame_start, total_epochs: int) -> int: + """ + Log results if it is better than any previous evaluation + """ + current = frame_start + 1 + i = 0 + for i, v in enumerate(batch_results, 1): + is_best = self.is_best_loss(v, self.current_best_loss) + current = frame_start + i + v['is_best'] = is_best + v['current_epoch'] = current + logger.debug(f"Optimizer epoch evaluated: {v}") + if is_best: + self.current_best_loss = v['loss'] + self.update_max_epoch(v, current) + self.print_results(v) + self.trials.append(v) + # Save results and optimizers after every batch + self._save_results() + # track new points if in multi mode + if self.multi: + self.track_points(trials=self.trials[frame_start:]) + # clear points used by optimizers intra batch + backend.results_shared.update(self.opt_empty_tuple()) + # give up if no best since max epochs + if current + 1 > self.epochs_limit(): + self.max_epoch_reached = True + return i + + def setup_epochs(self) -> bool: + """ used to resume the best epochs state from previous trials """ + len_trials = len(self.trials) + if len_trials > 0: + best_epochs = list(filter(lambda k: k["is_best"], self.trials)) + len_best = len(best_epochs) + if len_best > 0: + # sorting from lowest to highest, the first value is the current best + best = sorted(best_epochs, key=lambda k: k["loss"])[0] + self.current_best_epoch = best["current_epoch"] + self.current_best_loss = best["loss"] + self.avg_best_occurrence = len_trials // len_best + return True + return False @staticmethod def load_previous_results(results_file: Path) -> List: @@ -625,10 +1066,267 @@ class Hyperopt: logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") return epochs + @staticmethod + def load_previous_optimizers(opts_file: Path) -> List: + """ Load the state of previous optimizers from file """ + opts: List[Optimizer] = [] + if opts_file.is_file() and opts_file.stat().st_size > 0: + opts = load(opts_file) + n_opts = len(opts) + if n_opts > 0 and type(opts[-1]) != Optimizer: + raise OperationalException("The file storing optimizers state might be corrupted " + "and cannot be loaded.") + else: + logger.info(f"Loaded {n_opts} previous {plural(n_opts, 'optimizer')} from disk.") + return opts + def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) + @staticmethod + def calc_epochs( + dimensions: List[Dimension], n_jobs: int, effort: float, total_epochs: int, n_points: int + ): + """ Compute a reasonable number of initial points and + a minimum number of epochs to evaluate """ + n_dimensions = len(dimensions) + n_parameters = 0 + opt_points = n_jobs * n_points + # sum all the dimensions discretely, granting minimum values + for d in dimensions: + if type(d).__name__ == 'Integer': + n_parameters += max(1, d.high - d.low) + elif type(d).__name__ == 'Real': + n_parameters += max(10, int(d.high - d.low)) + else: + n_parameters += len(d.bounds) + # guess the size of the search space as the count of the + # unordered combination of the dimensions entries + try: + search_space_size = int( + (factorial(n_parameters) / + (factorial(n_parameters - n_dimensions) * factorial(n_dimensions)))) + except OverflowError: + search_space_size = VOID_LOSS + # logger.info(f'Search space size: {search_space_size}') + log_opt = int(log(opt_points, 2)) if opt_points > 4 else 2 + if search_space_size < opt_points: + # don't waste if the space is small + n_initial_points = opt_points // 3 + min_epochs = opt_points + elif total_epochs > 0: + # coefficients from total epochs + log_epp = int(log(total_epochs, 2)) * log_opt + n_initial_points = min(log_epp, total_epochs // 3) + min_epochs = total_epochs + else: + # extract coefficients from the search space + log_sss = int(log(search_space_size, 10)) * log_opt + # never waste + n_initial_points = min(log_sss, search_space_size // 3) + # it shall run for this much, I say + min_epochs = int(max(n_initial_points, opt_points) + 2 * n_initial_points) + return int(n_initial_points * effort) or 1, int(min_epochs * effort), search_space_size + + def update_max_epoch(self, val: Dict, current: int): + """ calculate max epochs: store the number of non best epochs + between each best, and get the mean of that value """ + if val['is_initial_point'] is not True: + self.epochs_since_last_best.append(current - self.current_best_epoch) + self.avg_best_occurrence = (sum(self.epochs_since_last_best) // + len(self.epochs_since_last_best)) + self.current_best_epoch = current + self.max_epoch = int( + (self.current_best_epoch + self.avg_best_occurrence + self.min_epochs) * + max(1, self.effort)) + if self.max_epoch > self.search_space_size: + self.max_epoch = self.search_space_size + logger.debug(f'\nMax epoch set to: {self.epochs_limit()}') + + @staticmethod + def params_Xi(v: dict): + return list(v["params_dict"].values()) + + def track_points(self, trials: List = None): + """ + keep tracking of the evaluated points per optimizer random state + """ + # if no trials are given, use saved trials + if not trials: + if len(self.trials) > 0: + if self.config.get('hyperopt_continue_filtered', False): + raise ValueError() + # trials = filter_trials(self.trials, self.config) + else: + trials = self.trials + else: + return + for v in trials: + rs = v["random_state"] + try: + self.Xi[rs].append(Hyperopt.params_Xi(v)) + self.yi[rs].append(v["loss"]) + except IndexError: # Hyperopt was started with different random_state or number of jobs + pass + + def setup_optimizers(self): + """ Setup the optimizers objects, try to load from disk, or create new ones """ + # try to load previous optimizers + opts = self.load_previous_optimizers(self.opts_file) + n_opts = len(opts) + + if self.multi: + max_opts = self.n_jobs + rngs = [] + # when sharing results there is only one optimizer that gets copied + if self.shared: + max_opts = 1 + # put the restored optimizers in the queue + # only if they match the current number of jobs + if n_opts == max_opts: + for n in range(n_opts): + rngs.append(opts[n].rs) + # make sure to not store points and models in the optimizer + backend.optimizers.put(Hyperopt.opt_clear(opts[n])) + # generate as many optimizers as are still needed to fill the job count + remaining = max_opts - backend.optimizers.qsize() + if remaining > 0: + opt = self.get_optimizer() + rngs = [] + for _ in range(remaining): # generate optimizers + # random state is preserved + rs = opt.rng.randint(0, iinfo(int32).max) + opt_copy = opt.copy(random_state=rs) + opt_copy.void_loss = VOID_LOSS + opt_copy.void = False + opt_copy.rs = rs + rngs.append(rs) + backend.optimizers.put(opt_copy) + del opt, opt_copy + # reconstruct observed points from epochs + # in shared mode each worker will remove the results once all the workers + # have read it (counter < 1) + counter = self.n_jobs + + def empty_dict(): + return {rs: [] for rs in rngs} + self.opt_empty_tuple = lambda: {rs: ((), ()) for rs in rngs} + self.Xi.update(empty_dict()) + self.yi.update(empty_dict()) + self.track_points() + # this is needed to keep track of results discovered within the same batch + # by each optimizer, use tuples! as the SyncManager doesn't handle nested dicts + Xi, yi = self.Xi, self.yi + results = {tuple(X): [yi[r][n], counter] for r in Xi for n, X in enumerate(Xi[r])} + results.update(self.opt_empty_tuple()) + backend.results_shared = backend.manager.dict(results) + else: + # if we have more than 1 optimizer but are using single opt, + # pick one discard the rest... + if n_opts > 0: + self.opt = opts[-1] + else: + self.opt = self.get_optimizer() + self.opt.void_loss = VOID_LOSS + self.opt.void = False + self.opt.rs = self.random_state + # in single mode restore the points directly to the optimizer + # but delete first in case we have filtered the starting list of points + self.opt = Hyperopt.opt_clear(self.opt) + rs = self.random_state + self.Xi[rs] = [] + self.track_points() + if len(self.Xi[rs]) > 0: + self.opt.tell(self.Xi[rs], self.yi[rs], fit=False) + # delete points since in single mode the optimizer state sits in the main + # process and is not discarded + self.Xi, self.yi = {}, {} + del opts[:] + + def setup_points(self): + self.n_initial_points, self.min_epochs, self.search_space_size = self.calc_epochs( + self.dimensions, self.n_jobs, self.effort, self.total_epochs, self.n_points + ) + logger.info(f"Min epochs set to: {self.min_epochs}") + # reduce random points by n_points in multi mode because asks are per job + if self.multi: + self.opt_n_initial_points = self.n_initial_points // self.n_points + else: + self.opt_n_initial_points = self.n_initial_points + logger.info(f'Initial points: {self.n_initial_points}') + # if total epochs are not set, max_epoch takes its place + if self.total_epochs < 1: + self.max_epoch = int(self.min_epochs + len(self.trials)) + # initialize average best occurrence + self.avg_best_occurrence = self.min_epochs // self.n_jobs + + def return_results(self): + """ + results are passed by queue in multi mode, or stored by ask_and_tell in single mode + """ + batch_results = [] + if self.multi: + while not backend.results_batch.empty(): + worker_results = backend.results_batch.get() + batch_results.extend(worker_results) + else: + batch_results.extend(self.batch_results) + del self.batch_results[:] + return batch_results + + def main_loop(self, jobs_scheduler): + """ main parallel loop """ + try: + with parallel_backend('loky', inner_max_num_threads=2): + with Parallel(n_jobs=self.n_jobs, verbose=0, backend='loky') as parallel: + jobs = parallel._effective_n_jobs() + logger.info(f'Effective number of parallel workers used: {jobs}') + # update epochs count + opt_points = self.opt_points + prev_batch = -1 + epochs_so_far = len(self.trials) + epochs_limit = self.epochs_limit + columns, _ = os.get_terminal_size() + columns -= 1 + while epochs_so_far > prev_batch or epochs_so_far < self.min_epochs: + prev_batch = epochs_so_far + occurrence = int(self.avg_best_occurrence * max(1, self.effort)) + # pad the batch length to the number of jobs to avoid desaturation + batch_len = (occurrence + jobs - + occurrence % jobs) + # when using multiple optimizers each worker performs + # n_points (epochs) in 1 dispatch but this reduces the batch len too much + # if self.multi: batch_len = batch_len // self.n_points + # don't go over the limit + if epochs_so_far + batch_len * opt_points >= epochs_limit(): + q, r = divmod(epochs_limit() - epochs_so_far, opt_points) + batch_len = q + r + print( + f"{epochs_so_far+1}-{epochs_so_far+batch_len}" + f"/{epochs_limit()}: ", + end='') + jobs_scheduler(parallel, batch_len, epochs_so_far, jobs) + batch_results = self.return_results() + print(end='\r') + saved = self.log_results(batch_results, epochs_so_far, epochs_limit()) + print('\r', ' ' * columns, end='\r') + # stop if no epochs have been evaluated + if len(batch_results) < batch_len: + logger.warning("Some evaluated epochs were void, " + "check the loss function and the search space.") + if (not saved and len(batch_results) > 1) or batch_len < 1 or \ + (not saved and self.search_space_size < batch_len + epochs_limit()): + break + # log_results add + epochs_so_far += saved + if self.max_epoch_reached: + logger.info("Max epoch reached, terminating.") + break + except KeyboardInterrupt: + print('User interrupted..') + def start(self) -> None: + """ Broom Broom """ self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") self.hyperopt_table_header = -1 @@ -639,6 +1337,7 @@ class Hyperopt: # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): preprocessed[pair] = trim_dataframe(df, timerange) + self.n_candles += len(preprocessed[pair]) min_date, max_date = get_timerange(data) logger.info( @@ -652,84 +1351,25 @@ class Hyperopt: self.backtesting.pairlists = None # type: ignore self.epochs = self.load_previous_results(self.results_file) + self.setup_epochs() - cpus = cpu_count() - logger.info(f"Found {cpus} CPU cores. Let's make them scream!") - config_jobs = self.config.get('hyperopt_jobs', -1) - logger.info(f'Number of parallel jobs set as: {config_jobs}') + logger.info(f"Found {cpu_count()} CPU cores. Let's make them scream!") + logger.info(f'Number of parallel jobs set as: {self.n_jobs}') self.dimensions: List[Dimension] = self.hyperopt_space() - self.opt = self.get_optimizer(self.dimensions, config_jobs) - try: - with Parallel(n_jobs=config_jobs) as parallel: - jobs = parallel._effective_n_jobs() - logger.info(f'Effective number of parallel workers used: {jobs}') + self.setup_points() - # Define progressbar - if self.print_colorized: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - fill_wrap=Fore.GREEN + '{}' + Fore.RESET, - marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] - else: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] - with progressbar.ProgressBar( - max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, - widgets=widgets - ) as pbar: - EVALS = ceil(self.total_epochs / jobs) - for i in range(EVALS): - # Correct the number of epochs to be processed for the last - # iteration (should not exceed self.total_epochs in total) - n_rest = (i + 1) * jobs - self.total_epochs - current_jobs = jobs - n_rest if n_rest > 0 else jobs + if self.print_colorized: + colorama_init(autoreset=True) - asked = self.opt.ask(n_points=current_jobs) - f_val = self.run_optimizer_parallel(parallel, asked, i) - self.opt.tell(asked, [v['loss'] for v in f_val]) + self.setup_optimizers() - # Calculate progressbar outputs - for j, val in enumerate(f_val): - # Use human-friendly indexes here (starting from 1) - current = i * jobs + j + 1 - val['current_epoch'] = current - val['is_initial_point'] = current <= INITIAL_POINTS + if self.multi: + jobs_scheduler = self.run_multi_backtest_parallel + else: + jobs_scheduler = self.run_backtest_parallel - logger.debug(f"Optimizer epoch evaluated: {val}") - - is_best = self.is_best_loss(val, self.current_best_loss) - # This value is assigned here and not in the optimization method - # to keep proper order in the list of results. That's because - # evaluations can take different time. Here they are aligned in the - # order they will be shown to the user. - val['is_best'] = is_best - self.print_results(val) - - if is_best: - self.current_best_loss = val['loss'] - self.epochs.append(val) - - # Save results after each best epoch and every 100 epochs - if is_best or current % 100 == 0: - self._save_results() - - pbar.update(current) - - except KeyboardInterrupt: - print('User interrupted..') + self.main_loop(jobs_scheduler) self._save_results() logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " @@ -737,9 +1377,14 @@ class Hyperopt: if self.epochs: sorted_epochs = sorted(self.epochs, key=itemgetter('loss')) - best_epoch = sorted_epochs[0] - self.print_epoch_details(best_epoch, self.total_epochs, self.print_json) + results = sorted_epochs[0] + self.print_epoch_details(results, self.epochs_limit(), self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. print("No epochs evaluated yet, no best result.") + + def __getstate__(self): + state = self.__dict__.copy() + del state['trials'] + return state diff --git a/freqtrade/optimize/hyperopt_backend.py b/freqtrade/optimize/hyperopt_backend.py new file mode 100644 index 000000000..1416f358d --- /dev/null +++ b/freqtrade/optimize/hyperopt_backend.py @@ -0,0 +1,18 @@ +from typing import Any, Dict, List, Tuple +from queue import Queue +from multiprocessing.managers import SyncManager + +hyperopt: Any = None +manager: SyncManager +# stores the optimizers in multi opt mode +optimizers: Queue +# stores the results to share between optimizers +# in the form of key = Tuple[Xi], value = Tuple[float, int] +# where float is the loss and int is a decreasing counter of optimizers +# that have registered the result +results_shared: Dict[Tuple, Tuple] +# in single mode the results_list is used to pass the results to the optimizer +# to fit new models +results_list: List +# results_batch stores keeps results per batch that are eventually logged and stored +results_batch: Queue diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index a6541f55b..a2db044fe 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -482,6 +482,7 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> None: epochs = create_results(mocker, hyperopt, testdatadir) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.save_opts') results_file = testdatadir / 'optimize' / 'ut_results.pickle' caplog.set_level(logging.DEBUG) @@ -529,7 +530,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}, @@ -564,8 +565,11 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called 3 times, from: + # 1 tickerdata + # 1 save_trials + # 1 save_opts + assert dumper.call_count == 3 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -686,13 +690,14 @@ def test_buy_strategy_generator(hyperopt, testdatadir) -> None: assert 1 in result['buy'] -def test_generate_optimizer(mocker, default_conf) -> None: - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'timerange': None, - 'spaces': 'all', - 'hyperopt_min_trades': 1, - }) +def test_backtest_params(mocker, default_conf) -> None: + default_conf.update({ + 'config': 'config.json.example', + 'hyperopt': 'DefaultHyperOpt', + 'timerange': None, + 'spaces': 'all', + 'hyperopt_min_trades': 1, + }) trades = [ ('TRX/BTC', 0.023117, 0.000233, 100) @@ -792,8 +797,8 @@ def test_generate_optimizer(mocker, default_conf) -> None: hyperopt = Hyperopt(default_conf) hyperopt.dimensions = hyperopt.hyperopt_space() - generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) - assert generate_optimizer_value == response_expected + backtest_params_value = hyperopt.backtest_params(list(optimizer_param.values())) + assert backtest_params_value == response_expected def test_clean_hyperopt(mocker, default_conf, caplog): @@ -809,7 +814,8 @@ def test_clean_hyperopt(mocker, default_conf, caplog): unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) h = Hyperopt(default_conf) - assert unlinkmock.call_count == 2 + # once for tickerdata, once for trials, once for optimizers (list) + assert unlinkmock.call_count == 3 assert log_has(f"Removing `{h.data_pickle_file}`.", caplog) @@ -841,7 +847,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {}, 'params_details': { @@ -886,8 +892,11 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: ) assert result_str in out # noqa: E501 assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called 3 times from: + # 1 tickerdata + # 1 save_trials + # 1 save_opts + assert dumper.call_count == 3 def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None: @@ -900,7 +909,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {}, 'params_details': { @@ -940,8 +949,11 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None out, err = capsys.readouterr() assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501 assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called three times, from: + # 1 tickerdata + # 1 save_trials + # 1 save_opts + assert dumper.call_count == 3 def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: @@ -954,7 +966,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {}, 'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}, @@ -990,8 +1002,11 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> out, err = capsys.readouterr() assert '{"minimal_roi":{},"stoploss":null}' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called three times from: + # 1 for tickerdata + # 1 for save_trials + # 1 for save_opts + assert dumper.call_count == 3 def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: @@ -1004,7 +1019,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}, 'results_metrics': @@ -1042,8 +1057,11 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called three times, from: + # 1 for tickerdata + # 1 for save_trials + # 1 for save_opts + assert dumper.call_count == 3 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -1092,7 +1110,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {}, 'results_metrics': @@ -1119,7 +1137,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) # TODO: sell_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked + # run_backtest_parallel() is mocked del hyperopt.custom_hyperopt.__class__.sell_strategy_generator del hyperopt.custom_hyperopt.__class__.sell_indicator_space @@ -1130,8 +1148,11 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called three times, from: + # 1 tickerdata + # 1 save_trials + # 1 save_opts + assert dumper.call_count == 3 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -1149,7 +1170,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None ) parallel = mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', + 'freqtrade.optimize.hyperopt.Hyperopt.run_backtest_parallel', MagicMock(return_value=[{ 'loss': 1, 'results_explanation': 'foo result', 'params': {}, 'results_metrics': @@ -1176,7 +1197,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) # TODO: buy_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked + # run_backtest_parallel() is mocked del hyperopt.custom_hyperopt.__class__.buy_strategy_generator del hyperopt.custom_hyperopt.__class__.indicator_space @@ -1187,8 +1208,11 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called three times, from: + # 1 tickerdata + # 1 save_trials + # 1 save_opt s + assert dumper.call_count == 3 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades")