diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 3f61ea66c..517f47d16 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -38,33 +38,33 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) } - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + results_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) + epochs = Hyperopt.load_previous_results(results_file) + total_epochs = len(epochs) - trials = _hyperopt_filter_trials(trials, filteroptions) + epochs = _hyperopt_filter_epochs(epochs, filteroptions) if print_colorized: colorama_init(autoreset=True) if not export_csv: try: - print(Hyperopt.get_result_table(config, trials, total_epochs, + print(Hyperopt.get_result_table(config, epochs, total_epochs, not filteroptions['only_best'], print_colorized, 0)) except KeyboardInterrupt: print('User interrupted..') - if trials and not no_details: - sorted_trials = sorted(trials, key=itemgetter('loss')) - results = sorted_trials[0] + if epochs and not no_details: + sorted_epochs = sorted(epochs, key=itemgetter('loss')) + results = sorted_epochs[0] Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) - if trials and export_csv: + if epochs and export_csv: Hyperopt.export_csv_file( - config, trials, total_epochs, not filteroptions['only_best'], export_csv + config, epochs, total_epochs, not filteroptions['only_best'], export_csv ) @@ -78,8 +78,8 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: print_json = config.get('print_json', False) no_header = config.get('hyperopt_show_no_header', False) - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + results_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') n = config.get('hyperopt_show_index', -1) filteroptions = { @@ -96,89 +96,87 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: } # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) + epochs = Hyperopt.load_previous_results(results_file) + total_epochs = len(epochs) - trials = _hyperopt_filter_trials(trials, filteroptions) - trials_epochs = len(trials) + epochs = _hyperopt_filter_epochs(epochs, filteroptions) + filtered_epochs = len(epochs) - if n > trials_epochs: + if n > filtered_epochs: raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") - if n < -trials_epochs: + f"The index of the epoch to show should be less than {filtered_epochs + 1}.") + if n < -filtered_epochs: raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") + f"The index of the epoch to show should be greater than {-filtered_epochs - 1}.") # Translate epoch index from human-readable format to pythonic if n > 0: n -= 1 - if trials: - val = trials[n] + if epochs: + val = epochs[n] Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") -def _hyperopt_filter_trials(trials: List, filteroptions: dict) -> List: +def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: """ Filter our items from the list of hyperopt results """ if filteroptions['only_best']: - trials = [x for x in trials if x['is_best']] + epochs = [x for x in epochs if x['is_best']] if filteroptions['only_profitable']: - trials = [x for x in trials if x['results_metrics']['profit'] > 0] + epochs = [x for x in epochs if x['results_metrics']['profit'] > 0] if filteroptions['filter_min_trades'] > 0: - trials = [ - x for x in trials + epochs = [ + x for x in epochs if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] ] if filteroptions['filter_max_trades'] > 0: - trials = [ - x for x in trials + epochs = [ + x for x in epochs if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] ] if filteroptions['filter_min_avg_time'] is not None: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] ] if filteroptions['filter_max_avg_time'] is not None: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] ] if filteroptions['filter_min_avg_profit'] is not None: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - > filteroptions['filter_min_avg_profit'] + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit'] ] if filteroptions['filter_max_avg_profit'] is not None: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials - if x['results_metrics']['avg_profit'] - < filteroptions['filter_max_avg_profit'] + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit'] ] if filteroptions['filter_min_total_profit'] is not None: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] ] if filteroptions['filter_max_total_profit'] is not None: - trials = [x for x in trials if x['results_metrics']['trade_count'] > 0] - trials = [ - x for x in trials + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] ] - logger.info(f"{len(trials)} " + + logger.info(f"{len(epochs)} " + ("best " if filteroptions['only_best'] else "") + ("profitable " if filteroptions['only_profitable'] else "") + "epochs found.") - return trials + return epochs diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 1e8077f10..a3e636988 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -75,8 +75,8 @@ class Hyperopt: self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - self.trials_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + self.results_file = (self.config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') self.data_pickle_file = (self.config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_tickerdata.pkl') self.total_epochs = config.get('epochs', 0) @@ -88,10 +88,10 @@ class Hyperopt: else: logger.info("Continuing on previous hyperopt results.") - self.num_trials_saved = 0 + self.num_epochs_saved = 0 # Previous evaluations - self.trials: List = [] + self.epochs: List = [] # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): @@ -132,7 +132,7 @@ class Hyperopt: """ Remove hyperopt pickle files to restart hyperopt. """ - for f in [self.data_pickle_file, self.trials_file]: + for f in [self.data_pickle_file, self.results_file]: p = Path(f) if p.is_file(): logger.info(f"Removing `{p}`.") @@ -151,27 +151,26 @@ class Hyperopt: # and the values are taken from the list of parameters. return {d.name: v for d, v in zip(dimensions, raw_params)} - def save_trials(self, final: bool = False) -> None: + def _save_results(self) -> None: """ - Save hyperopt trials to file + Save hyperopt results to file """ - num_trials = len(self.trials) - if num_trials > self.num_trials_saved: - logger.debug(f"Saving {num_trials} {plural(num_trials, 'epoch')}.") - dump(self.trials, self.trials_file) - self.num_trials_saved = num_trials - if final: - logger.info(f"{num_trials} {plural(num_trials, 'epoch')} " - f"saved to '{self.trials_file}'.") + num_epochs = len(self.epochs) + if num_epochs > self.num_epochs_saved: + logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.") + dump(self.epochs, self.results_file) + self.num_epochs_saved = num_epochs + logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'.") @staticmethod - def _read_trials(trials_file: Path) -> List: + def _read_results(results_file: Path) -> List: """ - Read hyperopt trials file + Read hyperopt results from file """ - logger.info("Reading Trials from '%s'", trials_file) - trials = load(trials_file) - return trials + logger.info("Reading epochs from '%s'", results_file) + data = load(results_file) + return data def _get_params_details(self, params: Dict) -> Dict: """ @@ -588,19 +587,20 @@ class Hyperopt: wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @staticmethod - def load_previous_results(trials_file: Path) -> List: + def load_previous_results(results_file: Path) -> List: """ Load data for epochs from the file if we have one """ - trials: List = [] - if trials_file.is_file() and trials_file.stat().st_size > 0: - trials = Hyperopt._read_trials(trials_file) - if trials[0].get('is_best') is None: + epochs: List = [] + if results_file.is_file() and results_file.stat().st_size > 0: + epochs = Hyperopt._read_results(results_file) + # Detection of some old format, without 'is_best' field saved + if epochs[0].get('is_best') is None: raise OperationalException( "The file with Hyperopt results is incompatible with this version " "of Freqtrade and cannot be loaded.") - logger.info(f"Loaded {len(trials)} previous evaluations from disk.") - return trials + logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") + return epochs def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) @@ -628,7 +628,7 @@ class Hyperopt: self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore - self.trials = self.load_previous_results(self.trials_file) + self.epochs = self.load_previous_results(self.results_file) cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") @@ -698,23 +698,25 @@ class Hyperopt: if is_best: self.current_best_loss = val['loss'] - self.trials.append(val) + self.epochs.append(val) # Save results after each best epoch and every 100 epochs if is_best or current % 100 == 0: - self.save_trials() + self._save_results() pbar.update(current) except KeyboardInterrupt: print('User interrupted..') - self.save_trials(final=True) + self._save_results() + logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'.") - if self.trials: - sorted_trials = sorted(self.trials, key=itemgetter('loss')) - results = sorted_trials[0] - self.print_epoch_details(results, self.total_epochs, self.print_json) + if self.epochs: + sorted_epochs = sorted(self.epochs, key=itemgetter('loss')) + best_epoch = sorted_epochs[0] + self.print_epoch_details(best_epoch, self.total_epochs, self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index b5106be0c..cc8b9aa37 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import locale +import logging from datetime import datetime from pathlib import Path from typing import Dict, List @@ -56,14 +57,14 @@ def hyperopt_results(): # Functions for recurrent object patching -def create_trials(mocker, hyperopt, testdatadir) -> List[Dict]: +def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: """ - When creating trials, mock the hyperopt Trials so that *by default* + When creating results, mock the hyperopt so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - hyperopt.trials_file = testdatadir / 'optimize/ut_trials.pickle' + hyperopt.results_file = testdatadir / 'optimize/ut_results.pickle' mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) stat_mock = MagicMock() @@ -477,28 +478,30 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None: - trials = create_trials(mocker, hyperopt, testdatadir) +def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> None: + epochs = create_results(mocker, hyperopt, testdatadir) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) - trials_file = testdatadir / 'optimize' / 'ut_trials.pickle' + results_file = testdatadir / 'optimize' / 'ut_results.pickle' - hyperopt.trials = trials - hyperopt.save_trials(final=True) - assert log_has(f"1 epoch saved to '{trials_file}'.", caplog) + caplog.set_level(logging.DEBUG) + + hyperopt.epochs = epochs + hyperopt._save_results() + assert log_has(f"1 epoch saved to '{results_file}'.", caplog) mock_dump.assert_called_once() - hyperopt.trials = trials + trials - hyperopt.save_trials(final=True) - assert log_has(f"2 epochs saved to '{trials_file}'.", caplog) + hyperopt.epochs = epochs + epochs + hyperopt._save_results() + assert log_has(f"2 epochs saved to '{results_file}'.", caplog) -def test_read_trials_returns_trials_file(mocker, hyperopt, testdatadir, caplog) -> None: - trials = create_trials(mocker, hyperopt, testdatadir) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) - trials_file = testdatadir / 'optimize' / 'ut_trials.pickle' - hyperopt_trial = hyperopt._read_trials(trials_file) - assert log_has(f"Reading Trials from '{trials_file}'", caplog) - assert hyperopt_trial == trials +def test_read_results_returns_epochs(mocker, hyperopt, testdatadir, caplog) -> None: + epochs = create_results(mocker, hyperopt, testdatadir) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) + results_file = testdatadir / 'optimize' / 'ut_results.pickle' + hyperopt_epochs = hyperopt._read_results(results_file) + assert log_has(f"Reading epochs from '{results_file}'", caplog) + assert hyperopt_epochs == epochs mock_load.assert_called_once()