adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2563 from hroff-1902/hyperopt-save

Browse Source 
Hyperopt: Save epochs at intermediate points
This commit is contained in:
Matthias 2019-11-23 19:25:35 +01:00 committed by GitHub
commit e9da4d8505
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
freqtrade/optimize/hyperopt.py
View File

@ -23,7 +23,7 @@ from skopt import Optimizer
from skopt.space import Dimension
from freqtrade.data.history import get_timeframe, trim_dataframe
from freqtrade.misc import round_dict
from freqtrade.misc import plural, round_dict
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F4
@ -77,6 +77,8 @@ class Hyperopt:
# Previous evaluations
self.trials: List = []
self.num_trials_saved = 0
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = \
@ -132,13 +134,18 @@ class Hyperopt:
arg_dict = {dim.name: value for dim, value in zip(dimensions, params)}
return arg_dict
def save_trials(self) -> None:
def save_trials(self, final: bool = False) -> None:
"""
Save hyperopt trials to file
"""
if self.trials:
logger.info("Saving %d evaluations to '%s'", len(self.trials), self.trials_file)
num_trials = len(self.trials)
if num_trials > self.num_trials_saved:
logger.info(f"Saving {num_trials} {plural(num_trials, 'epoch')}.")
dump(self.trials, self.trials_file)
self.num_trials_saved = num_trials
if final:
logger.info(f"{num_trials} {plural(num_trials, 'epoch')} "
f"saved to '{self.trials_file}'.")
def read_trials(self) -> List:
"""
@ -153,6 +160,12 @@ class Hyperopt:
"""
Display Best hyperopt result
"""
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.
if not self.trials:
print("No epochs evaluated yet, no best result.")
return
results = sorted(self.trials, key=itemgetter('loss'))
best_result = results[0]
params = best_result['params']
@ -197,12 +210,20 @@ class Hyperopt:
# Also round to 5 digits after the decimal point
print(f"Stoploss: {round(params.get('stoploss'), 5)}")
def is_best(self, results) -> bool:
return results['loss'] < self.current_best_loss
def log_results(self, results) -> None:
"""
Log results if it is better than any previous evaluation
"""
print_all = self.config.get('print_all', False)
is_best_loss = results['loss'] < self.current_best_loss
is_best_loss = self.is_best(results)
if not print_all:
print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore
sys.stdout.flush()
if print_all or is_best_loss:
if is_best_loss:
self.current_best_loss = results['loss']
@ -217,13 +238,9 @@ class Hyperopt:
print(log_str)
else:
print(f'\n{log_str}')
else:
print('.', end='')
sys.stdout.flush()
def format_results_logstring(self, results) -> str:
# Output human-friendly index here (starting from 1)
current = results['current_epoch'] + 1
current = results['current_epoch']
total = self.total_epochs
res = results['results_explanation']
loss = results['loss']
@ -422,15 +439,19 @@ class Hyperopt:
self.opt.tell(asked, [v['loss'] for v in f_val])
self.fix_optimizer_models_list()
for j in range(jobs):
current = i * jobs + j
# Use human-friendly index here (starting from 1)
current = i * jobs + j + 1
val = f_val[j]
val['current_epoch'] = current
val['is_initial_point'] = current < INITIAL_POINTS
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")
is_best = self.is_best(val)
self.log_results(val)
self.trials.append(val)
logger.debug(f"Optimizer epoch evaluated: {val}")
if is_best or current % 100 == 0:
self.save_trials()
except KeyboardInterrupt:
print('User interrupted..')
self.save_trials()
self.save_trials(final=True)
self.log_trials_result()

17
tests/optimize/test_hyperopt.py
View File

@ -360,7 +360,7 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
hyperopt.log_results(
{
'loss': 1,
'current_epoch': 1,
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
'results_explanation': 'foo.',
'is_initial_point': False
}
@ -374,6 +374,7 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None:
hyperopt.log_results(
{
'loss': 3,
'current_epoch': 1,
}
)
assert caplog.record_tuples == []
@ -382,13 +383,19 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None:
def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None:
trials = create_trials(mocker, hyperopt, testdatadir)
mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None)
hyperopt.trials = trials
hyperopt.save_trials()
trials_file = testdatadir / 'optimize' / 'ut_trials.pickle'
assert log_has(f"Saving 1 evaluations to '{trials_file}'", caplog)
hyperopt.trials = trials
hyperopt.save_trials(final=True)
assert log_has("Saving 1 epoch.", caplog)
assert log_has(f"1 epoch saved to '{trials_file}'.", caplog)
mock_dump.assert_called_once()
hyperopt.trials = trials + trials
hyperopt.save_trials(final=True)
assert log_has("Saving 2 epochs.", caplog)
assert log_has(f"2 epochs saved to '{trials_file}'.", caplog)
def test_read_trials_returns_trials_file(mocker, hyperopt, testdatadir, caplog) -> None:
trials = create_trials(mocker, hyperopt, testdatadir)