adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2581 from hroff-1902/hyperopt-list

Browse Source 
Add hyperopt-list and hyperopt-show commands
This commit is contained in:
hroff-1902 2019-12-10 00:30:26 +03:00 committed by GitHub
commit 0e4ef33d6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 660 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/hyperopt.md
View File

@ -450,7 +450,11 @@ If you are optimizing trailing stop values, Freqtrade creates the 'trailing' opt
Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py).
### Validate backtesting results
## Show details of Hyperopt results
After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` subcommands. The usage of these subcommands is described in the [Utils](utils.md#list-hyperopt-results) chapter.
## Validate backtesting results
Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected.

65
docs/utils.md
View File

@ -262,3 +262,68 @@ Show whitelist when using a [dynamic pairlist](configuration.md#pairlists).
```
freqtrade test-pairlist --config config.json --quote USDT BTC
```
## List Hyperopt results
You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` subcommand.
```
usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-d PATH] [--userdir PATH] [--best]
[--profitable] [--no-color] [--print-json]
[--no-details]
optional arguments:
-h, --help show this help message and exit
--best Select only best epochs.
--profitable Select only profitable epochs.
--no-color Disable colorization of hyperopt results. May be
useful if you are redirecting output to a file.
--print-json Print best result detailization in JSON format.
--no-details Do not print best epoch details.
```
### Examples
List all results, print details of the best result at the end:
```
freqtrade hyperopt-list
```
List only epochs with positive profit. Do not print the details of the best epoch, so that the list can be iterated in a script:
```
freqtrade hyperopt-list --profitable --no-details
```
## Show details of Hyperopt results
You can show the details of any hyperoptimization epoch previously evaluated by the Hyperopt module with the `hyperopt-show` subcommand.
```
usage: freqtrade hyperopt-show [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-d PATH] [--userdir PATH] [--best]
[--profitable] [-n INT] [--print-json]
[--no-header]
optional arguments:
-h, --help show this help message and exit
--best Select only best epochs.
--profitable Select only profitable epochs.
-n INT, --index INT Specify the index of the epoch to print details for.
--print-json Print best result detailization in JSON format.
--no-header Do not print epoch details header.
```
### Examples
Print details for the epoch 168 (the number of the epoch is shown by the `hyperopt-list` subcommand or by Hyperopt itself during hyperoptimization run):
```
freqtrade hyperopt-show -n 168
```
Prints JSON data with details for the last best epoch (i.e., the best of all epochs):
```
freqtrade hyperopt-show --best -n -1 --print-json --no-header
```

27
freqtrade/configuration/arguments.py
View File

@ -55,8 +55,14 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "ticker_interval"]
ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_colorized",
"print_json", "hyperopt_list_no_details"]
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperopt_show_no_header"]
NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs",
"plot-dataframe", "plot-profit"]
"hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]
@ -123,6 +129,7 @@ class Arguments:
from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge
from freqtrade.utils import (start_create_userdir, start_download_data,
start_hyperopt_list, start_hyperopt_show,
start_list_exchanges, start_list_markets,
start_new_hyperopt, start_new_strategy,
start_list_timeframes, start_test_pairlist, start_trading)
@ -248,3 +255,21 @@ class Arguments:
)
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
# Add hyperopt-list subcommand
hyperopt_list_cmd = subparsers.add_parser(
'hyperopt-list',
help='List Hyperopt results',
parents=[_common_parser],
)
hyperopt_list_cmd.set_defaults(func=start_hyperopt_list)
self._build_args(optionlist=ARGS_HYPEROPT_LIST, parser=hyperopt_list_cmd)
# Add hyperopt-show subcommand
hyperopt_show_cmd = subparsers.add_parser(
'hyperopt-show',
help='Show details of Hyperopt results',
parents=[_common_parser],
)
hyperopt_show_cmd.set_defaults(func=start_hyperopt_show)
self._build_args(optionlist=ARGS_HYPEROPT_SHOW, parser=hyperopt_show_cmd)

39
freqtrade/configuration/cli_options.py
View File

@ -18,6 +18,18 @@ def check_int_positive(value: str) -> int:
return uint
def check_int_nonzero(value: str) -> int:
try:
uint = int(value)
if uint == 0:
raise ValueError
except ValueError:
raise argparse.ArgumentTypeError(
f"{value} is invalid for this parameter, should be a non-zero integer value"
)
return uint
class Arg:
# Optional CLI arguments
def __init__(self, *args, **kwargs):
@ -377,4 +389,31 @@ AVAILABLE_CLI_OPTIONS = {
choices=["DB", "file"],
default="file",
),
# hyperopt-list, hyperopt-show
"hyperopt_list_profitable": Arg(
'--profitable',
help='Select only profitable epochs.',
action='store_true',
),
"hyperopt_list_best": Arg(
'--best',
help='Select only best epochs.',
action='store_true',
),
"hyperopt_list_no_details": Arg(
'--no-details',
help='Do not print best epoch details.',
action='store_true',
),
"hyperopt_show_index": Arg(
'-n', '--index',
help='Specify the index of the epoch to print details for.',
type=check_int_nonzero,
metavar='INT',
),
"hyperopt_show_no_header": Arg(
'--no-header',
help='Do not print epoch details header.',
action='store_true',
),
}

15
freqtrade/configuration/configuration.py
View File

@ -300,6 +300,21 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_loss',
logstring='Using Hyperopt loss class name: {}')
self._args_to_config(config, argname='hyperopt_show_index',
logstring='Parameter -n/--index detected: {}')
self._args_to_config(config, argname='hyperopt_list_best',
logstring='Parameter --best detected: {}')
self._args_to_config(config, argname='hyperopt_list_profitable',
logstring='Parameter --profitable detected: {}')
self._args_to_config(config, argname='hyperopt_list_no_details',
logstring='Parameter --no-details detected: {}')
self._args_to_config(config, argname='hyperopt_show_no_header',
logstring='Parameter --no-header detected: {}')
def _process_plot_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='pairs',

334
freqtrade/optimize/hyperopt.py
View File

@ -22,6 +22,7 @@ from pandas import DataFrame
from skopt import Optimizer
from skopt.space import Dimension
from freqtrade import OperationalException
from freqtrade.data.history import get_timeframe, trim_dataframe
from freqtrade.misc import plural, round_dict
from freqtrade.optimize.backtesting import Backtesting
@ -74,11 +75,11 @@ class Hyperopt:
else:
logger.info("Continuing on previous hyperopt results.")
self.num_trials_saved = 0
# Previous evaluations
self.trials: List = []
self.num_trials_saved = 0
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = \
@ -104,6 +105,10 @@ class Hyperopt:
self.config['ask_strategy'] = {}
self.config['ask_strategy']['use_sell_signal'] = True
self.print_all = self.config.get('print_all', False)
self.print_colorized = self.config.get('print_colorized', False)
self.print_json = self.config.get('print_json', False)
@staticmethod
def get_lock_filename(config) -> str:
@ -119,20 +124,18 @@ class Hyperopt:
logger.info(f"Removing `{p}`.")
p.unlink()
def get_args(self, params):
def _get_params_dict(self, raw_params: List[Any]) -> Dict:
dimensions = self.dimensions
dimensions: List[Dimension] = self.dimensions
# Ensure the number of dimensions match
# the number of parameters in the list x.
if len(params) != len(dimensions):
raise ValueError('Mismatch in number of search-space dimensions. '
f'len(dimensions)=={len(dimensions)} and len(x)=={len(params)}')
# the number of parameters in the list.
if len(raw_params) != len(dimensions):
raise ValueError('Mismatch in number of search-space dimensions.')
# Create a dict where the keys are the names of the dimensions
# and the values are taken from the list of parameters x.
arg_dict = {dim.name: value for dim, value in zip(dimensions, params)}
return arg_dict
# Return a dict where the keys are the names of the dimensions
# and the values are taken from the list of parameters.
return {d.name: v for d, v in zip(dimensions, raw_params)}
def save_trials(self, final: bool = False) -> None:
"""
@ -147,48 +150,71 @@ class Hyperopt:
logger.info(f"{num_trials} {plural(num_trials, 'epoch')} "
f"saved to '{self.trials_file}'.")
def read_trials(self) -> List:
@staticmethod
def _read_trials(trials_file) -> List:
"""
Read hyperopt trials file
"""
logger.info("Reading Trials from '%s'", self.trials_file)
trials = load(self.trials_file)
self.trials_file.unlink()
logger.info("Reading Trials from '%s'", trials_file)
trials = load(trials_file)
return trials
def log_trials_result(self) -> None:
def _get_params_details(self, params: Dict) -> Dict:
"""
Display Best hyperopt result
Return the params for each space
"""
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.
if not self.trials:
print("No epochs evaluated yet, no best result.")
return
result: Dict = {}
results = sorted(self.trials, key=itemgetter('loss'))
best_result = results[0]
params = best_result['params']
log_str = self.format_results_logstring(best_result)
if self.has_space('buy'):
result['buy'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('buy')}
if self.has_space('sell'):
result['sell'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('sell')}
if self.has_space('roi'):
result['roi'] = self.custom_hyperopt.generate_roi_table(params)
if self.has_space('stoploss'):
result['stoploss'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('stoploss')}
if self.has_space('trailing'):
result['trailing'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('trailing')}
print(f"\nBest result:\n\n{log_str}\n")
return result
if self.config.get('print_json'):
@staticmethod
def print_epoch_details(results, total_epochs, print_json: bool,
no_header: bool = False, header_str: str = None) -> None:
"""
Display details of the hyperopt result
"""
params = results.get('params_details', {})
# Default header string
if header_str is None:
header_str = "Best result"
if not no_header:
explanation_str = Hyperopt._format_explanation_string(results, total_epochs)
print(f"\n{header_str}:\n\n{explanation_str}\n")
if print_json:
result_dict: Dict = {}
for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
self._params_update_for_json(result_dict, params, s)
Hyperopt._params_update_for_json(result_dict, params, s)
print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
else:
self._params_pretty_print(params, 'buy', "Buy hyperspace params:")
self._params_pretty_print(params, 'sell', "Sell hyperspace params:")
self._params_pretty_print(params, 'roi', "ROI table:")
self._params_pretty_print(params, 'stoploss', "Stoploss:")
self._params_pretty_print(params, 'trailing', "Trailing stop:")
Hyperopt._params_pretty_print(params, 'buy', "Buy hyperspace params:")
Hyperopt._params_pretty_print(params, 'sell', "Sell hyperspace params:")
Hyperopt._params_pretty_print(params, 'roi', "ROI table:")
Hyperopt._params_pretty_print(params, 'stoploss', "Stoploss:")
Hyperopt._params_pretty_print(params, 'trailing', "Trailing stop:")
def _params_update_for_json(self, result_dict, params, space: str):
if self.has_space(space):
space_params = self.space_params(params, space)
@staticmethod
def _params_update_for_json(result_dict, params, space: str):
if space in params:
space_params = Hyperopt._space_params(params, space)
if space in ['buy', 'sell']:
result_dict.setdefault('params', {}).update(space_params)
elif space == 'roi':
@ -202,49 +228,65 @@ class Hyperopt:
else: # 'stoploss', 'trailing'
result_dict.update(space_params)
def _params_pretty_print(self, params, space: str, header: str):
if self.has_space(space):
space_params = self.space_params(params, space, 5)
print(header)
pprint(space_params, indent=4)
@staticmethod
def _params_pretty_print(params, space: str, header: str):
if space in params:
space_params = Hyperopt._space_params(params, space, 5)
if space == 'stoploss':
print(header, space_params.get('stoploss'))
else:
print(header)
pprint(space_params, indent=4)
def is_best(self, results) -> bool:
return results['loss'] < self.current_best_loss
@staticmethod
def _space_params(params, space: str, r: int = None) -> Dict:
d = params[space]
# Round floats to `r` digits after the decimal point if requested
return round_dict(d, r) if r else d
def log_results(self, results) -> None:
@staticmethod
def is_best_loss(results, current_best_loss) -> bool:
return results['loss'] < current_best_loss
def print_results(self, results) -> None:
"""
Log results if it is better than any previous evaluation
"""
print_all = self.config.get('print_all', False)
is_best_loss = self.is_best(results)
if not print_all:
is_best = results['is_best']
if not self.print_all:
# Print '\n' after each 100th epoch to separate dots from the log messages.
# Otherwise output is messy on a terminal.
print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore
sys.stdout.flush()
if print_all or is_best_loss:
if is_best_loss:
self.current_best_loss = results['loss']
log_str = self.format_results_logstring(results)
# Colorize output
if self.config.get('print_colorized', False):
if results['total_profit'] > 0:
log_str = Fore.GREEN + log_str
if print_all and is_best_loss:
log_str = Style.BRIGHT + log_str
if print_all:
print(log_str)
else:
print(f'\n{log_str}')
if self.print_all or is_best:
if not self.print_all:
# Separate the results explanation string from dots
print("\n")
self.print_results_explanation(results, self.total_epochs, self.print_all,
self.print_colorized)
def format_results_logstring(self, results) -> str:
current = results['current_epoch']
total = self.total_epochs
res = results['results_explanation']
loss = results['loss']
log_str = f'{current:5d}/{total}: {res} Objective: {loss:.5f}'
log_str = f'*{log_str}' if results['is_initial_point'] else f' {log_str}'
return log_str
@staticmethod
def print_results_explanation(results, total_epochs, highlight_best: bool,
print_colorized: bool) -> None:
"""
Log results explanation string
"""
explanation_str = Hyperopt._format_explanation_string(results, total_epochs)
# Colorize output
if print_colorized:
if results['total_profit'] > 0:
explanation_str = Fore.GREEN + explanation_str
if highlight_best and results['is_best']:
explanation_str = Style.BRIGHT + explanation_str
print(explanation_str)
@staticmethod
def _format_explanation_string(results, total_epochs) -> str:
return (("*" if results['is_initial_point'] else " ") +
f"{results['current_epoch']:5d}/{total_epochs}: " +
f"{results['results_explanation']} " +
f"Objective: {results['loss']:.5f}")
def has_space(self, space: str) -> bool:
"""
@ -287,49 +329,43 @@ class Hyperopt:
return spaces
def space_params(self, params, space: str, r: int = None) -> Dict:
if space == 'roi':
d = self.custom_hyperopt.generate_roi_table(params)
else:
d = {p.name: params.get(p.name) for p in self.hyperopt_space(space)}
# Round floats to `r` digits after the decimal point if requested
return round_dict(d, r) if r else d
def generate_optimizer(self, _params: Dict, iteration=None) -> Dict:
def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
"""
Used Optimize function. Called once per epoch to optimize whatever is configured.
Keep this function as optimized as possible!
"""
params = self.get_args(_params)
params_dict = self._get_params_dict(raw_params)
params_details = self._get_params_details(params_dict)
if self.has_space('roi'):
self.backtesting.strategy.minimal_roi = \
self.custom_hyperopt.generate_roi_table(params)
self.custom_hyperopt.generate_roi_table(params_dict)
if self.has_space('buy'):
self.backtesting.strategy.advise_buy = \
self.custom_hyperopt.buy_strategy_generator(params)
self.custom_hyperopt.buy_strategy_generator(params_dict)
if self.has_space('sell'):
self.backtesting.strategy.advise_sell = \
self.custom_hyperopt.sell_strategy_generator(params)
self.custom_hyperopt.sell_strategy_generator(params_dict)
if self.has_space('stoploss'):
self.backtesting.strategy.stoploss = params['stoploss']
self.backtesting.strategy.stoploss = params_dict['stoploss']
if self.has_space('trailing'):
self.backtesting.strategy.trailing_stop = params['trailing_stop']
self.backtesting.strategy.trailing_stop_positive = params['trailing_stop_positive']
self.backtesting.strategy.trailing_stop = params_dict['trailing_stop']
self.backtesting.strategy.trailing_stop_positive = \
params_dict['trailing_stop_positive']
self.backtesting.strategy.trailing_stop_positive_offset = \
params['trailing_stop_positive_offset']
params_dict['trailing_stop_positive_offset']
self.backtesting.strategy.trailing_only_offset_is_reached = \
params['trailing_only_offset_is_reached']
params_dict['trailing_only_offset_is_reached']
processed = load(self.tickerdata_pickle)
min_date, max_date = get_timeframe(processed)
results = self.backtesting.backtest(
backtesting_results = self.backtesting.backtest(
{
'stake_amount': self.config['stake_amount'],
'processed': processed,
@ -339,58 +375,63 @@ class Hyperopt:
'end_date': max_date,
}
)
results_explanation = self.format_results(results)
return self._get_results_dict(backtesting_results, min_date, max_date,
params_dict, params_details)
trade_count = len(results.index)
total_profit = results.profit_abs.sum()
def _get_results_dict(self, backtesting_results, min_date, max_date,
params_dict, params_details):
results_metrics = self._calculate_results_metrics(backtesting_results)
results_explanation = self._format_results_explanation_string(results_metrics)
trade_count = results_metrics['trade_count']
total_profit = results_metrics['total_profit']
# If this evaluation contains too short amount of trades to be
# interesting -- consider it as 'bad' (assigned max. loss value)
# in order to cast this hyperspace point away from optimization
# path. We do not want to optimize 'hodl' strategies.
if trade_count < self.config['hyperopt_min_trades']:
return {
'loss': MAX_LOSS,
'params': params,
'results_explanation': results_explanation,
'total_profit': total_profit,
}
loss = self.calculate_loss(results=results, trade_count=trade_count,
min_date=min_date.datetime, max_date=max_date.datetime)
loss: float = MAX_LOSS
if trade_count >= self.config['hyperopt_min_trades']:
loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count,
min_date=min_date.datetime, max_date=max_date.datetime)
return {
'loss': loss,
'params': params,
'params_dict': params_dict,
'params_details': params_details,
'results_metrics': results_metrics,
'results_explanation': results_explanation,
'total_profit': total_profit,
}
def format_results(self, results: DataFrame) -> str:
def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
return {
'trade_count': len(backtesting_results.index),
'avg_profit': backtesting_results.profit_percent.mean() * 100.0,
'total_profit': backtesting_results.profit_abs.sum(),
'profit': backtesting_results.profit_percent.sum() * 100.0,
'duration': backtesting_results.trade_duration.mean(),
}
def _format_results_explanation_string(self, results_metrics: Dict) -> str:
"""
Return the formatted results explanation in a string
"""
trades = len(results.index)
avg_profit = results.profit_percent.mean() * 100.0
total_profit = results.profit_abs.sum()
stake_cur = self.config['stake_currency']
profit = results.profit_percent.sum() * 100.0
duration = results.trade_duration.mean()
return (f'{trades:6d} trades. Avg profit {avg_profit: 5.2f}%. '
f'Total profit {total_profit: 11.8f} {stake_cur} '
f'({profit: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). '
f'Avg duration {duration:5.1f} mins.'
return (f"{results_metrics['trade_count']:6d} trades. "
f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
f"Avg duration {results_metrics['duration']:5.1f} mins."
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
def get_optimizer(self, dimensions, cpu_count) -> Optimizer:
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
return Optimizer(
dimensions,
base_estimator="ET",
acq_optimizer="auto",
n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.config.get('hyperopt_random_state', None)
random_state=self.config.get('hyperopt_random_state', None),
)
def fix_optimizer_models_list(self):
@ -414,14 +455,20 @@ class Hyperopt:
return parallel(delayed(
wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked)
def load_previous_results(self):
""" read trials file if we have one """
if self.trials_file.is_file() and self.trials_file.stat().st_size > 0:
self.trials = self.read_trials()
logger.info(
'Loaded %d previous evaluations from disk.',
len(self.trials)
)
@staticmethod
def load_previous_results(trials_file) -> List:
"""
Load data for epochs from the file if we have one
"""
trials: List = []
if trials_file.is_file() and trials_file.stat().st_size > 0:
trials = Hyperopt._read_trials(trials_file)
if trials[0].get('is_best') is None:
raise OperationalException(
"The file with Hyperopt results is incompatible with this version "
"of Freqtrade and cannot be loaded.")
logger.info(f"Loaded {len(trials)} previous evaluations from disk.")
return trials
def start(self) -> None:
data, timerange = self.backtesting.load_bt_data()
@ -442,17 +489,17 @@ class Hyperopt:
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange = None # type: ignore
self.load_previous_results()
self.trials = self.load_previous_results(self.trials_file)
cpus = cpu_count()
logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
config_jobs = self.config.get('hyperopt_jobs', -1)
logger.info(f'Number of parallel jobs set as: {config_jobs}')
self.dimensions = self.hyperopt_space()
self.dimensions: List[Dimension] = self.hyperopt_space()
self.opt = self.get_optimizer(self.dimensions, config_jobs)
if self.config.get('print_colorized', False):
if self.print_colorized:
colorama_init(autoreset=True)
try:
@ -466,19 +513,38 @@ class Hyperopt:
self.opt.tell(asked, [v['loss'] for v in f_val])
self.fix_optimizer_models_list()
for j in range(jobs):
# Use human-friendly index here (starting from 1)
# Use human-friendly indexes here (starting from 1)
current = i * jobs + j + 1
val = f_val[j]
val['current_epoch'] = current
val['is_initial_point'] = current <= INITIAL_POINTS
logger.debug(f"Optimizer epoch evaluated: {val}")
is_best = self.is_best(val)
self.log_results(val)
is_best = self.is_best_loss(val, self.current_best_loss)
# This value is assigned here and not in the optimization method
# to keep proper order in the list of results. That's because
# evaluations can take different time. Here they are aligned in the
# order they will be shown to the user.
val['is_best'] = is_best
self.print_results(val)
if is_best:
self.current_best_loss = val['loss']
self.trials.append(val)
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self.save_trials()
except KeyboardInterrupt:
print('User interrupted..')
self.save_trials(final=True)
self.log_trials_result()
if self.trials:
sorted_trials = sorted(self.trials, key=itemgetter('loss'))
results = sorted_trials[0]
self.print_epoch_details(results, self.total_epochs, self.print_json)
else:
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.
print("No epochs evaluated yet, no best result.")

105
freqtrade/utils.py
View File

@ -2,11 +2,13 @@ import csv
import logging
import sys
from collections import OrderedDict
from operator import itemgetter
from pathlib import Path
from typing import Any, Dict, List
import arrow
import rapidjson
from colorama import init as colorama_init
from tabulate import tabulate
from freqtrade import OperationalException
@ -354,3 +356,106 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
print(rapidjson.dumps(list(pairlist), default=str))
else:
print(pairlist)
def start_hyperopt_list(args: Dict[str, Any]) -> None:
"""
List hyperopt epochs previously evaluated
"""
from freqtrade.optimize.hyperopt import Hyperopt
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
only_best = config.get('hyperopt_list_best', False)
only_profitable = config.get('hyperopt_list_profitable', False)
print_colorized = config.get('print_colorized', False)
print_json = config.get('print_json', False)
no_details = config.get('hyperopt_list_no_details', False)
no_header = False
trials_file = (config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
# Previous evaluations
trials = Hyperopt.load_previous_results(trials_file)
total_epochs = len(trials)
trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
# TODO: fetch the interval for epochs to print from the cli option
epoch_start, epoch_stop = 0, None
if print_colorized:
colorama_init(autoreset=True)
try:
# Human-friendly indexes used here (starting from 1)
for val in trials[epoch_start:epoch_stop]:
Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized)
except KeyboardInterrupt:
print('User interrupted..')
if trials and not no_details:
sorted_trials = sorted(trials, key=itemgetter('loss'))
results = sorted_trials[0]
Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
def start_hyperopt_show(args: Dict[str, Any]) -> None:
"""
Show details of a hyperopt epoch previously evaluated
"""
from freqtrade.optimize.hyperopt import Hyperopt
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
only_best = config.get('hyperopt_list_best', False)
only_profitable = config.get('hyperopt_list_profitable', False)
no_header = config.get('hyperopt_show_no_header', False)
trials_file = (config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_results.pickle')
# Previous evaluations
trials = Hyperopt.load_previous_results(trials_file)
total_epochs = len(trials)
trials = _hyperopt_filter_trials(trials, only_best, only_profitable)
trials_epochs = len(trials)
n = config.get('hyperopt_show_index', -1)
if n > trials_epochs:
raise OperationalException(
f"The index of the epoch to show should be less than {trials_epochs + 1}.")
if n < -trials_epochs:
raise OperationalException(
f"The index of the epoch to show should be greater than {-trials_epochs - 1}.")
# Translate epoch index from human-readable format to pythonic
if n > 0:
n -= 1
print_json = config.get('print_json', False)
if trials:
val = trials[n]
Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
header_str="Epoch details")
def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List:
"""
Filter our items from the list of hyperopt results
"""
if only_best:
trials = [x for x in trials if x['is_best']]
if only_profitable:
trials = [x for x in trials if x['results_metrics']['profit'] > 0]
logger.info(f"{len(trials)} " +
("best " if only_best else "") +
("profitable " if only_profitable else "") +
"epochs found.")
return trials

5
tests/conftest.py
View File

File diff suppressed because one or more lines are too long

82
tests/optimize/test_hyperopt.py
View File

@ -357,8 +357,9 @@ def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results)
def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
hyperopt.current_best_loss = 2
hyperopt.total_epochs = 2
hyperopt.log_results(
hyperopt.print_results(
{
'is_best': True,
'loss': 1,
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
'results_explanation': 'foo.',
@ -371,8 +372,9 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None:
hyperopt.current_best_loss = 2
hyperopt.log_results(
hyperopt.print_results(
{
'is_best': False,
'loss': 3,
'current_epoch': 1,
}
@ -400,8 +402,8 @@ def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None
def test_read_trials_returns_trials_file(mocker, hyperopt, testdatadir, caplog) -> None:
trials = create_trials(mocker, hyperopt, testdatadir)
mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials)
hyperopt_trial = hyperopt.read_trials()
trials_file = testdatadir / 'optimize' / 'ut_trials.pickle'
hyperopt_trial = hyperopt._read_trials(trials_file)
assert log_has(f"Reading Trials from '{trials_file}'", caplog)
assert hyperopt_trial == trials
mock_load.assert_called_once()
@ -473,8 +475,22 @@ def test_format_results(hyperopt):
]
labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration']
df = pd.DataFrame.from_records(trades, columns=labels)
results_metrics = hyperopt._calculate_results_metrics(df)
results_explanation = hyperopt._format_results_explanation_string(results_metrics)
total_profit = results_metrics['total_profit']
result = hyperopt.format_results(df)
results = {
'loss': 0.0,
'params_dict': None,
'params_details': None,
'results_metrics': results_metrics,
'results_explanation': results_explanation,
'total_profit': total_profit,
'current_epoch': 1,
'is_initial_point': True,
}
result = hyperopt._format_explanation_string(results, 1)
assert result.find(' 66.67%')
assert result.find('Total profit 1.00000000 BTC')
assert result.find('2.0000Σ %')
@ -486,7 +502,9 @@ def test_format_results(hyperopt):
('XPR/EUR', -1, -2, -246)
]
df = pd.DataFrame.from_records(trades, columns=labels)
result = hyperopt.format_results(df)
results_metrics = hyperopt._calculate_results_metrics(df)
results['total_profit'] = results_metrics['total_profit']
result = hyperopt._format_explanation_string(results, 1)
assert result.find('Total profit 1.00000000 EUR')
@ -623,10 +641,42 @@ def test_generate_optimizer(mocker, default_conf) -> None:
}
response_expected = {
'loss': 1.9840569076926293,
'results_explanation': (' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC '
'results_explanation': (' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC '
'( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). Avg duration 100.0 mins.'
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'),
'params': optimizer_param,
'params_details': {'buy': {'adx-enabled': False,
'adx-value': 0,
'fastd-enabled': True,
'fastd-value': 35,
'mfi-enabled': False,
'mfi-value': 0,
'rsi-enabled': False,
'rsi-value': 0,
'trigger': 'macd_cross_signal'},
'roi': {0: 0.12000000000000001,
20.0: 0.02,
50.0: 0.01,
110.0: 0},
'sell': {'sell-adx-enabled': False,
'sell-adx-value': 0,
'sell-fastd-enabled': True,
'sell-fastd-value': 75,
'sell-mfi-enabled': False,
'sell-mfi-value': 0,
'sell-rsi-enabled': False,
'sell-rsi-value': 0,
'sell-trigger': 'macd_cross_signal'},
'stoploss': {'stoploss': -0.4},
'trailing': {'trailing_only_offset_is_reached': False,
'trailing_stop': True,
'trailing_stop_positive': 0.02,
'trailing_stop_positive_offset': 0.1}},
'params_dict': optimizer_param,
'results_metrics': {'avg_profit': 2.3117,
'duration': 100.0,
'profit': 2.3117,
'total_profit': 0.000233,
'trade_count': 1},
'total_profit': 0.00023300
}
@ -682,7 +732,11 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
parallel = mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}])
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {},
'params_details': {'buy': {'mfi-value': None},
'sell': {'sell-mfi-value': None},
'roi': {}, 'stoploss': {'stoploss': None},
'trailing': {'trailing_stop': None}}}])
)
patch_exchange(mocker)
@ -704,7 +758,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None:
parallel.assert_called_once()
out, err = capsys.readouterr()
assert '{"params":{"mfi-value":null,"fastd-value":null,"adx-value":null,"rsi-value":null,"mfi-enabled":null,"fastd-enabled":null,"adx-enabled":null,"rsi-enabled":null,"trigger":null,"sell-mfi-value":null,"sell-fastd-value":null,"sell-adx-value":null,"sell-rsi-value":null,"sell-mfi-enabled":null,"sell-fastd-enabled":null,"sell-adx-enabled":null,"sell-rsi-enabled":null,"sell-trigger":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null,"trailing_stop_positive":null,"trailing_stop_positive_offset":null,"trailing_only_offset_is_reached":null}' in out # noqa: E501
assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null}' in out # noqa: E501
assert dumper.called
# Should be called twice, once for tickerdata, once to save evaluations
assert dumper.call_count == 2
@ -721,7 +775,10 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
parallel = mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}])
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {},
'params_details': {'buy': {'mfi-value': None},
'sell': {'sell-mfi-value': None},
'roi': {}, 'stoploss': {'stoploss': None}}}])
)
patch_exchange(mocker)
@ -743,7 +800,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None
parallel.assert_called_once()
out, err = capsys.readouterr()
assert '{"params":{"mfi-value":null,"fastd-value":null,"adx-value":null,"rsi-value":null,"mfi-enabled":null,"fastd-enabled":null,"adx-enabled":null,"rsi-enabled":null,"trigger":null,"sell-mfi-value":null,"sell-fastd-value":null,"sell-adx-value":null,"sell-rsi-value":null,"sell-mfi-enabled":null,"sell-fastd-enabled":null,"sell-adx-enabled":null,"sell-rsi-enabled":null,"sell-trigger":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501
assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501
assert dumper.called
# Should be called twice, once for tickerdata, once to save evaluations
assert dumper.call_count == 2
@ -760,7 +817,8 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) ->
parallel = mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}])
MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {},
'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}}])
)
patch_exchange(mocker)

131
tests/test_utils.py
View File

@ -10,7 +10,8 @@ from freqtrade.utils import (setup_utils_configuration, start_create_userdir,
start_download_data, start_list_exchanges,
start_list_markets, start_list_timeframes,
start_new_hyperopt, start_new_strategy,
start_test_pairlist, start_trading)
start_test_pairlist, start_trading,
start_hyperopt_list, start_hyperopt_show)
from tests.conftest import (get_args, log_has, log_has_re, patch_exchange,
patched_configuration_load_config_file)
@ -657,3 +658,131 @@ def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, cap
captured = capsys.readouterr()
assert re.match(r"Pairs for .*", captured.out)
assert re.match("['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', 'XRP/BTC']", captured.out)
def test_hyperopt_list(mocker, capsys, hyperopt_results):
mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.load_previous_results',
MagicMock(return_value=hyperopt_results)
)
args = [
"hyperopt-list",
"--no-details"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_list(pargs)
captured = capsys.readouterr()
assert all(x in captured.out
for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12",
" 6/12", " 7/12", " 8/12", " 9/12", " 10/12",
" 11/12", " 12/12"])
args = [
"hyperopt-list",
"--best",
"--no-details"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_list(pargs)
captured = capsys.readouterr()
assert all(x in captured.out
for x in [" 1/12", " 5/12", " 10/12"])
assert all(x not in captured.out
for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12",
" 11/12", " 12/12"])
args = [
"hyperopt-list",
"--profitable",
"--no-details"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_list(pargs)
captured = capsys.readouterr()
assert all(x in captured.out
for x in [" 2/12", " 10/12"])
assert all(x not in captured.out
for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12",
" 11/12", " 12/12"])
def test_hyperopt_show(mocker, capsys, hyperopt_results):
mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.load_previous_results',
MagicMock(return_value=hyperopt_results)
)
args = [
"hyperopt-show",
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_show(pargs)
captured = capsys.readouterr()
assert " 12/12" in captured.out
args = [
"hyperopt-show",
"--best"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_show(pargs)
captured = capsys.readouterr()
assert " 10/12" in captured.out
args = [
"hyperopt-show",
"-n", "1"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_show(pargs)
captured = capsys.readouterr()
assert " 1/12" in captured.out
args = [
"hyperopt-show",
"--best",
"-n", "2"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_show(pargs)
captured = capsys.readouterr()
assert " 5/12" in captured.out
args = [
"hyperopt-show",
"--best",
"-n", "-1"
]
pargs = get_args(args)
pargs['config'] = None
start_hyperopt_show(pargs)
captured = capsys.readouterr()
assert " 10/12" in captured.out
args = [
"hyperopt-show",
"--best",
"-n", "-4"
]
pargs = get_args(args)
pargs['config'] = None
with pytest.raises(OperationalException,
match="The index of the epoch to show should be greater than -4."):
start_hyperopt_show(pargs)
args = [
"hyperopt-show",
"--best",
"-n", "4"
]
pargs = get_args(args)
pargs['config'] = None
with pytest.raises(OperationalException,
match="The index of the epoch to show should be less than 4."):
start_hyperopt_show(pargs)