Merge pull request #6563 from italodamato/opt-ask-force-new-points
Optimize only new points
This commit is contained in:
commit
7328553c0b
@ -10,7 +10,7 @@ import warnings
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from math import ceil
|
from math import ceil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
import progressbar
|
import progressbar
|
||||||
import rapidjson
|
import rapidjson
|
||||||
@ -409,6 +409,51 @@ class Hyperopt:
|
|||||||
# Store non-trimmed data - will be trimmed after signal generation.
|
# Store non-trimmed data - will be trimmed after signal generation.
|
||||||
dump(preprocessed, self.data_pickle_file)
|
dump(preprocessed, self.data_pickle_file)
|
||||||
|
|
||||||
|
def get_asked_points(self, n_points: int) -> Tuple[List[List[Any]], List[bool]]:
|
||||||
|
'''
|
||||||
|
Enforce points returned from `self.opt.ask` have not been already evaluated
|
||||||
|
|
||||||
|
Steps:
|
||||||
|
1. Try to get points using `self.opt.ask` first
|
||||||
|
2. Discard the points that have already been evaluated
|
||||||
|
3. Retry using `self.opt.ask` up to 3 times
|
||||||
|
4. If still some points are missing in respect to `n_points`, random sample some points
|
||||||
|
5. Repeat until at least `n_points` points in the `asked_non_tried` list
|
||||||
|
6. Return a list with length truncated at `n_points`
|
||||||
|
'''
|
||||||
|
def unique_list(a_list):
|
||||||
|
new_list = []
|
||||||
|
for item in a_list:
|
||||||
|
if item not in new_list:
|
||||||
|
new_list.append(item)
|
||||||
|
return new_list
|
||||||
|
i = 0
|
||||||
|
asked_non_tried: List[List[Any]] = []
|
||||||
|
is_random: List[bool] = []
|
||||||
|
while i < 5 and len(asked_non_tried) < n_points:
|
||||||
|
if i < 3:
|
||||||
|
self.opt.cache_ = {}
|
||||||
|
asked = unique_list(self.opt.ask(n_points=n_points * 5))
|
||||||
|
is_random = [False for _ in range(len(asked))]
|
||||||
|
else:
|
||||||
|
asked = unique_list(self.opt.space.rvs(n_samples=n_points * 5))
|
||||||
|
is_random = [True for _ in range(len(asked))]
|
||||||
|
is_random += [rand for x, rand in zip(asked, is_random)
|
||||||
|
if x not in self.opt.Xi
|
||||||
|
and x not in asked_non_tried]
|
||||||
|
asked_non_tried += [x for x in asked
|
||||||
|
if x not in self.opt.Xi
|
||||||
|
and x not in asked_non_tried]
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
if asked_non_tried:
|
||||||
|
return (
|
||||||
|
asked_non_tried[:min(len(asked_non_tried), n_points)],
|
||||||
|
is_random[:min(len(asked_non_tried), n_points)]
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
return self.opt.ask(n_points=n_points), [False for _ in range(n_points)]
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
|
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
|
||||||
logger.info(f"Using optimizer random state: {self.random_state}")
|
logger.info(f"Using optimizer random state: {self.random_state}")
|
||||||
@ -473,7 +518,7 @@ class Hyperopt:
|
|||||||
n_rest = (i + 1) * jobs - self.total_epochs
|
n_rest = (i + 1) * jobs - self.total_epochs
|
||||||
current_jobs = jobs - n_rest if n_rest > 0 else jobs
|
current_jobs = jobs - n_rest if n_rest > 0 else jobs
|
||||||
|
|
||||||
asked = self.opt.ask(n_points=current_jobs)
|
asked, is_random = self.get_asked_points(n_points=current_jobs)
|
||||||
f_val = self.run_optimizer_parallel(parallel, asked, i)
|
f_val = self.run_optimizer_parallel(parallel, asked, i)
|
||||||
self.opt.tell(asked, [v['loss'] for v in f_val])
|
self.opt.tell(asked, [v['loss'] for v in f_val])
|
||||||
|
|
||||||
@ -492,6 +537,7 @@ class Hyperopt:
|
|||||||
# evaluations can take different time. Here they are aligned in the
|
# evaluations can take different time. Here they are aligned in the
|
||||||
# order they will be shown to the user.
|
# order they will be shown to the user.
|
||||||
val['is_best'] = is_best
|
val['is_best'] = is_best
|
||||||
|
val['is_random'] = is_random[j]
|
||||||
self.print_results(val)
|
self.print_results(val)
|
||||||
|
|
||||||
if is_best:
|
if is_best:
|
||||||
|
@ -310,6 +310,8 @@ class HyperoptTools():
|
|||||||
if not has_drawdown:
|
if not has_drawdown:
|
||||||
# Ensure compatibility with older versions of hyperopt results
|
# Ensure compatibility with older versions of hyperopt results
|
||||||
trials['results_metrics.max_drawdown_account'] = None
|
trials['results_metrics.max_drawdown_account'] = None
|
||||||
|
if 'is_random' not in trials.columns:
|
||||||
|
trials['is_random'] = False
|
||||||
|
|
||||||
# New mode, using backtest result for metrics
|
# New mode, using backtest result for metrics
|
||||||
trials['results_metrics.winsdrawslosses'] = trials.apply(
|
trials['results_metrics.winsdrawslosses'] = trials.apply(
|
||||||
@ -322,12 +324,12 @@ class HyperoptTools():
|
|||||||
'results_metrics.profit_total', 'results_metrics.holding_avg',
|
'results_metrics.profit_total', 'results_metrics.holding_avg',
|
||||||
'results_metrics.max_drawdown',
|
'results_metrics.max_drawdown',
|
||||||
'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs',
|
'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs',
|
||||||
'loss', 'is_initial_point', 'is_best']]
|
'loss', 'is_initial_point', 'is_random', 'is_best']]
|
||||||
|
|
||||||
trials.columns = [
|
trials.columns = [
|
||||||
'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
|
'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
|
||||||
'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account',
|
'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account',
|
||||||
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'
|
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_random', 'is_best'
|
||||||
]
|
]
|
||||||
|
|
||||||
return trials
|
return trials
|
||||||
@ -349,9 +351,11 @@ class HyperoptTools():
|
|||||||
trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown)
|
trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown)
|
||||||
|
|
||||||
trials['is_profit'] = False
|
trials['is_profit'] = False
|
||||||
trials.loc[trials['is_initial_point'], 'Best'] = '* '
|
trials.loc[trials['is_initial_point'] | trials['is_random'], 'Best'] = '* '
|
||||||
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
||||||
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
|
trials.loc[
|
||||||
|
(trials['is_initial_point'] | trials['is_random']) & trials['is_best'],
|
||||||
|
'Best'] = '* Best'
|
||||||
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
||||||
trials['Trades'] = trials['Trades'].astype(str)
|
trials['Trades'] = trials['Trades'].astype(str)
|
||||||
# perc_multi = 1 if legacy_mode else 100
|
# perc_multi = 1 if legacy_mode else 100
|
||||||
@ -407,7 +411,7 @@ class HyperoptTools():
|
|||||||
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
|
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
|
||||||
str(trials.loc[i][j]), Style.RESET_ALL)
|
str(trials.loc[i][j]), Style.RESET_ALL)
|
||||||
|
|
||||||
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit', 'is_random'])
|
||||||
if remove_header > 0:
|
if remove_header > 0:
|
||||||
table = tabulate.tabulate(
|
table = tabulate.tabulate(
|
||||||
trials.to_dict(orient='list'), tablefmt='orgtbl',
|
trials.to_dict(orient='list'), tablefmt='orgtbl',
|
||||||
|
@ -2672,6 +2672,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.00125625,
|
'total_profit': -0.00125625,
|
||||||
'current_epoch': 1,
|
'current_epoch': 1,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': True,
|
'is_best': True,
|
||||||
|
|
||||||
}, {
|
}, {
|
||||||
@ -2688,6 +2689,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': 6.185e-05,
|
'total_profit': 6.185e-05,
|
||||||
'current_epoch': 2,
|
'current_epoch': 2,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': 14.241196856510731,
|
'loss': 14.241196856510731,
|
||||||
@ -2698,6 +2700,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.13639474,
|
'total_profit': -0.13639474,
|
||||||
'current_epoch': 3,
|
'current_epoch': 3,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': 100000,
|
'loss': 100000,
|
||||||
@ -2705,7 +2708,7 @@ def saved_hyperopt_results():
|
|||||||
'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501
|
'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501
|
||||||
'results_metrics': {'total_trades': 0, 'wins': 0, 'draws': 0, 'losses': 0, 'profit_mean': None, 'profit_median': None, 'profit_total': 0, 'profit': 0.0, 'holding_avg': timedelta()}, # noqa: E501
|
'results_metrics': {'total_trades': 0, 'wins': 0, 'draws': 0, 'losses': 0, 'profit_mean': None, 'profit_median': None, 'profit_total': 0, 'profit': 0.0, 'holding_avg': timedelta()}, # noqa: E501
|
||||||
'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501
|
'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501
|
||||||
'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False
|
'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_random': False, 'is_best': False # noqa: E501
|
||||||
}, {
|
}, {
|
||||||
'loss': 0.22195522184191518,
|
'loss': 0.22195522184191518,
|
||||||
'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501
|
'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501
|
||||||
@ -2715,6 +2718,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.002480140000000001,
|
'total_profit': -0.002480140000000001,
|
||||||
'current_epoch': 5,
|
'current_epoch': 5,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': True
|
'is_best': True
|
||||||
}, {
|
}, {
|
||||||
'loss': 0.545315889154162,
|
'loss': 0.545315889154162,
|
||||||
@ -2725,6 +2729,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.0041773,
|
'total_profit': -0.0041773,
|
||||||
'current_epoch': 6,
|
'current_epoch': 6,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': 4.713497421432944,
|
'loss': 4.713497421432944,
|
||||||
@ -2737,6 +2742,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.06339929,
|
'total_profit': -0.06339929,
|
||||||
'current_epoch': 7,
|
'current_epoch': 7,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': 20.0, # noqa: E501
|
'loss': 20.0, # noqa: E501
|
||||||
@ -2747,6 +2753,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': 0.0,
|
'total_profit': 0.0,
|
||||||
'current_epoch': 8,
|
'current_epoch': 8,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': 2.4731817780991223,
|
'loss': 2.4731817780991223,
|
||||||
@ -2757,6 +2764,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.044050070000000004, # noqa: E501
|
'total_profit': -0.044050070000000004, # noqa: E501
|
||||||
'current_epoch': 9,
|
'current_epoch': 9,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': -0.2604606005845212, # noqa: E501
|
'loss': -0.2604606005845212, # noqa: E501
|
||||||
@ -2767,6 +2775,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': 0.00021629,
|
'total_profit': 0.00021629,
|
||||||
'current_epoch': 10,
|
'current_epoch': 10,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': True
|
'is_best': True
|
||||||
}, {
|
}, {
|
||||||
'loss': 4.876465945994304, # noqa: E501
|
'loss': 4.876465945994304, # noqa: E501
|
||||||
@ -2778,6 +2787,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': -0.07436117,
|
'total_profit': -0.07436117,
|
||||||
'current_epoch': 11,
|
'current_epoch': 11,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}, {
|
}, {
|
||||||
'loss': 100000,
|
'loss': 100000,
|
||||||
@ -2788,6 +2798,7 @@ def saved_hyperopt_results():
|
|||||||
'total_profit': 0,
|
'total_profit': 0,
|
||||||
'current_epoch': 12,
|
'current_epoch': 12,
|
||||||
'is_initial_point': True,
|
'is_initial_point': True,
|
||||||
|
'is_random': False,
|
||||||
'is_best': False
|
'is_best': False
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
@ -41,6 +41,7 @@ def generate_result_metrics():
|
|||||||
'max_drawdown_abs': 0.001,
|
'max_drawdown_abs': 0.001,
|
||||||
'loss': 0.001,
|
'loss': 0.001,
|
||||||
'is_initial_point': 0.001,
|
'is_initial_point': 0.001,
|
||||||
|
'is_random': False,
|
||||||
'is_best': 1,
|
'is_best': 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -247,6 +248,7 @@ def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
|
|||||||
'total_profit': 0,
|
'total_profit': 0,
|
||||||
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
|
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
|
||||||
'is_initial_point': False,
|
'is_initial_point': False,
|
||||||
|
'is_random': False,
|
||||||
'is_best': True
|
'is_best': True
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user