adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #5219 from freqtrade/hyperopt_paramfile

Browse Source 
automatic Hyperopt paramfile
This commit is contained in:
Matthias
2021-07-04 13:56:52 +02:00
committed by GitHub
parent 77293b1f1e dc8abd77df
commit 898bef1837
17 changed files with 588 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
freqtrade/optimize/hyperopt.py
View File

@@ -12,7 +12,6 @@ from math import ceil
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import progressbar
import rapidjson
from colorama import Fore, Style
@@ -20,16 +19,16 @@ from colorama import init as colorama_init
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.misc import file_dump_json, plural
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_auto import HyperOptAuto
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.optimize.hyperopt_tools import HyperoptTools
from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
from freqtrade.optimize.optimize_reports import generate_strategy_stats
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
@@ -78,8 +77,11 @@ class Hyperopt:
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
self.auto_hyperopt = True
else:
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
self.auto_hyperopt = False
self.backtesting._set_strategy(self.backtesting.strategylist[0])
self.custom_hyperopt.strategy = self.backtesting.strategy
@@ -163,13 +165,9 @@ class Hyperopt:
While not a valid json object - this allows appending easily.
:param epoch: result dictionary for this epoch.
"""
def default_parser(x):
if isinstance(x, np.integer):
return int(x)
return str(x)
epoch[FTHYPT_FILEVERSION] = 2
with self.results_file.open('a') as f:
rapidjson.dump(epoch, f, default=default_parser,
rapidjson.dump(epoch, f, default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
f.write("\n")
@@ -201,6 +199,25 @@ class Hyperopt:
return result
def _get_no_optimize_details(self) -> Dict[str, Any]:
"""
Get non-optimized parameters
"""
result: Dict[str, Any] = {}
strategy = self.backtesting.strategy
if not HyperoptTools.has_space(self.config, 'roi'):
result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()}
if not HyperoptTools.has_space(self.config, 'stoploss'):
result['stoploss'] = {'stoploss': strategy.stoploss}
if not HyperoptTools.has_space(self.config, 'trailing'):
result['trailing'] = {
'trailing_stop': strategy.trailing_stop,
'trailing_stop_positive': strategy.trailing_stop_positive,
'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset,
'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached,
}
return result
def print_results(self, results) -> None:
"""
Log results if it is better than any previous evaluation
@@ -310,7 +327,8 @@ class Hyperopt:
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
not_optimized = self.backtesting.strategy.get_params_dict()
not_optimized = self.backtesting.strategy.get_no_optimize_params()
not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details())
trade_count = strat_stats['total_trades']
total_profit = strat_stats['profit_total']
@@ -470,6 +488,12 @@ class Hyperopt:
f"saved to '{self.results_file}'.")
if self.current_best_epoch:
if self.auto_hyperopt:
HyperoptTools.try_export_params(
self.config,
self.backtesting.strategy.get_strategy_name(),
self.current_best_epoch)
HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
self.print_json)
else:

104
freqtrade/optimize/hyperopt_tools.py
View File

@@ -1,23 +1,82 @@
import io
import logging
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
import numpy as np
import rapidjson
import tabulate
from colorama import Fore, Style
from pandas import isna, json_normalize
from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.misc import round_coin_value, round_dict
from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
logger = logging.getLogger(__name__)
NON_OPT_PARAM_APPENDIX = " # value loaded from strategy"
def hyperopt_serializer(x):
if isinstance(x, np.integer):
return int(x)
if isinstance(x, np.bool_):
return bool(x)
return str(x)
class HyperoptTools():
@staticmethod
def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]:
"""
Get Strategy-location (filename) from strategy_name
"""
from freqtrade.resolvers.strategy_resolver import StrategyResolver
directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
strategy_objs = StrategyResolver.search_all_objects(directory, False)
strategies = [s for s in strategy_objs if s['name'] == strategy_name]
if strategies:
strategy = strategies[0]
return Path(strategy['location'])
return None
@staticmethod
def export_params(params, strategy_name: str, filename: Path):
"""
Generate files
"""
final_params = deepcopy(params['params_not_optimized'])
final_params = deep_merge_dicts(params['params_details'], final_params)
final_params = {
'strategy_name': strategy_name,
'params': final_params,
'ft_stratparam_v': 1,
'export_time': datetime.now(timezone.utc),
}
logger.info(f"Dumping parameters to {filename}")
rapidjson.dump(final_params, filename.open('w'), indent=2,
default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
)
@staticmethod
def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict):
if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False):
# Export parameters ...
fn = HyperoptTools.get_strategy_filename(config, strategy_name)
if fn:
HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json'))
else:
logger.warn("Strategy not found, not exporting parameter file.")
@staticmethod
def has_space(config: Dict[str, Any], space: str) -> bool:
"""
@@ -99,9 +158,9 @@ class HyperoptTools():
non_optimized)
HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
non_optimized)
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized)
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized)
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized)
@staticmethod
def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None:
@@ -127,23 +186,34 @@ class HyperoptTools():
def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
if space in params or space in non_optimized:
space_params = HyperoptTools._space_params(params, space, 5)
no_params = HyperoptTools._space_params(non_optimized, space, 5)
appendix = ''
if not space_params and not no_params:
# No parameters - don't print
return
if not space_params:
# Not optimized parameters - append string
appendix = NON_OPT_PARAM_APPENDIX
result = f"\n# {header}\n"
if space == 'stoploss':
result += f"stoploss = {space_params.get('stoploss')}"
elif space == 'roi':
if space == "stoploss":
stoploss = safe_value_fallback2(space_params, no_params, space, space)
result += (f"stoploss = {stoploss}{appendix}")
elif space == "roi":
result = result[:-1] + f'{appendix}\n'
minimal_roi_result = rapidjson.dumps({
str(k): v for k, v in space_params.items()
str(k): v for k, v in (space_params or no_params).items()
}, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
result += f"minimal_roi = {minimal_roi_result}"
elif space == 'trailing':
for k, v in space_params.items():
result += f'{k} = {v}\n'
elif space == "trailing":
for k, v in (space_params or no_params).items():
result += f"{k} = {v}{appendix}\n"
else:
no_params = HyperoptTools._space_params(non_optimized, space, 5)
# Buy / sell parameters
result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}"
result = result.replace("\n", "\n ")
print(result)
@@ -157,7 +227,7 @@ class HyperoptTools():
return {}
@staticmethod
def _pprint(params, non_optimized, indent: int = 4):
def _pprint_dict(params, non_optimized, indent: int = 4):
"""
Pretty-print hyperopt results (based on 2 dicts - with add. comment)
"""
@@ -169,7 +239,7 @@ class HyperoptTools():
result += " " * indent + f'"{k}": '
result += f'"{param}",' if isinstance(param, str) else f'{param},'
if k in non_optimized:
result += " # value loaded from strategy"
result += NON_OPT_PARAM_APPENDIX
result += "\n"
result += '}'
return result