stable/tests/optimize/test_hyperopt_tools.py

336 lines
12 KiB
Python
Raw Normal View History

2021-06-14 19:06:15 +00:00
import logging
import re
from pathlib import Path
from typing import Dict, List
2021-07-03 13:45:00 +00:00
import numpy as np
2021-06-14 19:06:15 +00:00
import pytest
2021-06-15 04:52:12 +00:00
import rapidjson
2021-06-14 19:06:15 +00:00
2021-06-30 04:33:40 +00:00
from freqtrade.constants import FTHYPT_FILEVERSION
2021-06-14 19:06:15 +00:00
from freqtrade.exceptions import OperationalException
2021-07-03 06:38:55 +00:00
from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
2022-01-22 16:25:21 +00:00
from tests.conftest import CURRENT_TEST_STRATEGY, log_has, log_has_re
2021-06-14 19:06:15 +00:00
# Functions for recurrent object patching
def create_results() -> List[Dict]:
return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}]
def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None:
hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt')
hyperopt_epochs = HyperoptTools.load_filtered_results(hyperopt.results_file, {})
assert log_has_re("Hyperopt file .* not found.", caplog)
assert hyperopt_epochs == ([], 0)
2021-06-14 19:06:15 +00:00
# Test writing to temp dir and reading again
epochs = create_results()
caplog.set_level(logging.DEBUG)
for epoch in epochs:
hyperopt._save_result(epoch)
assert log_has(f"1 epoch saved to '{hyperopt.results_file}'.", caplog)
hyperopt._save_result(epochs[0])
assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog)
hyperopt_epochs = HyperoptTools.load_filtered_results(hyperopt.results_file, {})
2021-06-14 19:06:15 +00:00
assert len(hyperopt_epochs) == 2
assert hyperopt_epochs[1] == 2
assert len(hyperopt_epochs[0]) == 2
result_gen = HyperoptTools._read_results(hyperopt.results_file, 1)
epoch = next(result_gen)
assert len(epoch) == 1
assert epoch[0] == epochs[0]
epoch = next(result_gen)
assert len(epoch) == 1
epoch = next(result_gen)
assert len(epoch) == 0
with pytest.raises(StopIteration):
next(result_gen)
2021-06-14 19:06:15 +00:00
def test_load_previous_results2(mocker, testdatadir, caplog) -> None:
results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle'
2021-08-09 05:03:13 +00:00
with pytest.raises(OperationalException,
match=r"Legacy hyperopt results are no longer supported.*"):
HyperoptTools.load_filtered_results(results_file, {})
2021-06-14 19:06:15 +00:00
@pytest.mark.parametrize("spaces, expected_results", [
(['buy'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['sell'],
2021-08-25 17:54:26 +00:00
{'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['roi'],
2021-08-25 17:54:26 +00:00
{'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['stoploss'],
2021-08-25 17:54:26 +00:00
{'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['trailing'],
2021-08-25 17:54:26 +00:00
{'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['buy', 'sell', 'roi', 'stoploss'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['buy', 'sell', 'roi', 'stoploss', 'trailing'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['buy', 'roi'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['all'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True,
'protection': True}),
2021-06-14 19:06:15 +00:00
(['default'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['default', 'trailing'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True,
'protection': False}),
2021-06-14 19:06:15 +00:00
(['all', 'buy'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True,
'protection': True}),
2021-06-14 19:06:15 +00:00
(['default', 'buy'],
2021-08-25 17:54:26 +00:00
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False,
'protection': False}),
(['all'],
{'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True,
'protection': True}),
(['protection'],
{'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False,
'protection': True}),
2021-06-14 19:06:15 +00:00
])
def test_has_space(hyperopt_conf, spaces, expected_results):
2021-08-25 17:54:26 +00:00
for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection']:
2021-06-14 19:06:15 +00:00
hyperopt_conf.update({'spaces': spaces})
assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s]
def test_show_epoch_details(capsys):
test_result = {
'params_details': {
'trailing': {
'trailing_stop': True,
'trailing_stop_positive': 0.02,
'trailing_stop_positive_offset': 0.04,
'trailing_only_offset_is_reached': True
},
'roi': {
0: 0.18,
90: 0.14,
225: 0.05,
430: 0},
},
'results_explanation': 'foo result',
'is_initial_point': False,
'total_profit': 0,
'current_epoch': 2, # This starts from 1 (in a human-friendly manner)
'is_best': True
}
HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True)
captured = capsys.readouterr()
assert '# Trailing stop:' in captured.out
# re.match(r"Pairs for .*", captured.out)
assert re.search(r'^\s+trailing_stop = True$', captured.out, re.MULTILINE)
assert re.search(r'^\s+trailing_stop_positive = 0.02$', captured.out, re.MULTILINE)
assert re.search(r'^\s+trailing_stop_positive_offset = 0.04$', captured.out, re.MULTILINE)
assert re.search(r'^\s+trailing_only_offset_is_reached = True$', captured.out, re.MULTILINE)
assert '# ROI table:' in captured.out
assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE)
assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE)
2021-06-15 18:33:35 +00:00
def test__pprint_dict():
2021-06-14 19:06:15 +00:00
params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'}
non_params = {'buy_notoptimied': 55}
2021-06-15 18:33:35 +00:00
x = HyperoptTools._pprint_dict(params, non_params)
2021-06-14 19:06:15 +00:00
assert x == """{
"buy_std": 1.2,
"buy_rsi": 31,
"buy_enable": True,
"buy_what": "asdf",
"buy_notoptimied": 55, # value loaded from strategy
}"""
2021-06-15 04:52:12 +00:00
def test_get_strategy_filename(default_conf):
2021-09-25 17:31:06 +00:00
x = HyperoptTools.get_strategy_filename(default_conf, 'StrategyTestV3')
2021-06-15 04:52:12 +00:00
assert isinstance(x, Path)
assert x == Path(__file__).parents[1] / 'strategy/strats/strategy_test_v3.py'
2021-06-15 04:52:12 +00:00
x = HyperoptTools.get_strategy_filename(default_conf, 'NonExistingStrategy')
assert x is None
def test_export_params(tmpdir):
filename = Path(tmpdir) / f"{CURRENT_TEST_STRATEGY}.json"
2021-06-15 04:52:12 +00:00
assert not filename.is_file()
params = {
"params_details": {
"buy": {
"buy_rsi": 30
},
"sell": {
"sell_rsi": 70
},
"roi": {
"0": 0.528,
2021-06-15 18:15:20 +00:00
"346": 0.08499,
2021-06-15 04:52:12 +00:00
"507": 0.049,
"1595": 0
}
},
"params_not_optimized": {
"stoploss": -0.05,
"trailing": {
"trailing_stop": False,
"trailing_stop_positive": 0.05,
"trailing_stop_positive_offset": 0.1,
"trailing_only_offset_is_reached": True
},
}
}
HyperoptTools.export_params(params, CURRENT_TEST_STRATEGY, filename)
2021-06-15 04:52:12 +00:00
assert filename.is_file()
with filename.open('r') as f:
content = rapidjson.load(f)
assert content['strategy_name'] == CURRENT_TEST_STRATEGY
2021-06-15 04:52:12 +00:00
assert 'params' in content
assert "buy" in content["params"]
assert "sell" in content["params"]
assert "roi" in content["params"]
assert "stoploss" in content["params"]
assert "trailing" in content["params"]
2021-06-30 04:33:40 +00:00
def test_try_export_params(default_conf, tmpdir, caplog, mocker):
default_conf['disableparamexport'] = False
export_mock = mocker.patch("freqtrade.optimize.hyperopt_tools.HyperoptTools.export_params")
filename = Path(tmpdir) / f"{CURRENT_TEST_STRATEGY}.json"
2021-06-30 04:33:40 +00:00
assert not filename.is_file()
params = {
"params_details": {
"buy": {
"buy_rsi": 30
},
"sell": {
"sell_rsi": 70
},
"roi": {
"0": 0.528,
"346": 0.08499,
"507": 0.049,
"1595": 0
}
},
"params_not_optimized": {
"stoploss": -0.05,
"trailing": {
"trailing_stop": False,
"trailing_stop_positive": 0.05,
"trailing_stop_positive_offset": 0.1,
"trailing_only_offset_is_reached": True
},
},
FTHYPT_FILEVERSION: 2,
}
HyperoptTools.try_export_params(default_conf, "StrategyTestVXXX", params)
2021-06-30 04:33:40 +00:00
assert log_has("Strategy not found, not exporting parameter file.", caplog)
assert export_mock.call_count == 0
caplog.clear()
HyperoptTools.try_export_params(default_conf, CURRENT_TEST_STRATEGY, params)
2021-06-30 04:33:40 +00:00
assert export_mock.call_count == 1
assert export_mock.call_args_list[0][0][1] == CURRENT_TEST_STRATEGY
assert export_mock.call_args_list[0][0][2].name == 'strategy_test_v3.json'
2021-06-30 04:33:40 +00:00
def test_params_print(capsys):
params = {
"buy": {
"buy_rsi": 30
},
"sell": {
"sell_rsi": 70
},
}
non_optimized = {
"buy": {
"buy_adx": 44
},
"sell": {
"sell_adx": 65
},
"stoploss": {
"stoploss": -0.05,
},
"roi": {
"0": 0.05,
"20": 0.01,
},
"trailing": {
"trailing_stop": False,
"trailing_stop_positive": 0.05,
"trailing_stop_positive_offset": 0.1,
"trailing_only_offset_is_reached": True
},
}
HyperoptTools._params_pretty_print(params, 'buy', 'No header', non_optimized)
captured = capsys.readouterr()
assert re.search("# No header", captured.out)
assert re.search('"buy_rsi": 30,\n', captured.out)
assert re.search('"buy_adx": 44, # value loaded.*\n', captured.out)
assert not re.search("sell", captured.out)
HyperoptTools._params_pretty_print(params, 'sell', 'Sell Header', non_optimized)
captured = capsys.readouterr()
assert re.search("# Sell Header", captured.out)
assert re.search('"sell_rsi": 70,\n', captured.out)
assert re.search('"sell_adx": 65, # value loaded.*\n', captured.out)
HyperoptTools._params_pretty_print(params, 'roi', 'ROI Table:', non_optimized)
captured = capsys.readouterr()
assert re.search("# ROI Table: # value loaded.*\n", captured.out)
assert re.search('minimal_roi = {\n', captured.out)
assert re.search('"20": 0.01\n', captured.out)
HyperoptTools._params_pretty_print(params, 'trailing', 'Trailing stop:', non_optimized)
captured = capsys.readouterr()
assert re.search("# Trailing stop:", captured.out)
assert re.search('trailing_stop = False # value loaded.*\n', captured.out)
assert re.search('trailing_stop_positive = 0.05 # value loaded.*\n', captured.out)
assert re.search('trailing_stop_positive_offset = 0.1 # value loaded.*\n', captured.out)
assert re.search('trailing_only_offset_is_reached = True # value loaded.*\n', captured.out)
2021-07-03 06:38:55 +00:00
def test_hyperopt_serializer():
assert isinstance(hyperopt_serializer(np.int_(5)), int)
assert isinstance(hyperopt_serializer(np.bool_(True)), bool)
assert isinstance(hyperopt_serializer(np.bool_(False)), bool)