From e1e2829ef3b5b1cbd77cac106039ceddda8ab621 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 28 Oct 2020 14:36:19 +0100 Subject: [PATCH] Improve and refactor hyperopt tests --- tests/optimize/conftest.py | 51 +++++++ tests/optimize/test_hyperopt.py | 206 +--------------------------- tests/optimize/test_hyperoptloss.py | 165 ++++++++++++++++++++++ 3 files changed, 219 insertions(+), 203 deletions(-) create mode 100644 tests/optimize/conftest.py create mode 100644 tests/optimize/test_hyperoptloss.py diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py new file mode 100644 index 000000000..f06b0ecd3 --- /dev/null +++ b/tests/optimize/conftest.py @@ -0,0 +1,51 @@ +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import pandas as pd +import pytest + +from freqtrade.optimize.hyperopt import Hyperopt +from freqtrade.strategy.interface import SellType +from tests.conftest import patch_exchange + + +@pytest.fixture(scope='function') +def hyperopt_conf(default_conf): + hyperconf = deepcopy(default_conf) + hyperconf.update({ + 'hyperopt': 'DefaultHyperOpt', + 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', + 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), + 'epochs': 1, + 'timerange': None, + 'spaces': ['default'], + 'hyperopt_jobs': 1, + }) + return hyperconf + + +@pytest.fixture(scope='function') +def hyperopt(hyperopt_conf, mocker): + + patch_exchange(mocker) + return Hyperopt(hyperopt_conf) + + +@pytest.fixture(scope='function') +def hyperopt_results(): + return pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [-0.1, 0.2, 0.3], + 'profit_abs': [-0.2, 0.4, 0.6], + 'trade_duration': [10, 30, 10], + 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], + 'close_date': + [ + datetime(2019, 1, 1, 9, 26, 3, 478039), + datetime(2019, 2, 1, 9, 26, 3, 478039), + datetime(2019, 3, 1, 9, 26, 3, 478039) + ] + } + ) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 41ad6f5de..82be894d3 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -2,7 +2,6 @@ import locale import logging import re -from copy import deepcopy from datetime import datetime from pathlib import Path from typing import Dict, List @@ -17,58 +16,15 @@ from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.optimize.default_hyperopt_loss import ShortTradeDurHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt -from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver +from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from .hyperopts.default_hyperopt import DefaultHyperOpt -@pytest.fixture(scope='function') -def hyperopt_conf(default_conf): - hyperconf = deepcopy(default_conf) - hyperconf.update({ - 'hyperopt': 'DefaultHyperOpt', - 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', - 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), - 'epochs': 1, - 'timerange': None, - 'spaces': ['default'], - 'hyperopt_jobs': 1, - }) - return hyperconf - - -@pytest.fixture(scope='function') -def hyperopt(hyperopt_conf, mocker): - - patch_exchange(mocker) - return Hyperopt(hyperopt_conf) - - -@pytest.fixture(scope='function') -def hyperopt_results(): - return pd.DataFrame( - { - 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_percent': [-0.1, 0.2, 0.3], - 'profit_abs': [-0.2, 0.4, 0.6], - 'trade_duration': [10, 30, 10], - 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], - 'close_date': - [ - datetime(2019, 1, 1, 9, 26, 3, 478039), - datetime(2019, 2, 1, 9, 26, 3, 478039), - datetime(2019, 3, 1, 9, 26, 3, 478039) - ] - } - ) - - # Functions for recurrent object patching def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: """ @@ -230,32 +186,6 @@ def test_hyperoptresolver_noname(default_conf): HyperOptResolver.load_hyperopt(default_conf) -def test_hyperoptlossresolver_noname(default_conf): - with pytest.raises(OperationalException, - match="No Hyperopt loss set. Please use `--hyperopt-loss` to specify " - "the Hyperopt-Loss class to use."): - HyperOptLossResolver.load_hyperoptloss(default_conf) - - -def test_hyperoptlossresolver(mocker, default_conf) -> None: - - hl = ShortTradeDurHyperOptLoss - mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', - MagicMock(return_value=hl) - ) - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) - x = HyperOptLossResolver.load_hyperoptloss(default_conf) - assert hasattr(x, "hyperopt_loss_function") - - -def test_hyperoptlossresolver_wrongname(default_conf) -> None: - default_conf.update({'hyperopt_loss': "NonExistingLossClass"}) - - with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'): - HyperOptLossResolver.load_hyperoptloss(default_conf) - - def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) @@ -269,7 +199,8 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: '--hyperopt', 'DefaultHyperOpt', '--hyperopt-path', str(Path(__file__).parent / "hyperopts"), - '--epochs', '5' + '--epochs', '5', + '--hyperopt-loss', 'SharpeHyperOptLossDaily', ] pargs = get_args(args) @@ -337,137 +268,6 @@ def test_start_filelock(mocker, hyperopt_conf, caplog) -> None: assert log_has("Another running instance of freqtrade Hyperopt detected.", caplog) -def test_loss_calculation_prefer_correct_trade_count(hyperopt_conf, hyperopt_results) -> None: - hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over > correct - assert under > correct - - -def test_loss_calculation_prefer_shorter_trades(hyperopt_conf, hyperopt_results) -> None: - resultsb = hyperopt_results.copy() - resultsb.loc[1, 'trade_duration'] = 20 - - hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) - longer = hl.hyperopt_loss_function(hyperopt_results, 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - shorter = hl.hyperopt_loss_function(resultsb, 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert shorter < longer - - -def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - def test_log_results_if_loss_improves(hyperopt, capsys) -> None: hyperopt.current_best_loss = 2 hyperopt.total_epochs = 2 diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py new file mode 100644 index 000000000..63012ee48 --- /dev/null +++ b/tests/optimize/test_hyperoptloss.py @@ -0,0 +1,165 @@ +from datetime import datetime +from unittest.mock import MagicMock + +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.default_hyperopt_loss import ShortTradeDurHyperOptLoss +from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver + + +def test_hyperoptlossresolver_noname(default_conf): + with pytest.raises(OperationalException, + match="No Hyperopt loss set. Please use `--hyperopt-loss` to specify " + "the Hyperopt-Loss class to use."): + HyperOptLossResolver.load_hyperoptloss(default_conf) + + +def test_hyperoptlossresolver(mocker, default_conf) -> None: + + hl = ShortTradeDurHyperOptLoss + mocker.patch( + 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', + MagicMock(return_value=hl) + ) + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) + x = HyperOptLossResolver.load_hyperoptloss(default_conf) + assert hasattr(x, "hyperopt_loss_function") + + +def test_hyperoptlossresolver_wrongname(default_conf) -> None: + default_conf.update({'hyperopt_loss': "NonExistingLossClass"}) + + with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'): + HyperOptLossResolver.load_hyperoptloss(default_conf) + + +def test_loss_calculation_prefer_correct_trade_count(hyperopt_conf, hyperopt_results) -> None: + hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over > correct + assert under > correct + + +def test_loss_calculation_prefer_shorter_trades(hyperopt_conf, hyperopt_results) -> None: + resultsb = hyperopt_results.copy() + resultsb.loc[1, 'trade_duration'] = 20 + + hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) + longer = hl.hyperopt_loss_function(hyperopt_results, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + shorter = hl.hyperopt_loss_function(resultsb, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert shorter < longer + + +def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct