stable/freqtrade/tests/optimize/test_hyperopt.py

529 lines
18 KiB
Python
Raw Normal View History

2017-12-26 08:08:10 +00:00
# pragma pylint: disable=missing-docstring,W0212,C0103
import json
import os
2019-04-26 17:51:24 +00:00
from datetime import datetime
from unittest.mock import MagicMock
from filelock import Timeout
2018-03-17 21:44:47 +00:00
import pandas as pd
import pytest
from freqtrade import DependencyException
from freqtrade.data.converter import parse_ticker_dataframe
2018-12-13 05:35:28 +00:00
from freqtrade.data.history import load_tickerdata_file
2019-01-06 18:38:32 +00:00
from freqtrade.optimize.default_hyperopt import DefaultHyperOpts
from freqtrade.optimize.hyperopt import Hyperopt, HYPEROPT_LOCKFILE
from freqtrade.optimize import setup_configuration, start_hyperopt
2019-05-25 14:53:35 +00:00
from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver
from freqtrade.state import RunMode
from freqtrade.tests.conftest import get_args, log_has, log_has_re, patch_exchange
@pytest.fixture(scope='function')
def hyperopt(default_conf, mocker):
patch_exchange(mocker)
return Hyperopt(default_conf)
2017-12-26 08:08:10 +00:00
# Functions for recurrent object patching
def create_trials(mocker, hyperopt) -> None:
"""
When creating trials, mock the hyperopt Trials so that *by default*
- we don't create any pickle'd files in the filesystem
- we might have a pickle'd file so make sure that we return
false when looking for it
"""
hyperopt.trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle')
mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False)
2018-03-06 06:02:03 +00:00
mocker.patch('freqtrade.optimize.hyperopt.os.path.getsize', return_value=1)
mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True)
2018-07-03 19:51:48 +00:00
mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None)
2018-06-24 12:27:53 +00:00
return [{'loss': 1, 'result': 'foo', 'params': {}}]
2017-12-26 08:08:10 +00:00
def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None:
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
args = [
'--config', 'config.json',
'hyperopt'
]
config = setup_configuration(get_args(args), RunMode.HYPEROPT)
assert 'max_open_trades' in config
assert 'stake_currency' in config
assert 'stake_amount' in config
assert 'exchange' in config
assert 'pair_whitelist' in config['exchange']
assert 'datadir' in config
assert log_has(
2019-07-04 17:53:50 +00:00
'Using data directory: {} ...'.format(config['datadir']),
caplog.record_tuples
)
assert 'ticker_interval' in config
2019-04-24 19:24:00 +00:00
assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog.record_tuples)
assert 'live' not in config
assert not log_has('Parameter -l/--live detected ...', caplog.record_tuples)
assert 'position_stacking' not in config
assert not log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
assert 'refresh_pairs' not in config
assert not log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
assert 'timerange' not in config
assert 'runmode' in config
assert config['runmode'] == RunMode.HYPEROPT
def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplog) -> None:
mocker.patch('freqtrade.configuration.open', mocker.mock_open(
read_data=json.dumps(default_conf)
))
mocker.patch('freqtrade.configuration.Configuration._create_datadir', lambda s, c, x: x)
args = [
'--config', 'config.json',
'--datadir', '/foo/bar',
'hyperopt',
'--ticker-interval', '1m',
'--timerange', ':100',
'--refresh-pairs-cached',
'--enable-position-stacking',
'--disable-max-market-positions',
'--epochs', '1000',
'--spaces', 'all',
'--print-all'
]
config = setup_configuration(get_args(args), RunMode.HYPEROPT)
assert 'max_open_trades' in config
assert 'stake_currency' in config
assert 'stake_amount' in config
assert 'exchange' in config
assert 'pair_whitelist' in config['exchange']
assert 'datadir' in config
assert config['runmode'] == RunMode.HYPEROPT
assert log_has(
2019-07-04 17:53:50 +00:00
'Using data directory: {} ...'.format(config['datadir']),
caplog.record_tuples
)
assert 'ticker_interval' in config
2019-04-24 19:24:00 +00:00
assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
caplog.record_tuples)
assert 'position_stacking' in config
assert log_has('Parameter --enable-position-stacking detected ...', caplog.record_tuples)
assert 'use_max_market_positions' in config
assert log_has('Parameter --disable-max-market-positions detected ...', caplog.record_tuples)
assert log_has('max_open_trades set to unlimited ...', caplog.record_tuples)
assert 'refresh_pairs' in config
assert log_has('Parameter -r/--refresh-pairs-cached detected ...', caplog.record_tuples)
assert 'timerange' in config
assert log_has(
'Parameter --timerange detected: {} ...'.format(config['timerange']),
caplog.record_tuples
)
assert 'epochs' in config
2019-04-24 19:12:23 +00:00
assert log_has('Parameter --epochs detected ... Will run Hyperopt with for 1000 epochs ...',
caplog.record_tuples)
assert 'spaces' in config
assert log_has(
'Parameter -s/--spaces detected: {}'.format(config['spaces']),
caplog.record_tuples
)
assert 'print_all' in config
2019-04-24 19:12:23 +00:00
assert log_has('Parameter --print-all detected ...', caplog.record_tuples)
2019-01-06 18:38:32 +00:00
def test_hyperoptresolver(mocker, default_conf, caplog) -> None:
mocker.patch(
'freqtrade.configuration.Configuration._load_config_file',
lambda *args, **kwargs: default_conf
)
hyperopts = DefaultHyperOpts
delattr(hyperopts, 'populate_buy_trend')
delattr(hyperopts, 'populate_sell_trend')
mocker.patch(
'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt',
MagicMock(return_value=hyperopts)
)
x = HyperOptResolver(default_conf, ).hyperopt
assert not hasattr(x, 'populate_buy_trend')
assert not hasattr(x, 'populate_sell_trend')
assert log_has("Custom Hyperopt does not provide populate_sell_trend. "
"Using populate_sell_trend from DefaultStrategy.", caplog.record_tuples)
assert log_has("Custom Hyperopt does not provide populate_buy_trend. "
"Using populate_buy_trend from DefaultStrategy.", caplog.record_tuples)
assert hasattr(x, "ticker_interval")
2019-01-06 18:38:32 +00:00
2018-03-06 06:02:03 +00:00
def test_start(mocker, default_conf, caplog) -> None:
start_mock = MagicMock()
mocker.patch(
'freqtrade.configuration.Configuration._load_config_file',
lambda *args, **kwargs: default_conf
)
2018-03-06 06:02:03 +00:00
mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock)
2018-06-17 20:32:56 +00:00
patch_exchange(mocker)
2018-03-06 06:02:03 +00:00
args = [
'--config', 'config.json',
'hyperopt',
'--epochs', '5'
]
args = get_args(args)
start_hyperopt(args)
2018-03-06 06:02:03 +00:00
import pprint
pprint.pprint(caplog.record_tuples)
assert log_has(
'Starting freqtrade in Hyperopt mode',
caplog.record_tuples
)
assert start_mock.call_count == 1
def test_start_no_data(mocker, default_conf, caplog) -> None:
mocker.patch(
'freqtrade.configuration.Configuration._load_config_file',
lambda *args, **kwargs: default_conf
)
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock(return_value={}))
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
patch_exchange(mocker)
args = [
'--config', 'config.json',
'hyperopt',
'--epochs', '5'
]
args = get_args(args)
start_hyperopt(args)
import pprint
pprint.pprint(caplog.record_tuples)
assert log_has('No data found. Terminating.', caplog.record_tuples)
def test_start_failure(mocker, default_conf, caplog) -> None:
start_mock = MagicMock()
mocker.patch(
'freqtrade.configuration.Configuration._load_config_file',
lambda *args, **kwargs: default_conf
)
mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock)
patch_exchange(mocker)
args = [
'--config', 'config.json',
'--strategy', 'TestStrategy',
'hyperopt',
'--epochs', '5'
]
args = get_args(args)
with pytest.raises(DependencyException):
start_hyperopt(args)
assert log_has(
"Please don't use --strategy for hyperopt.",
caplog.record_tuples
)
def test_start_filelock(mocker, default_conf, caplog) -> None:
start_mock = MagicMock(side_effect=Timeout(HYPEROPT_LOCKFILE))
mocker.patch(
'freqtrade.configuration.Configuration._load_config_file',
lambda *args, **kwargs: default_conf
)
mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock)
patch_exchange(mocker)
args = [
'--config', 'config.json',
'hyperopt',
'--epochs', '5'
]
args = get_args(args)
start_hyperopt(args)
assert log_has(
"Another running instance of freqtrade Hyperopt detected.",
caplog.record_tuples
)
def test_loss_calculation_prefer_correct_trade_count(hyperopt) -> None:
2017-12-26 08:08:10 +00:00
correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20)
over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20)
under = hyperopt.calculate_loss(1, hyperopt.target_trades - 100, 20)
assert over > correct
assert under > correct
2017-12-26 08:08:10 +00:00
def test_loss_calculation_prefer_shorter_trades(hyperopt) -> None:
shorter = hyperopt.calculate_loss(1, 100, 20)
longer = hyperopt.calculate_loss(1, 100, 30)
assert shorter < longer
2017-12-26 08:08:10 +00:00
def test_loss_calculation_has_limited_profit(hyperopt) -> None:
correct = hyperopt.calculate_loss(hyperopt.expected_max_profit, hyperopt.target_trades, 20)
over = hyperopt.calculate_loss(hyperopt.expected_max_profit * 2, hyperopt.target_trades, 20)
under = hyperopt.calculate_loss(hyperopt.expected_max_profit / 2, hyperopt.target_trades, 20)
assert over == correct
assert under > correct
2017-12-26 08:08:10 +00:00
def test_log_results_if_loss_improves(hyperopt, capsys) -> None:
hyperopt.current_best_loss = 2
hyperopt.log_results(
{
'loss': 1,
'current_tries': 1,
'total_tries': 2,
2019-05-11 07:17:46 +00:00
'result': 'foo.',
'initial_point': False
}
)
out, err = capsys.readouterr()
2019-05-11 07:17:46 +00:00
assert ' 2/2: foo. Objective: 1.00000' in out
2017-12-26 08:08:10 +00:00
def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None:
hyperopt.current_best_loss = 2
hyperopt.log_results(
{
'loss': 3,
}
)
assert caplog.record_tuples == []
def test_save_trials_saves_trials(mocker, hyperopt, caplog) -> None:
trials = create_trials(mocker, hyperopt)
2018-07-03 19:51:48 +00:00
mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None)
hyperopt.trials = trials
hyperopt.save_trials()
2018-03-26 11:14:36 +00:00
trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle')
assert log_has(
2018-06-24 12:27:53 +00:00
'Saving 1 evaluations to \'{}\''.format(trials_file),
caplog.record_tuples
)
mock_dump.assert_called_once()
def test_read_trials_returns_trials_file(mocker, hyperopt, caplog) -> None:
trials = create_trials(mocker, hyperopt)
2018-07-03 19:51:48 +00:00
mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials)
hyperopt_trial = hyperopt.read_trials()
2018-03-26 11:14:36 +00:00
trials_file = os.path.join('freqtrade', 'tests', 'optimize', 'ut_trials.pickle')
assert log_has(
2018-03-26 11:14:36 +00:00
'Reading Trials from \'{}\''.format(trials_file),
caplog.record_tuples
)
assert hyperopt_trial == trials
mock_load.assert_called_once()
2018-01-25 08:45:53 +00:00
def test_roi_table_generation(hyperopt) -> None:
2018-01-25 08:45:53 +00:00
params = {
'roi_t1': 5,
'roi_t2': 10,
'roi_t3': 15,
'roi_p1': 1,
'roi_p2': 2,
'roi_p3': 3,
}
assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0}
def test_start_calls_optimizer(mocker, default_conf, caplog) -> None:
dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
mocker.patch(
'freqtrade.optimize.hyperopt.get_timeframe',
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
2018-06-24 12:27:53 +00:00
parallel = mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel',
MagicMock(return_value=[{'loss': 1, 'result': 'foo result', 'params': {}}])
)
2018-06-17 20:32:56 +00:00
patch_exchange(mocker)
default_conf.update({'config': 'config.json.example',
'epochs': 1,
'timerange': None,
'spaces': 'all',
'hyperopt_jobs': 1, })
hyperopt = Hyperopt(default_conf)
hyperopt.strategy.tickerdata_to_dataframe = MagicMock()
hyperopt.start()
2018-06-24 12:27:53 +00:00
parallel.assert_called_once()
assert log_has('Best result:\nfoo result\nwith values:\n', caplog.record_tuples)
assert dumper.called
# Should be called twice, once for tickerdata, once to save evaluations
assert dumper.call_count == 2
def test_format_results(hyperopt):
# Test with BTC as stake_currency
trades = [
('ETH/BTC', 2, 2, 123),
('LTC/BTC', 1, 1, 123),
('XPR/BTC', -1, -2, -246)
]
labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration']
df = pd.DataFrame.from_records(trades, columns=labels)
result = hyperopt.format_results(df)
assert result.find(' 66.67%')
assert result.find('Total profit 1.00000000 BTC')
assert result.find('2.0000Σ %')
# Test with EUR as stake_currency
trades = [
('ETH/EUR', 2, 2, 123),
('LTC/EUR', 1, 1, 123),
('XPR/EUR', -1, -2, -246)
]
df = pd.DataFrame.from_records(trades, columns=labels)
result = hyperopt.format_results(df)
assert result.find('Total profit 1.00000000 EUR')
def test_has_space(hyperopt):
hyperopt.config.update({'spaces': ['buy', 'roi']})
assert hyperopt.has_space('roi')
assert hyperopt.has_space('buy')
assert not hyperopt.has_space('stoploss')
hyperopt.config.update({'spaces': ['all']})
assert hyperopt.has_space('buy')
2018-03-05 08:35:42 +00:00
def test_populate_indicators(hyperopt) -> None:
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC",
fill_missing=True)}
dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
{'pair': 'UNITTEST/BTC'})
2018-03-05 08:35:42 +00:00
# Check if some indicators are generated. We will not test all of them
assert 'adx' in dataframe
2018-06-24 12:27:53 +00:00
assert 'mfi' in dataframe
assert 'rsi' in dataframe
2018-03-05 08:35:42 +00:00
def test_buy_strategy_generator(hyperopt) -> None:
tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m')
tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC",
fill_missing=True)}
dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist)
dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'],
{'pair': 'UNITTEST/BTC'})
2018-03-05 08:35:42 +00:00
populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator(
2018-03-05 08:35:42 +00:00
{
2018-06-24 12:27:53 +00:00
'adx-value': 20,
'fastd-value': 20,
'mfi-value': 20,
'rsi-value': 20,
'adx-enabled': True,
'fastd-enabled': True,
'mfi-enabled': True,
'rsi-enabled': True,
'trigger': 'bb_lower'
2018-03-05 08:35:42 +00:00
}
)
result = populate_buy_trend(dataframe, {'pair': 'UNITTEST/BTC'})
2018-03-05 08:35:42 +00:00
# Check if some indicators are generated. We will not test all of them
2018-03-06 06:02:03 +00:00
assert 'buy' in result
assert 1 in result['buy']
def test_generate_optimizer(mocker, default_conf) -> None:
default_conf.update({'config': 'config.json.example'})
default_conf.update({'timerange': None})
default_conf.update({'spaces': 'all'})
2019-05-01 12:55:56 +00:00
default_conf.update({'hyperopt_min_trades': 1})
2018-03-06 06:02:03 +00:00
trades = [
('POWR/BTC', 0.023117, 0.000233, 100)
2018-03-06 06:02:03 +00:00
]
labels = ['currency', 'profit_percent', 'profit_abs', 'trade_duration']
2018-03-06 06:02:03 +00:00
backtest_result = pd.DataFrame.from_records(trades, columns=labels)
mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.backtest',
MagicMock(return_value=backtest_result)
)
2018-10-16 18:32:33 +00:00
mocker.patch(
2018-11-04 12:43:09 +00:00
'freqtrade.optimize.hyperopt.get_timeframe',
2018-10-16 18:32:33 +00:00
MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13)))
)
2018-06-17 20:32:56 +00:00
patch_exchange(mocker)
mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock())
2018-03-06 06:02:03 +00:00
optimizer_param = {
2018-06-24 12:27:53 +00:00
'adx-value': 0,
'fastd-value': 35,
'mfi-value': 0,
'rsi-value': 0,
'adx-enabled': False,
'fastd-enabled': True,
'mfi-enabled': False,
'rsi-enabled': False,
'trigger': 'macd_cross_signal',
'sell-adx-value': 0,
'sell-fastd-value': 75,
'sell-mfi-value': 0,
'sell-rsi-value': 0,
'sell-adx-enabled': False,
'sell-fastd-enabled': True,
'sell-mfi-enabled': False,
'sell-rsi-enabled': False,
'sell-trigger': 'macd_cross_signal',
2018-03-06 06:02:03 +00:00
'roi_t1': 60.0,
'roi_t2': 30.0,
'roi_t3': 20.0,
2018-06-24 12:27:53 +00:00
'roi_p1': 0.01,
'roi_p2': 0.01,
'roi_p3': 0.1,
2018-03-06 06:02:03 +00:00
'stoploss': -0.4,
}
response_expected = {
'loss': 1.9840569076926293,
2019-05-12 18:19:20 +00:00
'result': ' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC '
'( 2.31Σ%). Avg duration 100.0 mins.',
2018-06-24 12:27:53 +00:00
'params': optimizer_param
2018-03-06 06:02:03 +00:00
}
hyperopt = Hyperopt(default_conf)
2018-06-24 12:27:53 +00:00
generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values()))
2018-03-06 06:02:03 +00:00
assert generate_optimizer_value == response_expected