adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Complete Backtesting and Hyperopt unit tests

Browse Source
This commit is contained in:
Gerald Lonlas
2018-03-02 21:46:32 +08:00
parent f4ec073099
commit 6ef7b7d93d
10 changed files with 851 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
freqtrade/tests/optimize/test_backtesting.py
View File

@@ -5,6 +5,7 @@ import math
from typing import List
from copy import deepcopy
from unittest.mock import MagicMock
from arrow import Arrow
import pandas as pd
from freqtrade import optimize
from freqtrade.optimize.backtesting import Backtesting, start, setup_configuration
@@ -255,6 +256,25 @@ def test_backtesting_init(default_conf) -> None:
assert callable(backtesting.populate_sell_trend)
def test_tickerdata_to_dataframe(default_conf) -> None:
"""
Test Backtesting.tickerdata_to_dataframe() method
"""
timerange = ((None, 'line'), None, -100)
tick = optimize.load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange)
tickerlist = {'BTC_UNITEST': tick}
backtesting = _BACKTESTING
data = backtesting.tickerdata_to_dataframe(tickerlist)
assert len(data['BTC_UNITEST']) == 100
# Load Analyze to compare the result between Backtesting function and Analyze are the same
analyze = Analyze(default_conf)
data2 = analyze.tickerdata_to_dataframe(tickerlist)
assert data['BTC_UNITEST'].equals(data2['BTC_UNITEST'])
def test_get_timeframe() -> None:
"""
Test Backtesting.get_timeframe() method
@@ -308,8 +328,18 @@ def test_backtesting_start(default_conf, mocker, caplog) -> None:
"""
Test Backtesting.start() method
"""
mocker.patch.multiple('freqtrade.optimize', load_data=mocked_load_data)
mocker.patch('freqtrade.exchange.get_ticker_history', MagicMock)
def get_timeframe(input1, input2):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
mocker.patch('freqtrade.freqtradebot.Analyze', MagicMock())
mocker.patch('freqtrade.optimize.load_data', mocked_load_data)
mocker.patch('freqtrade.exchange.get_ticker_history')
mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting',
backtest=MagicMock(),
_generate_text_table=MagicMock(return_value='1'),
get_timeframe=get_timeframe,
)
conf = deepcopy(default_conf)
conf['exchange']['pair_whitelist'] = ['BTC_UNITEST']

373
freqtrade/tests/optimize/test_hyperopt.py
View File

@@ -1,117 +1,108 @@
# pragma pylint: disable=missing-docstring,W0212,C0103
import logging
import os
import pytest
from copy import deepcopy
from freqtrade.optimize.hyperopt import calculate_loss, TARGET_TRADES, EXPECTED_MAX_PROFIT, start, \
log_results, save_trials, read_trials, generate_roi_table
#from freqtrade.optimize.hyperopt import EXPECTED_MAX_PROFIT, start, \
# log_results, save_trials, read_trials, generate_roi_table
from unittest.mock import MagicMock
from freqtrade.optimize.hyperopt import Hyperopt, start
import freqtrade.tests.conftest as tt # test tools
def test_loss_calculation_prefer_correct_trade_count():
correct = calculate_loss(1, TARGET_TRADES, 20)
over = calculate_loss(1, TARGET_TRADES + 100, 20)
under = calculate_loss(1, TARGET_TRADES - 100, 20)
assert over > correct
assert under > correct
# Avoid to reinit the same object again and again
_HYPEROPT = Hyperopt(tt.default_conf())
def test_loss_calculation_prefer_shorter_trades():
shorter = calculate_loss(1, 100, 20)
longer = calculate_loss(1, 100, 30)
assert shorter < longer
def test_loss_calculation_has_limited_profit():
correct = calculate_loss(EXPECTED_MAX_PROFIT, TARGET_TRADES, 20)
over = calculate_loss(EXPECTED_MAX_PROFIT * 2, TARGET_TRADES, 20)
under = calculate_loss(EXPECTED_MAX_PROFIT / 2, TARGET_TRADES, 20)
assert over == correct
assert under > correct
def create_trials(mocker):
# Functions for recurrent object patching
def create_trials(mocker) -> None:
"""
When creating trials, mock the hyperopt Trials so that *by default*
- we don't create any pickle'd files in the filesystem
- we might have a pickle'd file so make sure that we return
false when looking for it
"""
mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE',
return_value='freqtrade/tests/optimize/ut_trials.pickle')
mocker.patch('freqtrade.optimize.hyperopt.os.path.exists',
return_value=False)
mocker.patch('freqtrade.optimize.hyperopt.save_trials',
return_value=None)
mocker.patch('freqtrade.optimize.hyperopt.read_trials',
return_value=None)
mocker.patch('freqtrade.optimize.hyperopt.os.remove',
return_value=True)
_HYPEROPT.trials_file = os.path.join('freqtrade', 'tests', 'optimize','ut_trials.pickle')
mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=False)
mocker.patch('freqtrade.optimize.hyperopt.os.remove', return_value=True)
mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None)
return mocker.Mock(
results=[{
'loss': 1,
'result': 'foo',
'status': 'ok'
}],
results=[
{
'loss': 1,
'result': 'foo',
'status': 'ok'
}
],
best_trial={'misc': {'vals': {'adx': 999}}}
)
def test_start_calls_fmin(mocker):
trials = create_trials(mocker)
mocker.patch('freqtrade.optimize.tickerdata_to_dataframe')
mocker.patch('freqtrade.optimize.hyperopt.TRIALS', return_value=trials)
mocker.patch('freqtrade.optimize.hyperopt.sorted',
return_value=trials.results)
mocker.patch('freqtrade.optimize.preprocess')
mocker.patch('freqtrade.optimize.load_data')
mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
# Unit tests
def test_loss_calculation_prefer_correct_trade_count() -> None:
"""
Test Hyperopt.calculate_loss()
"""
hyperopt = _HYPEROPT
args = mocker.Mock(epochs=1, config='config.json.example', mongodb=False,
timerange=None)
start(args)
mock_fmin.assert_called_once()
correct = hyperopt.calculate_loss(1, hyperopt.target_trades, 20)
over = hyperopt.calculate_loss(1, hyperopt.target_trades + 100, 20)
under = hyperopt.calculate_loss(1, hyperopt.target_trades - 100, 20)
assert over > correct
assert under > correct
def test_start_uses_mongotrials(mocker):
mock_mongotrials = mocker.patch('freqtrade.optimize.hyperopt.MongoTrials',
return_value=create_trials(mocker))
mocker.patch('freqtrade.optimize.tickerdata_to_dataframe')
mocker.patch('freqtrade.optimize.load_data')
mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
def test_loss_calculation_prefer_shorter_trades() -> None:
"""
Test Hyperopt.calculate_loss()
"""
hyperopt = _HYPEROPT
args = mocker.Mock(epochs=1, config='config.json.example', mongodb=True,
timerange=None)
start(args)
mock_mongotrials.assert_called_once()
shorter = hyperopt.calculate_loss(1, 100, 20)
longer = hyperopt.calculate_loss(1, 100, 30)
assert shorter < longer
def test_log_results_if_loss_improves(mocker):
logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info')
global CURRENT_BEST_LOSS
CURRENT_BEST_LOSS = 2
log_results({
'loss': 1,
'current_tries': 1,
'total_tries': 2,
'result': 'foo'
})
def test_loss_calculation_has_limited_profit() -> None:
hyperopt = _HYPEROPT
logger.assert_called_once()
correct = hyperopt.calculate_loss(hyperopt.expected_max_profit, hyperopt.target_trades, 20)
over = hyperopt.calculate_loss(hyperopt.expected_max_profit * 2, hyperopt.target_trades, 20)
under = hyperopt.calculate_loss(hyperopt.expected_max_profit / 2, hyperopt.target_trades, 20)
assert over == correct
assert under > correct
def test_no_log_if_loss_does_not_improve(mocker):
logger = mocker.patch('freqtrade.optimize.hyperopt.logger.info')
global CURRENT_BEST_LOSS
CURRENT_BEST_LOSS = 2
log_results({
'loss': 3,
})
assert not logger.called
def test_log_results_if_loss_improves(caplog) -> None:
hyperopt = _HYPEROPT
hyperopt.current_best_loss = 2
hyperopt.log_results(
{
'loss': 1,
'current_tries': 1,
'total_tries': 2,
'result': 'foo'
}
)
assert tt.log_has(' 1/2: foo. Loss 1.00000', caplog.record_tuples)
def test_fmin_best_results(mocker, caplog):
caplog.set_level(logging.INFO)
def test_no_log_if_loss_does_not_improve(caplog) -> None:
hyperopt = _HYPEROPT
hyperopt.current_best_loss = 2
hyperopt.log_results(
{
'loss': 3,
}
)
assert caplog.record_tuples == []
def test_fmin_best_results(mocker, default_conf, caplog) -> None:
fmin_result = {
"macd_below_zero": 0,
"adx": 1,
@@ -136,38 +127,65 @@ def test_fmin_best_results(mocker, caplog):
"roi_p3": 3,
}
mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker))
mocker.patch('freqtrade.optimize.tickerdata_to_dataframe')
mocker.patch('freqtrade.optimize.load_data')
mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result)
conf = deepcopy(default_conf)
conf.update({'config': 'config.json.example'})
conf.update({'epochs': 1})
conf.update({'timerange': None})
args = mocker.Mock(epochs=1, config='config.json.example',
timerange=None)
start(args)
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value=fmin_result)
mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf)
mocker.patch('freqtrade.logger.Logger.set_format', MagicMock())
hyperopt = Hyperopt(conf)
hyperopt.trials = create_trials(mocker)
hyperopt.tickerdata_to_dataframe = MagicMock()
hyperopt.start()
exists = [
'Best parameters',
'Best parameters:',
'"adx": {\n "enabled": true,\n "value": 15.0\n },',
'"fastd": {\n "enabled": true,\n "value": 40.0\n },',
'"green_candle": {\n "enabled": true\n },',
'"macd_below_zero": {\n "enabled": false\n },',
'"mfi": {\n "enabled": false\n },',
'"over_sar": {\n "enabled": false\n },',
'"roi_p1": 1.0,',
'"roi_p2": 2.0,',
'"roi_p3": 3.0,',
'"roi_t1": 1.0,',
'"roi_t2": 2.0,',
'"roi_t3": 3.0,',
'"rsi": {\n "enabled": true,\n "value": 37.0\n },',
'"stoploss": -0.1,',
'"trigger": {\n "type": "faststoch10"\n },',
'"stoploss": -0.1',
'"uptrend_long_ema": {\n "enabled": true\n },',
'"uptrend_short_ema": {\n "enabled": false\n },',
'"uptrend_sma": {\n "enabled": false\n }',
'ROI table:\n{\'0\': 6.0, \'3.0\': 3.0, \'5.0\': 1.0, \'6.0\': 0}',
'Best Result:\nfoo'
]
for line in exists:
assert line in caplog.text
def test_fmin_throw_value_error(mocker, caplog):
caplog.set_level(logging.INFO)
mocker.patch('freqtrade.optimize.hyperopt.MongoTrials', return_value=create_trials(mocker))
mocker.patch('freqtrade.optimize.tickerdata_to_dataframe')
mocker.patch('freqtrade.optimize.load_data')
def test_fmin_throw_value_error(mocker, default_conf, caplog) -> None:
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
mocker.patch('freqtrade.optimize.hyperopt.fmin', side_effect=ValueError())
args = mocker.Mock(epochs=1, config='config.json.example',
timerange=None)
start(args)
conf = deepcopy(default_conf)
conf.update({'config': 'config.json.example'})
conf.update({'epochs': 1})
conf.update({'timerange': None})
mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf)
mocker.patch('freqtrade.logger.Logger.set_format', MagicMock())
hyperopt = Hyperopt(conf)
hyperopt.trials = create_trials(mocker)
hyperopt.tickerdata_to_dataframe = MagicMock()
hyperopt.start()
exists = [
'Best Result:',
@@ -179,68 +197,80 @@ def test_fmin_throw_value_error(mocker, caplog):
assert line in caplog.text
def test_resuming_previous_hyperopt_results_succeeds(mocker):
import freqtrade.optimize.hyperopt as hyperopt
def test_resuming_previous_hyperopt_results_succeeds(mocker, default_conf) -> None:
trials = create_trials(mocker)
mocker.patch('freqtrade.optimize.hyperopt.TRIALS',
return_value=trials)
mocker.patch('freqtrade.optimize.hyperopt.os.path.exists',
return_value=True)
mocker.patch('freqtrade.optimize.hyperopt.len',
return_value=len(trials.results))
mock_read = mocker.patch('freqtrade.optimize.hyperopt.read_trials',
return_value=trials)
mock_save = mocker.patch('freqtrade.optimize.hyperopt.save_trials',
return_value=None)
mocker.patch('freqtrade.optimize.hyperopt.sorted',
return_value=trials.results)
mocker.patch('freqtrade.optimize.preprocess')
mocker.patch('freqtrade.optimize.load_data')
mocker.patch('freqtrade.optimize.hyperopt.fmin',
return_value={})
args = mocker.Mock(epochs=1,
config='config.json.example',
mongodb=False,
timerange=None)
start(args)
conf = deepcopy(default_conf)
conf.update({'config': 'config.json.example'})
conf.update({'epochs': 1})
conf.update({'mongodb': False})
conf.update({'timerange': None})
mocker.patch('freqtrade.optimize.hyperopt.os.path.exists', return_value=True)
mocker.patch('freqtrade.optimize.hyperopt.len', return_value=len(trials.results))
mock_read = mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.read_trials',
return_value=trials
)
mock_save = mocker.patch(
'freqtrade.optimize.hyperopt.Hyperopt.save_trials',
return_value=None
)
mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results)
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf)
mocker.patch('freqtrade.logger.Logger.set_format', MagicMock())
hyperopt = Hyperopt(conf)
hyperopt.trials = trials
hyperopt.tickerdata_to_dataframe = MagicMock()
hyperopt.start()
mock_read.assert_called_once()
mock_save.assert_called_once()
current_tries = hyperopt._CURRENT_TRIES
total_tries = hyperopt.TOTAL_TRIES
current_tries = hyperopt.current_tries
total_tries = hyperopt.total_tries
assert current_tries == len(trials.results)
assert total_tries == (current_tries + len(trials.results))
def test_save_trials_saves_trials(mocker):
def test_save_trials_saves_trials(mocker, caplog) -> None:
create_trials(mocker)
mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump', return_value=None)
hyperopt = _HYPEROPT
mocker.patch('freqtrade.optimize.hyperopt.open', return_value=hyperopt.trials_file)
hyperopt.save_trials()
assert tt.log_has(
'Saving Trials to \'freqtrade/tests/optimize/ut_trials.pickle\'',
caplog.record_tuples
)
mock_dump.assert_called_once()
def test_read_trials_returns_trials_file(mocker, default_conf, caplog) -> None:
trials = create_trials(mocker)
mock_dump = mocker.patch('freqtrade.optimize.hyperopt.pickle.dump',
return_value=None)
trials_path = mocker.patch('freqtrade.optimize.hyperopt.TRIALS_FILE',
return_value='ut_trials.pickle')
mocker.patch('freqtrade.optimize.hyperopt.open',
return_value=trials_path)
save_trials(trials, trials_path)
mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load', return_value=trials)
mock_open = mocker.patch('freqtrade.optimize.hyperopt.open', return_value=mock_load)
mock_dump.assert_called_once_with(trials, trials_path)
def test_read_trials_returns_trials_file(mocker):
trials = create_trials(mocker)
mock_load = mocker.patch('freqtrade.optimize.hyperopt.pickle.load',
return_value=trials)
mock_open = mocker.patch('freqtrade.optimize.hyperopt.open',
return_value=mock_load)
assert read_trials() == trials
hyperopt = _HYPEROPT
hyperopt_trial = hyperopt.read_trials()
assert tt.log_has(
'Reading Trials from \'freqtrade/tests/optimize/ut_trials.pickle\'',
caplog.record_tuples
)
assert hyperopt_trial == trials
mock_open.assert_called_once()
mock_load.assert_called_once()
def test_roi_table_generation():
def test_roi_table_generation() -> None:
params = {
'roi_t1': 5,
'roi_t2': 10,
@@ -249,4 +279,49 @@ def test_roi_table_generation():
'roi_p2': 2,
'roi_p3': 3,
}
assert generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0}
hyperopt = _HYPEROPT
assert hyperopt.generate_roi_table(params) == {'0': 6, '15': 3, '25': 1, '30': 0}
def test_start_calls_fmin(mocker, default_conf) -> None:
trials = create_trials(mocker)
mocker.patch('freqtrade.optimize.hyperopt.sorted', return_value=trials.results)
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
conf = deepcopy(default_conf)
conf.update({'config': 'config.json.example'})
conf.update({'epochs': 1})
conf.update({'mongodb': False})
conf.update({'timerange': None})
hyperopt = Hyperopt(conf)
hyperopt.trials = trials
hyperopt.tickerdata_to_dataframe = MagicMock()
hyperopt.start()
mock_fmin.assert_called_once()
def test_start_uses_mongotrials(mocker, default_conf) -> None:
mocker.patch('freqtrade.optimize.hyperopt.load_data', MagicMock())
mock_fmin = mocker.patch('freqtrade.optimize.hyperopt.fmin', return_value={})
mock_mongotrials = mocker.patch(
'freqtrade.optimize.hyperopt.MongoTrials',
return_value=create_trials(mocker)
)
conf = deepcopy(default_conf)
conf.update({'config': 'config.json.example'})
conf.update({'epochs': 1})
conf.update({'mongodb': True})
conf.update({'timerange': None})
mocker.patch('freqtrade.optimize.hyperopt.hyperopt_optimize_conf', return_value=conf)
hyperopt = Hyperopt(conf)
hyperopt.tickerdata_to_dataframe = MagicMock()
hyperopt.start()
mock_mongotrials.assert_called_once()
mock_fmin.assert_called_once()

11
freqtrade/tests/optimize/test_optimize.py
View File

@@ -6,7 +6,6 @@ import logging
import uuid
from shutil import copyfile
from freqtrade import optimize
from freqtrade.analyze import Analyze
from freqtrade.optimize.__init__ import make_testdata_path, download_pairs,\
download_backtesting_testdata, load_tickerdata_file, trim_tickerlist
from freqtrade.misc import file_dump_json
@@ -220,16 +219,6 @@ def test_init(default_conf, mocker) -> None:
)
def test_tickerdata_to_dataframe(default_conf) -> None:
analyze = Analyze(default_conf)
timerange = ((None, 'line'), None, -100)
tick = load_tickerdata_file(None, 'BTC_UNITEST', 1, timerange=timerange)
tickerlist = {'BTC_UNITEST': tick}
data = analyze.tickerdata_to_dataframe(tickerlist)
assert len(data['BTC_UNITEST']) == 100
def test_trim_tickerlist() -> None:
with open('freqtrade/tests/testdata/BTC_ETH-1.json') as data_file:
ticker_list = json.load(data_file)