Merge pull request #1306 from xmatthias/feat/functional_tests

Funcional tests / backtest stoploss alignment
This commit is contained in:
Matthias 2018-11-06 19:25:31 +01:00 committed by GitHub
commit e0489878d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 282 additions and 27 deletions

View File

@ -206,21 +206,37 @@ class Backtesting(object):
buy_signal = sell_row.buy buy_signal = sell_row.buy
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
sell_row.sell) sell_row.sell, low=sell_row.low, high=sell_row.high)
if sell.sell_flag: if sell.sell_flag:
trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60)
# Special handling if high or low hit STOP_LOSS or ROI
if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
# Set close_rate to stoploss
closerate = trade.stop_loss
elif sell.sell_type == (SellType.ROI):
# get entry in min_roi >= to trade duration
roi_entry = max(list(filter(lambda x: trade_dur >= x,
self.strategy.minimal_roi.keys())))
roi = self.strategy.minimal_roi[roi_entry]
# - (Expected abs profit + open_rate + open_fee) / (fee_close -1)
closerate = - (trade.open_rate * roi + trade.open_rate *
(1 + trade.fee_open)) / (trade.fee_close - 1)
else:
closerate = sell_row.open
return BacktestResult(pair=pair, return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_percent=trade.calc_profit_percent(rate=closerate),
profit_abs=trade.calc_profit(rate=sell_row.open), profit_abs=trade.calc_profit(rate=closerate),
open_time=buy_row.date, open_time=buy_row.date,
close_time=sell_row.date, close_time=sell_row.date,
trade_duration=int(( trade_duration=trade_dur,
sell_row.date - buy_row.date).total_seconds() // 60),
open_index=buy_row.Index, open_index=buy_row.Index,
close_index=sell_row.Index, close_index=sell_row.Index,
open_at_end=False, open_at_end=False,
open_rate=buy_row.open, open_rate=buy_row.open,
close_rate=sell_row.open, close_rate=closerate,
sell_reason=sell.sell_type sell_reason=sell.sell_type
) )
if partial_ticker: if partial_ticker:
@ -260,7 +276,7 @@ class Backtesting(object):
position_stacking: do we allow position stacking? (default: False) position_stacking: do we allow position stacking? (default: False)
:return: DataFrame :return: DataFrame
""" """
headers = ['date', 'buy', 'open', 'close', 'sell'] headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
processed = args['processed'] processed = args['processed']
max_open_trades = args.get('max_open_trades', 0) max_open_trades = args.get('max_open_trades', 0)
position_stacking = args.get('position_stacking', False) position_stacking = args.get('position_stacking', False)

View File

@ -272,10 +272,10 @@ class Trade(_DECL_BASE):
self, self,
fee: Optional[float] = None) -> float: fee: Optional[float] = None) -> float:
""" """
Calculate the open_rate in BTC Calculate the open_rate including fee.
:param fee: fee to use on the open rate (optional). :param fee: fee to use on the open rate (optional).
If rate is not set self.fee will be used If rate is not set self.fee will be used
:return: Price in BTC of the open trade :return: Price in of the open trade incl. Fees
""" """
buy_trade = (Decimal(self.amount) * Decimal(self.open_rate)) buy_trade = (Decimal(self.amount) * Decimal(self.open_rate))
@ -287,7 +287,7 @@ class Trade(_DECL_BASE):
rate: Optional[float] = None, rate: Optional[float] = None,
fee: Optional[float] = None) -> float: fee: Optional[float] = None) -> float:
""" """
Calculate the close_rate in BTC Calculate the close_rate including fee
:param fee: fee to use on the close rate (optional). :param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used If rate is not set self.fee will be used
:param rate: rate to compare with (optional). :param rate: rate to compare with (optional).
@ -307,12 +307,12 @@ class Trade(_DECL_BASE):
rate: Optional[float] = None, rate: Optional[float] = None,
fee: Optional[float] = None) -> float: fee: Optional[float] = None) -> float:
""" """
Calculate the profit in BTC between Close and Open trade Calculate the absolute profit in stake currency between Close and Open trade
:param fee: fee to use on the close rate (optional). :param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used If rate is not set self.fee will be used
:param rate: close rate to compare with (optional). :param rate: close rate to compare with (optional).
If rate is not set self.close_rate will be used If rate is not set self.close_rate will be used
:return: profit in BTC as float :return: profit in stake currency as float
""" """
open_trade_price = self.calc_open_trade_price() open_trade_price = self.calc_open_trade_price()
close_trade_price = self.calc_close_trade_price( close_trade_price = self.calc_close_trade_price(

View File

@ -203,18 +203,22 @@ class IStrategy(ABC):
return buy, sell return buy, sell
def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool,
sell: bool) -> SellCheckTuple: sell: bool, low: float = None, high: float = None) -> SellCheckTuple:
""" """
This function evaluate if on the condition required to trigger a sell has been reached This function evaluate if on the condition required to trigger a sell has been reached
if the threshold is reached and updates the trade record. if the threshold is reached and updates the trade record.
:return: True if trade should be sold, False otherwise :return: True if trade should be sold, False otherwise
""" """
current_profit = trade.calc_profit_percent(rate) # Set current rate to low for backtesting sell
stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, current_rate = low or rate
current_profit=current_profit) current_profit = trade.calc_profit_percent(current_rate)
stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade,
current_time=date, current_profit=current_profit)
if stoplossflag.sell_flag: if stoplossflag.sell_flag:
return stoplossflag return stoplossflag
# Set current rate to low for backtesting sell
current_rate = high or rate
current_profit = trade.calc_profit_percent(current_rate)
experimental = self.config.get('experimental', {}) experimental = self.config.get('experimental', {})
if buy and experimental.get('ignore_roi_if_buy_signal', False): if buy and experimental.get('ignore_roi_if_buy_signal', False):

View File

@ -0,0 +1,45 @@
from typing import NamedTuple, List
import arrow
from pandas import DataFrame
from freqtrade.strategy.interface import SellType
ticker_start_time = arrow.get(2018, 10, 3)
ticker_interval_in_minute = 60
class BTrade(NamedTuple):
"""
Minimalistic Trade result used for functional backtesting
"""
sell_reason: SellType
open_tick: int
close_tick: int
class BTContainer(NamedTuple):
"""
Minimal BacktestContainer defining Backtest inputs and results.
"""
data: List[float]
stop_loss: float
roi: float
trades: List[BTrade]
profit_perc: float
def _get_frame_time_from_offset(offset):
return ticker_start_time.shift(
minutes=(offset * ticker_interval_in_minute)).datetime
def _build_backtest_dataframe(ticker_with_signals):
columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell']
frame = DataFrame.from_records(ticker_with_signals, columns=columns)
frame['date'] = frame['date'].apply(_get_frame_time_from_offset)
# Ensure floats are in place
for column in ['open', 'high', 'low', 'close', 'volume']:
frame[column] = frame[column].astype('float64')
return frame

View File

@ -0,0 +1,188 @@
# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
import logging
from unittest.mock import MagicMock
from pandas import DataFrame
import pytest
from freqtrade.optimize.backtesting import Backtesting
from freqtrade.strategy.interface import SellType
from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe,
_get_frame_time_from_offset)
from freqtrade.tests.conftest import patch_exchange
# Test 0 Minus 8% Close
# Test with Stop-loss at 1%
# TC1: Stop-Loss Triggered 1% loss
tc0 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4975, 4987, 6172, 1, 0],
[1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle)
[2, 4987, 5012, 4600, 4600, 6172, 0, 0], # exit with stoploss hit
[3, 4975, 5000, 4980, 4977, 6172, 0, 0],
[4, 4977, 4987, 4977, 4995, 6172, 0, 0],
[5, 4995, 4995, 4995, 4950, 6172, 0, 0]],
stop_loss=-0.01, roi=1, profit_perc=-0.01,
trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
)
# Test 1 Minus 4% Low, minus 1% close
# Test with Stop-Loss at 3%
# TC2: Stop-Loss Triggered 3% Loss
tc1 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4975, 4987, 6172, 1, 0],
[1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle)
[2, 4987, 5012, 4962, 4975, 6172, 0, 0],
[3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit
[4, 4962, 4987, 4937, 4950, 6172, 0, 0],
[5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
stop_loss=-0.03, roi=1, profit_perc=-0.03,
trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)]
)
# Test 3 Candle drops 4%, Recovers 1%.
# Entry Criteria Met
# Candle drops 20%
# Candle Data for test 3
# Test with Stop-Loss at 2%
# TC3: Trade-A: Stop-Loss Triggered 2% Loss
# Trade-B: Stop-Loss Triggered 2% Loss
tc2 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4975, 4987, 6172, 1, 0],
[1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle)
[2, 4987, 5012, 4800, 4975, 6172, 0, 0], # exit with stoploss hit
[3, 4975, 5000, 4950, 4962, 6172, 1, 0],
[4, 4975, 5000, 4950, 4962, 6172, 0, 0], # enter trade 2 (signal on last candle)
[5, 4962, 4987, 4000, 4000, 6172, 0, 0], # exit with stoploss hit
[6, 4950, 4975, 4975, 4950, 6172, 0, 0]],
stop_loss=-0.02, roi=1, profit_perc=-0.04,
trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2),
BTrade(sell_reason=SellType.STOP_LOSS, open_tick=4, close_tick=5)]
)
# Test 4 Minus 3% / recovery +15%
# Candle Data for test 3 Candle drops 3% Closed 15% up
# Test with Stop-loss at 2% ROI 6%
# TC4: Stop-Loss Triggered 2% Loss
tc3 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4975, 4987, 6172, 1, 0],
[1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle)
[2, 4987, 5750, 4850, 5750, 6172, 0, 0], # Exit with stoploss hit
[3, 4975, 5000, 4950, 4962, 6172, 0, 0],
[4, 4962, 4987, 4937, 4950, 6172, 0, 0],
[5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
stop_loss=-0.02, roi=0.06, profit_perc=-0.02,
trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
)
# Test 4 / Drops 0.5% Closes +20%
# Set stop-loss at 1% ROI 3%
# TC5: ROI triggers 3% Gain
tc4 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4980, 4987, 6172, 1, 0],
[1, 5000, 5025, 4980, 4987, 6172, 0, 0], # enter trade (signal on last candle)
[2, 4987, 5025, 4975, 4987, 6172, 0, 0],
[3, 4975, 6000, 4975, 6000, 6172, 0, 0], # ROI
[4, 4962, 4987, 4972, 4950, 6172, 0, 0],
[5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
stop_loss=-0.01, roi=0.03, profit_perc=0.03,
trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)]
)
# Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve
# Candle Data for test 6
# Set stop-loss at 2% ROI at 5%
# TC6: Stop-Loss triggers 2% Loss
tc5 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4975, 4987, 6172, 1, 0],
[1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle)
[2, 4987, 5300, 4850, 5050, 6172, 0, 0], # Exit with stoploss
[3, 4975, 5000, 4950, 4962, 6172, 0, 0],
[4, 4962, 4987, 4972, 4950, 6172, 0, 0],
[5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
stop_loss=-0.02, roi=0.05, profit_perc=-0.02,
trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
)
# Test 7 - 6% Positive / 1% Negative / Close 1% Positve
# Candle Data for test 7
# Set stop-loss at 2% ROI at 3%
# TC7: ROI Triggers 3% Gain
tc6 = BTContainer(data=[
# D O H L C V B S
[0, 5000, 5025, 4975, 4987, 6172, 1, 0],
[1, 5000, 5025, 4975, 4987, 6172, 0, 0],
[2, 4987, 5300, 4950, 5050, 6172, 0, 0],
[3, 4975, 5000, 4950, 4962, 6172, 0, 0],
[4, 4962, 4987, 4972, 4950, 6172, 0, 0],
[5, 4950, 4975, 4925, 4950, 6172, 0, 0]],
stop_loss=-0.02, roi=0.03, profit_perc=0.03,
trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)]
)
TESTS = [
tc0,
tc1,
tc2,
tc3,
tc4,
tc5,
tc6,
]
@pytest.mark.parametrize("data", TESTS)
def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
"""
run functional tests
"""
default_conf["stoploss"] = data.stop_loss
default_conf["minimal_roi"] = {"0": data.roi}
mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0))
patch_exchange(mocker)
frame = _build_backtest_dataframe(data.data)
backtesting = Backtesting(default_conf)
backtesting.advise_buy = lambda a, m: frame
backtesting.advise_sell = lambda a, m: frame
caplog.set_level(logging.DEBUG)
pair = 'UNITTEST/BTC'
# Dummy data as we mock the analyze functions
data_processed = {pair: DataFrame()}
results = backtesting.backtest(
{
'stake_amount': default_conf['stake_amount'],
'processed': data_processed,
'max_open_trades': 10,
}
)
print(results.T)
assert len(results) == len(data.trades)
assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3)
# if data.sell_r == SellType.STOP_LOSS:
# assert log_has("Stop loss hit.", caplog.record_tuples)
# else:
# assert not log_has("Stop loss hit.", caplog.record_tuples)
# log_test = (f'Force_selling still open trade UNITTEST/BTC with '
# f'{results.iloc[-1].profit_percent} perc - {results.iloc[-1].profit_abs}')
# if data.sell_r == SellType.FORCE_SELL:
# assert log_has(log_test,
# caplog.record_tuples)
# else:
# assert not log_has(log_test,
# caplog.record_tuples)
for c, trade in enumerate(data.trades):
res = results.iloc[c]
assert res.sell_reason == trade.sell_reason
assert res.open_time == _get_frame_time_from_offset(trade.open_tick)
assert res.close_time == _get_frame_time_from_offset(trade.close_tick)

View File

@ -518,18 +518,18 @@ def test_backtest(default_conf, fee, mocker) -> None:
expected = pd.DataFrame( expected = pd.DataFrame(
{'pair': [pair, pair], {'pair': [pair, pair],
'profit_percent': [0.00029977, 0.00056716], 'profit_percent': [0.0, 0.0],
'profit_abs': [1.49e-06, 7.6e-07], 'profit_abs': [0.0, 0.0],
'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime, 'open_time': [Arrow(2018, 1, 29, 18, 40, 0).datetime,
Arrow(2018, 1, 30, 3, 30, 0).datetime], Arrow(2018, 1, 30, 3, 30, 0).datetime],
'close_time': [Arrow(2018, 1, 29, 22, 40, 0).datetime, 'close_time': [Arrow(2018, 1, 29, 22, 35, 0).datetime,
Arrow(2018, 1, 30, 4, 20, 0).datetime], Arrow(2018, 1, 30, 4, 15, 0).datetime],
'open_index': [77, 183], 'open_index': [77, 183],
'close_index': [125, 193], 'close_index': [124, 192],
'trade_duration': [240, 50], 'trade_duration': [235, 45],
'open_at_end': [False, False], 'open_at_end': [False, False],
'open_rate': [0.104445, 0.10302485], 'open_rate': [0.104445, 0.10302485],
'close_rate': [0.105, 0.10359999], 'close_rate': [0.104969, 0.103541],
'sell_reason': [SellType.ROI, SellType.ROI] 'sell_reason': [SellType.ROI, SellType.ROI]
}) })
pd.testing.assert_frame_equal(results, expected) pd.testing.assert_frame_equal(results, expected)
@ -539,9 +539,11 @@ def test_backtest(default_conf, fee, mocker) -> None:
# Check open trade rate alignes to open rate # Check open trade rate alignes to open rate
assert ln is not None assert ln is not None
assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6)
# check close trade rate alignes to close rate # check close trade rate alignes to close rate or is between high and low
ln = data_pair.loc[data_pair["date"] == t["close_time"]] ln = data_pair.loc[data_pair["date"] == t["close_time"]]
assert round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or
round(ln.iloc[0]["low"], 6) < round(
t["close_rate"], 6) < round(ln.iloc[0]["high"], 6))
def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None: def test_backtest_1min_ticker_interval(default_conf, fee, mocker) -> None:
@ -580,7 +582,7 @@ def test_processed(default_conf, mocker) -> None:
def test_backtest_pricecontours(default_conf, fee, mocker) -> None: def test_backtest_pricecontours(default_conf, fee, mocker) -> None:
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
tests = [['raise', 18], ['lower', 0], ['sine', 16]] tests = [['raise', 18], ['lower', 0], ['sine', 19]]
for [contour, numres] in tests: for [contour, numres] in tests:
simple_backtest(default_conf, contour, numres, mocker) simple_backtest(default_conf, contour, numres, mocker)