Merge pull request #1673 from freqtrade/refactor/persistance_stoplossupdate

trailing stop backtest problems

freqtrade/persistence.py

@@ -83,7 +83,7 @@ def check_migrate(engine) -> None:
         logger.debug(f'trying {table_back_name}')
 
     # Check for latest column
-    if not has_column(cols, 'stoploss_last_update'):
+    if not has_column(cols, 'min_rate'):
         logger.info(f'Running database migration - backup available as {table_back_name}')
 
         fee_open = get_column_def(cols, 'fee_open', 'fee')
@@ -95,6 +95,7 @@ def check_migrate(engine) -> None:
         stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
         stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
         max_rate = get_column_def(cols, 'max_rate', '0.0')
+        min_rate = get_column_def(cols, 'min_rate', 'null')
         sell_reason = get_column_def(cols, 'sell_reason', 'null')
         strategy = get_column_def(cols, 'strategy', 'null')
         ticker_interval = get_column_def(cols, 'ticker_interval', 'null')
@@ -113,7 +114,7 @@ def check_migrate(engine) -> None:
                 open_rate_requested, close_rate, close_rate_requested, close_profit,
                 stake_amount, amount, open_date, close_date, open_order_id,
                 stop_loss, initial_stop_loss, stoploss_order_id, stoploss_last_update,
-                max_rate, sell_reason, strategy,
+                max_rate, min_rate, sell_reason, strategy,
                 ticker_interval
                 )
             select id, lower(exchange),
@@ -130,7 +131,7 @@ def check_migrate(engine) -> None:
                 stake_amount, amount, open_date, close_date, open_order_id,
                 {stop_loss} stop_loss, {initial_stop_loss} initial_stop_loss,
                 {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
-                {max_rate} max_rate, {sell_reason} sell_reason,
+                {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
                 {strategy} strategy, {ticker_interval} ticker_interval
                 from {table_back_name}
              """)
@@ -191,6 +192,8 @@ class Trade(_DECL_BASE):
     stoploss_last_update = Column(DateTime, nullable=True)
     # absolute value of the highest reached price
     max_rate = Column(Float, nullable=True, default=0.0)
+    # Lowest price reached
+    min_rate = Column(Float, nullable=True)
     sell_reason = Column(String, nullable=True)
     strategy = Column(String, nullable=True)
     ticker_interval = Column(Integer, nullable=True)
@@ -201,8 +204,22 @@ class Trade(_DECL_BASE):
         return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
                 f'open_rate={self.open_rate:.8f}, open_since={open_since})')
 
+    def adjust_min_max_rates(self, current_price: float):
+        """
+        Adjust the max_rate and min_rate.
+        """
+        logger.debug("Adjusting min/max rates")
+        self.max_rate = max(current_price, self.max_rate or self.open_rate)
+        self.min_rate = min(current_price, self.min_rate or self.open_rate)
+
     def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False):
-        """this adjusts the stop loss to it's most recently observed setting"""
+        """
+        This adjusts the stop loss to it's most recently observed setting
+        :param current_price: Current rate the asset is traded
+        :param stoploss: Stoploss as factor (sample -0.05 -> -5% below current price).
+        :param initial: Called to initiate stop_loss.
+            Skips everything if self.stop_loss is already set.
+        """
 
         if initial and not (self.stop_loss is None or self.stop_loss == 0):
             # Don't modify if called with initial and nothing to do
@@ -210,13 +227,6 @@ class Trade(_DECL_BASE):
 
         new_loss = float(current_price * (1 - abs(stoploss)))
 
-        # keeping track of the highest observed rate for this trade
-        if self.max_rate is None:
-            self.max_rate = current_price
-        else:
-            if current_price > self.max_rate:
-                self.max_rate = current_price
-
         # no stop loss assigned yet
         if not self.stop_loss:
             logger.debug("assigning new stop loss")

freqtrade/strategy/interface.py

@@ -247,6 +247,9 @@ class IStrategy(ABC):
         """
         This function evaluate if on the condition required to trigger a sell has been reached
         if the threshold is reached and updates the trade record.
+        :param low: Only used during backtesting to simulate stoploss
+        :param high: Only used during backtesting, to simulate ROI
+        :param force_stoploss: Externally provided stoploss
         :return: True if trade should be sold, False otherwise
         """
 
@@ -254,14 +257,16 @@ class IStrategy(ABC):
         current_rate = low or rate
         current_profit = trade.calc_profit_percent(current_rate)
 
+        trade.adjust_min_max_rates(high or current_rate)
+
         stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade,
                                               current_time=date, current_profit=current_profit,
-                                              force_stoploss=force_stoploss)
+                                              force_stoploss=force_stoploss, high=high)
 
         if stoplossflag.sell_flag:
             return stoplossflag
 
-        # Set current rate to low for backtesting sell
+        # Set current rate to high for backtesting sell
         current_rate = high or rate
         current_profit = trade.calc_profit_percent(current_rate)
         experimental = self.config.get('experimental', {})
@@ -285,8 +290,9 @@ class IStrategy(ABC):
 
         return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
 
-    def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime,
+    def stop_loss_reached(self, current_rate: float, trade: Trade,
-                          current_profit: float, force_stoploss: float) -> SellCheckTuple:
+                          current_time: datetime, current_profit: float,
+                          force_stoploss: float, high: float = None) -> SellCheckTuple:
         """
         Based on current profit of the trade and configured (trailing) stoploss,
         decides to sell or not
@@ -294,13 +300,33 @@ class IStrategy(ABC):
         """
 
         trailing_stop = self.config.get('trailing_stop', False)
-        trade.adjust_stop_loss(trade.open_rate, force_stoploss if force_stoploss
+        stop_loss_value = force_stoploss if force_stoploss else self.stoploss
-                               else self.stoploss, initial=True)
+
+        # Initiate stoploss with open_rate. Does nothing if stoploss is already set.
+        trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True)
+
+        if trailing_stop:
+            # trailing stoploss handling
+
+            sl_offset = self.config.get('trailing_stop_positive_offset') or 0.0
+            tsl_only_offset = self.config.get('trailing_only_offset_is_reached', False)
+
+            # Don't update stoploss if trailing_only_offset_is_reached is true.
+            if not (tsl_only_offset and current_profit < sl_offset):
+                # Specific handling for trailing_stop_positive
+                if 'trailing_stop_positive' in self.config and current_profit > sl_offset:
+                    # Ignore mypy error check in configuration that this is a float
+                    stop_loss_value = self.config.get('trailing_stop_positive')  # type: ignore
+                    logger.debug(f"using positive stop loss: {stop_loss_value} "
+                                 f"offset: {sl_offset:.4g} profit: {current_profit:.4f}%")
+
+                trade.adjust_stop_loss(high or current_rate, stop_loss_value)
 
         # evaluate if the stoploss was hit if stoploss is not on exchange
         if ((self.stoploss is not None) and
             (trade.stop_loss >= current_rate) and
                 (not self.order_types.get('stoploss_on_exchange'))):
+
             selltype = SellType.STOP_LOSS
             # If Trailing stop (and max-rate did move above open rate)
             if trailing_stop and trade.open_rate != trade.max_rate:
@@ -315,29 +341,6 @@ class IStrategy(ABC):
             logger.debug('Stop loss hit.')
             return SellCheckTuple(sell_flag=True, sell_type=selltype)
 
-        # update the stop loss afterwards, after all by definition it's supposed to be hanging
-        if trailing_stop:
-
-            # check if we have a special stop loss for positive condition
-            # and if profit is positive
-            stop_loss_value = force_stoploss if force_stoploss else self.stoploss
-
-            sl_offset = self.config.get('trailing_stop_positive_offset') or 0.0
-
-            if 'trailing_stop_positive' in self.config and current_profit > sl_offset:
-
-                # Ignore mypy error check in configuration that this is a float
-                stop_loss_value = self.config.get('trailing_stop_positive')  # type: ignore
-                logger.debug(f"using positive stop loss mode: {stop_loss_value} "
-                             f"with offset {sl_offset:.4g} "
-                             f"since we have profit {current_profit:.4f}%")
-
-            # if trailing_only_offset_is_reached is true,
-            # we update trailing stoploss only if offset is reached.
-            tsl_only_offset = self.config.get('trailing_only_offset_is_reached', False)
-            if not (tsl_only_offset and current_profit < sl_offset):
-                trade.adjust_stop_loss(current_rate, stop_loss_value)
-
         return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
 
     def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool:

freqtrade/tests/optimize/__init__.py

@@ -28,6 +28,7 @@ class BTContainer(NamedTuple):
     roi: float
     trades: List[BTrade]
     profit_perc: float
+    trailing_stop: bool = False
 
 
 def _get_frame_time_from_offset(offset):

freqtrade/tests/optimize/test_backtest_detail.py

@@ -14,10 +14,10 @@ from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataf
 from freqtrade.tests.conftest import patch_exchange
 
 
-# Test 0 Minus 8% Close
+# Test 1 Minus 8% Close
 # Test with Stop-loss at 1%
 # TC1: Stop-Loss Triggered 1% loss
-tc0 = BTContainer(data=[
+tc1 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 0],  # enter trade (signal on last candle)
@@ -30,10 +30,10 @@ tc0 = BTContainer(data=[
 )
 
 
-# Test 1 Minus 4% Low, minus 1% close
+# Test 2 Minus 4% Low, minus 1% close
 # Test with Stop-Loss at 3%
 # TC2: Stop-Loss Triggered 3% Loss
-tc1 = BTContainer(data=[
+tc2 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 0],  # enter trade (signal on last candle)
@@ -49,11 +49,10 @@ tc1 = BTContainer(data=[
 # Test 3 Candle drops 4%, Recovers 1%.
 #               Entry Criteria Met
 # 	            Candle drops 20%
-# Candle Data for test 3
 # Test with Stop-Loss at 2%
 # TC3: Trade-A: Stop-Loss Triggered 2% Loss
 #          Trade-B: Stop-Loss Triggered 2% Loss
-tc2 = BTContainer(data=[
+tc3 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 0],  # enter trade (signal on last candle)
@@ -71,7 +70,7 @@ tc2 = BTContainer(data=[
 # Candle Data for test 3 – Candle drops 3% Closed 15% up
 # Test with Stop-loss at 2% ROI 6%
 # TC4: Stop-Loss Triggered 2% Loss
-tc3 = BTContainer(data=[
+tc4 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 0],  # enter trade (signal on last candle)
@@ -83,10 +82,10 @@ tc3 = BTContainer(data=[
     trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=2)]
 )
 
-# Test 4 / Drops 0.5% Closes +20%
+# Test 5 / Drops 0.5% Closes +20%
 # Set stop-loss at 1% ROI 3%
 # TC5: ROI triggers 3% Gain
-tc4 = BTContainer(data=[
+tc5 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4980, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4980, 4987, 6172, 0, 0],  # enter trade (signal on last candle)
@@ -99,10 +98,9 @@ tc4 = BTContainer(data=[
 )
 
 # Test 6 / Drops 3% / Recovers 6% Positive / Closes 1% positve
-# Candle Data for test 6
 # Set stop-loss at 2% ROI at 5%
 # TC6: Stop-Loss triggers 2% Loss
-tc5 = BTContainer(data=[
+tc6 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 0],  # enter trade (signal on last candle)
@@ -115,10 +113,9 @@ tc5 = BTContainer(data=[
 )
 
 # Test 7 - 6% Positive / 1% Negative / Close 1% Positve
-# Candle Data for test 7
 # Set stop-loss at 2% ROI at 3%
 # TC7: ROI Triggers 3% Gain
-tc6 = BTContainer(data=[
+tc7 = BTContainer(data=[
     # D  O     H     L     C     V    B  S
     [0, 5000, 5025, 4975, 4987, 6172, 1, 0],
     [1, 5000, 5025, 4975, 4987, 6172, 0, 0],
@@ -130,14 +127,47 @@ tc6 = BTContainer(data=[
     trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)]
 )
 
+
+# Test 8 - trailing_stop should raise so candle 3 causes a stoploss.
+# Set stop-loss at 10%, ROI at 10% (should not apply)
+# TC8: Trailing stoploss - stoploss should be adjusted candle 2
+tc8 = BTContainer(data=[
+    # D   O     H     L    C     V    B  S
+    [0, 5000, 5050, 4950, 5000, 6172, 1, 0],
+    [1, 5000, 5050, 4950, 5000, 6172, 0, 0],
+    [2, 5000, 5250, 4750, 4850, 6172, 0, 0],
+    [3, 4850, 5050, 4650, 4750, 6172, 0, 0],
+    [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
+    stop_loss=-0.10, roi=0.10, profit_perc=-0.055, trailing_stop=True,
+    trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)]
+)
+
+
+# Test 9 - trailing_stop should raise - high and low in same candle.
+# Candle Data for test 9
+# Set stop-loss at 10%, ROI at 10% (should not apply)
+# TC9: Trailing stoploss - stoploss should be adjusted candle 2
+tc9 = BTContainer(data=[
+    # D   O     H     L     C    V    B  S
+    [0, 5000, 5050, 4950, 5000, 6172, 1, 0],
+    [1, 5000, 5050, 4950, 5000, 6172, 0, 0],
+    [2, 5000, 5050, 4950, 5000, 6172, 0, 0],
+    [3, 5000, 5200, 4550, 4850, 6172, 0, 0],
+    [4, 4750, 4950, 4350, 4750, 6172, 0, 0]],
+    stop_loss=-0.10, roi=0.10, profit_perc=-0.064, trailing_stop=True,
+    trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)]
+)
+
 TESTS = [
-    tc0,
     tc1,
     tc2,
     tc3,
     tc4,
     tc5,
     tc6,
+    tc7,
+    tc8,
+    tc9,
 ]
 
 
@@ -148,8 +178,9 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     """
     default_conf["stoploss"] = data.stop_loss
     default_conf["minimal_roi"] = {"0": data.roi}
-    default_conf['ticker_interval'] = tests_ticker_interval
+    default_conf["ticker_interval"] = tests_ticker_interval
-    mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.0))
+    default_conf["trailing_stop"] = data.trailing_stop
+    mocker.patch("freqtrade.exchange.Exchange.get_fee", MagicMock(return_value=0.0))
     patch_exchange(mocker)
     frame = _build_backtest_dataframe(data.data)
     backtesting = Backtesting(default_conf)
@@ -157,7 +188,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None:
     backtesting.advise_sell = lambda a, m: frame
     caplog.set_level(logging.DEBUG)
 
-    pair = 'UNITTEST/BTC'
+    pair = "UNITTEST/BTC"
     # Dummy data as we mock the analyze functions
     data_processed = {pair: DataFrame()}
     min_date, max_date = get_timeframe({pair: frame})

freqtrade/tests/test_freqtradebot.py

@@ -2265,9 +2265,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market
     }
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
-    freqtrade.strategy.stop_loss_reached = \
+    freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple(
-        lambda current_rate, trade, current_time, force_stoploss, current_profit: SellCheckTuple(
+            sell_flag=False, sell_type=SellType.NONE))
-            sell_flag=False, sell_type=SellType.NONE)
     freqtrade.create_trade()
 
     trade = Trade.query.first()
@@ -2413,8 +2412,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, markets
                  }))
     # stop-loss not reached, adjusted stoploss
     assert freqtrade.handle_trade(trade) is False
-    assert log_has(f'using positive stop loss mode: 0.01 with offset 0 '
+    assert log_has(f'using positive stop loss: 0.01 offset: 0 profit: 0.2666%',
-                   f'since we have profit 0.2666%',
                    caplog.record_tuples)
     assert log_has(f'adjusted stop loss', caplog.record_tuples)
     assert trade.stop_loss == 0.0000138501
@@ -2473,8 +2471,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
                  }))
     # stop-loss not reached, adjusted stoploss
     assert freqtrade.handle_trade(trade) is False
-    assert log_has(f'using positive stop loss mode: 0.01 with offset 0.011 '
+    assert log_has(f'using positive stop loss: 0.01 offset: 0.011 profit: 0.2666%',
-                   f'since we have profit 0.2666%',
                    caplog.record_tuples)
     assert log_has(f'adjusted stop loss', caplog.record_tuples)
     assert trade.stop_loss == 0.0000138501
@@ -2553,8 +2550,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee,
                  }))
 
     assert freqtrade.handle_trade(trade) is False
-    assert log_has(f'using positive stop loss mode: 0.05 with offset 0.055 '
+    assert log_has(f'using positive stop loss: 0.05 offset: 0.055 profit: 0.1218%',
-                   f'since we have profit 0.1218%',
                    caplog.record_tuples)
     assert log_has(f'adjusted stop loss', caplog.record_tuples)
     assert trade.stop_loss == 0.0000117705

freqtrade/tests/test_persistence.py

@@ -510,6 +510,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.pair == "ETC/BTC"
     assert trade.exchange == "binance"
     assert trade.max_rate == 0.0
+    assert trade.min_rate is None
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
     assert trade.sell_reason is None
@@ -585,7 +586,48 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
                    caplog.record_tuples)
 
 
-def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee):
+def test_adjust_stop_loss(fee):
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        exchange='bittrex',
+        open_rate=1,
+        max_rate=1,
+    )
+
+    trade.adjust_stop_loss(trade.open_rate, 0.05, True)
+    assert trade.stop_loss == 0.95
+    assert trade.initial_stop_loss == 0.95
+
+    # Get percent of profit with a lower rate
+    trade.adjust_stop_loss(0.96, 0.05)
+    assert trade.stop_loss == 0.95
+    assert trade.initial_stop_loss == 0.95
+
+    # Get percent of profit with a custom rate (Higher than open rate)
+    trade.adjust_stop_loss(1.3, -0.1)
+    assert round(trade.stop_loss, 8) == 1.17
+    assert trade.initial_stop_loss == 0.95
+
+    # current rate lower again ... should not change
+    trade.adjust_stop_loss(1.2, 0.1)
+    assert round(trade.stop_loss, 8) == 1.17
+    assert trade.initial_stop_loss == 0.95
+
+    # current rate higher... should raise stoploss
+    trade.adjust_stop_loss(1.4, 0.1)
+    assert round(trade.stop_loss, 8) == 1.26
+    assert trade.initial_stop_loss == 0.95
+
+    #  Initial is true but stop_loss set - so doesn't do anything
+    trade.adjust_stop_loss(1.7, 0.1, True)
+    assert round(trade.stop_loss, 8) == 1.26
+    assert trade.initial_stop_loss == 0.95
+
+
+def test_adjust_min_max_rates(fee):
     trade = Trade(
         pair='ETH/BTC',
         stake_amount=0.001,
@@ -595,40 +637,24 @@ def test_adjust_stop_loss(limit_buy_order, limit_sell_order, fee):
         open_rate=1,
     )
 
-    trade.adjust_stop_loss(trade.open_rate, 0.05, True)
+    trade.adjust_min_max_rates(trade.open_rate)
-    assert trade.stop_loss == 0.95
     assert trade.max_rate == 1
-    assert trade.initial_stop_loss == 0.95
+    assert trade.min_rate == 1
 
-    # Get percent of profit with a lowre rate
+    # check min adjusted, max remained
-    trade.adjust_stop_loss(0.96, 0.05)
+    trade.adjust_min_max_rates(0.96)
-    assert trade.stop_loss == 0.95
     assert trade.max_rate == 1
-    assert trade.initial_stop_loss == 0.95
+    assert trade.min_rate == 0.96
 
-    # Get percent of profit with a custom rate (Higher than open rate)
+    # check max adjusted, min remains
-    trade.adjust_stop_loss(1.3, -0.1)
+    trade.adjust_min_max_rates(1.05)
-    assert round(trade.stop_loss, 8) == 1.17
+    assert trade.max_rate == 1.05
-    assert trade.max_rate == 1.3
+    assert trade.min_rate == 0.96
-    assert trade.initial_stop_loss == 0.95
 
-    # current rate lower again ... should not change
+    # current rate "in the middle" - no adjustment
-    trade.adjust_stop_loss(1.2, 0.1)
+    trade.adjust_min_max_rates(1.03)
-    assert round(trade.stop_loss, 8) == 1.17
+    assert trade.max_rate == 1.05
-    assert trade.max_rate == 1.3
+    assert trade.min_rate == 0.96
-    assert trade.initial_stop_loss == 0.95
-
-    # current rate higher... should raise stoploss
-    trade.adjust_stop_loss(1.4, 0.1)
-    assert round(trade.stop_loss, 8) == 1.26
-    assert trade.max_rate == 1.4
-    assert trade.initial_stop_loss == 0.95
-
-    #  Initial is true but stop_loss set - so doesn't do anything
-    trade.adjust_stop_loss(1.7, 0.1, True)
-    assert round(trade.stop_loss, 8) == 1.26
-    assert trade.max_rate == 1.4
-    assert trade.initial_stop_loss == 0.95
 
 
 def test_get_open(default_conf, fee):