Merge pull request #6857 from froggleston/develop

Add support for fudging unavailable funding rates, allowing backtesti…

docs/configuration.md

@@ -230,6 +230,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `dataformat_trades` | Data format to use to store historical trades data. <br> *Defaults to `jsongz`*. <br> **Datatype:** String
 | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position). <br> [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*<br> **Datatype:** Boolean
 | `max_entry_position_adjustment` | Maximum additional order(s) for each open trade on top of the first entry Order. Set it to `-1` for unlimited additional orders. [More information here](strategy-callbacks.md#adjust-trade-position). <br> [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `-1`.*<br> **Datatype:** Positive Integer or -1
+| `futures_funding_rate` | User-specified funding rate to be used when historical funding rates are not available from the exchange. This does not overwrite real historical rates. It is recommended that this be set to 0 unless you are testing a specific coin and you understand how the funding rate will affect freqtrade's profit calculations. [More information here](leverage.md#unavailable-funding-rates) <br>*Defaults to None.*<br> **Datatype:** Float
 
 ### Parameters in the strategy
 

docs/leverage.md

@@ -101,6 +101,13 @@ Possible values are any floats between 0.0 and 0.99
 !!! Danger "A `liquidation_buffer` of 0.0, or a low `liquidation_buffer` is likely to result in liquidations, and liquidation fees"
     Currently Freqtrade is able to calculate liquidation prices, but does not calculate liquidation fees. Setting your `liquidation_buffer` to 0.0, or using a low `liquidation_buffer` could result in your positions being liquidated. Freqtrade does not track liquidation fees, so liquidations will result in inaccurate profit/loss results for your bot. If you use a low `liquidation_buffer`, it is recommended to use `stoploss_on_exchange` if your exchange supports this.
 
+## Unavailable funding rates
+
+For futures data, exchanges commonly provide the futures candles, the marks, and the funding rates. However, it is common that whilst candles and marks might be available, the funding rates are not. This can affect backtesting timeranges, i.e. you may only be able to test recent timeranges and not earlier, experiencing the `No data found. Terminating.` error. To get around this, add the `futures_funding_rate` config option as listed in [configuration.md](configuration.md), and it is recommended that you set this to `0`, unless you know a given specific funding rate for your pair, exchange and timerange. Setting this to anything other than `0` can have drastic effects on your profit calculations within strategy, e.g. within the `custom_exit`, `custom_stoploss`, etc functions.
+
+!!! Warning "This will mean your backtests are inaccurate."
+    This will not overwrite funding rates that are available from the exchange, but bear in mind that setting a false funding rate will mean backtesting results will be inaccurate for historical timeranges where funding rates are not available.
+
 ### Developer
 
 #### Margin mode

freqtrade/data/history/history_utils.py

@@ -68,7 +68,8 @@ def load_data(datadir: Path,
               startup_candles: int = 0,
               fail_without_data: bool = False,
               data_format: str = 'json',
-              candle_type: CandleType = CandleType.SPOT
+              candle_type: CandleType = CandleType.SPOT,
+              user_futures_funding_rate: int = None,
               ) -> Dict[str, DataFrame]:
     """
     Load ohlcv history data for a list of pairs.
@@ -100,6 +101,10 @@ def load_data(datadir: Path,
                                  )
         if not hist.empty:
             result[pair] = hist
+        else:
+            if candle_type is CandleType.FUNDING_RATE and user_futures_funding_rate is not None:
+                logger.warn(f"{pair} using user specified [{user_futures_funding_rate}]")
+                result[pair] = DataFrame(columns=["open", "close", "high", "low", "volume"])
 
     if fail_without_data and not result:
         raise OperationalException("No data found. Terminating.")

freqtrade/exchange/exchange.py

@@ -2419,14 +2419,35 @@ class Exchange:
         )
 
     @staticmethod
-    def combine_funding_and_mark(funding_rates: DataFrame, mark_rates: DataFrame) -> DataFrame:
+    def combine_funding_and_mark(funding_rates: DataFrame, mark_rates: DataFrame,
+                                 futures_funding_rate: Optional[int] = None) -> DataFrame:
         """
         Combine funding-rates and mark-rates dataframes
         :param funding_rates: Dataframe containing Funding rates (Type FUNDING_RATE)
         :param mark_rates: Dataframe containing Mark rates (Type mark_ohlcv_price)
+        :param futures_funding_rate: Fake funding rate to use if funding_rates are not available
         """
+        if futures_funding_rate is None:
+            return mark_rates.merge(
+                funding_rates, on='date', how="inner", suffixes=["_mark", "_fund"])
+        else:
+            if len(funding_rates) == 0:
+                # No funding rate candles - full fillup with fallback variable
+                mark_rates['open_fund'] = futures_funding_rate
+                return mark_rates.rename(
+                        columns={'open': 'open_mark',
+                                 'close': 'close_mark',
+                                 'high': 'high_mark',
+                                 'low': 'low_mark',
+                                 'volume': 'volume_mark'})
 
-        return funding_rates.merge(mark_rates, on='date', how="inner", suffixes=["_fund", "_mark"])
+            else:
+                # Fill up missing funding_rate candles with fallback value
+                combined = mark_rates.merge(
+                    funding_rates, on='date', how="outer", suffixes=["_mark", "_fund"]
+                    )
+                combined['open_fund'] = combined['open_fund'].fillna(futures_funding_rate)
+                return combined
 
     def calculate_funding_fees(
         self,

freqtrade/optimize/backtesting.py

@@ -275,8 +275,12 @@ class Backtesting:
                 if pair not in self.exchange._leverage_tiers:
                     unavailable_pairs.append(pair)
                     continue
-                self.futures_data[pair] = funding_rates_dict[pair].merge(
+
-                    mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"])
+                self.futures_data[pair] = self.exchange.combine_funding_and_mark(
+                    funding_rates=funding_rates_dict[pair],
+                    mark_rates=mark_rates_dict[pair],
+                    futures_funding_rate=self.config.get('futures_funding_rate', None),
+                )
 
             if unavailable_pairs:
                 raise OperationalException(

tests/exchange/test_exchange.py

@@ -3915,6 +3915,70 @@ def test_calculate_funding_fees(
         ) == kraken_fee
 
 
+@pytest.mark.parametrize(
+    'mark_price,funding_rate,futures_funding_rate', [
+        (1000, 0.001, None),
+        (1000, 0.001, 0.01),
+        (1000, 0.001, 0.0),
+        (1000, 0.001, -0.01),
+    ])
+def test_combine_funding_and_mark(
+    default_conf,
+    mocker,
+    funding_rate,
+    mark_price,
+    futures_funding_rate,
+):
+    exchange = get_patched_exchange(mocker, default_conf)
+    prior2_date = timeframe_to_prev_date('1h', datetime.now(timezone.utc) - timedelta(hours=2))
+    prior_date = timeframe_to_prev_date('1h', datetime.now(timezone.utc) - timedelta(hours=1))
+    trade_date = timeframe_to_prev_date('1h', datetime.now(timezone.utc))
+    funding_rates = DataFrame([
+        {'date': prior2_date, 'open': funding_rate},
+        {'date': prior_date, 'open': funding_rate},
+        {'date': trade_date, 'open': funding_rate},
+    ])
+    mark_rates = DataFrame([
+        {'date': prior2_date, 'open': mark_price},
+        {'date': prior_date, 'open': mark_price},
+        {'date': trade_date, 'open': mark_price},
+    ])
+
+    df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate)
+    assert 'open_mark' in df.columns
+    assert 'open_fund' in df.columns
+    assert len(df) == 3
+
+    funding_rates = DataFrame([
+        {'date': trade_date, 'open': funding_rate},
+    ])
+    mark_rates = DataFrame([
+        {'date': prior2_date, 'open': mark_price},
+        {'date': prior_date, 'open': mark_price},
+        {'date': trade_date, 'open': mark_price},
+    ])
+    df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate)
+
+    if futures_funding_rate is not None:
+        assert len(df) == 3
+        assert df.iloc[0]['open_fund'] == futures_funding_rate
+        assert df.iloc[1]['open_fund'] == futures_funding_rate
+        assert df.iloc[2]['open_fund'] == funding_rate
+    else:
+        assert len(df) == 1
+
+    # Empty funding rates
+    funding_rates = DataFrame([], columns=['date', 'open'])
+    df = exchange.combine_funding_and_mark(funding_rates, mark_rates, futures_funding_rate)
+    if futures_funding_rate is not None:
+        assert len(df) == 3
+        assert df.iloc[0]['open_fund'] == futures_funding_rate
+        assert df.iloc[1]['open_fund'] == futures_funding_rate
+        assert df.iloc[2]['open_fund'] == futures_funding_rate
+    else:
+        assert len(df) == 0
+
+
 def test_get_or_calculate_liquidation_price(mocker, default_conf):
 
     api_mock = MagicMock()