Merge pull request #7615 from freqtrade/price_jump_warn

Add price jump warning

docs/faq.md

@@ -102,6 +102,12 @@ If this happens for all pairs in the pairlist, this might indicate a recent exch
 
 Irrespectively of the reason, Freqtrade will fill up these candles with "empty" candles, where open, high, low and close are set to the previous candle close - and volume is empty. In a chart, this will look like a `_` - and is aligned with how exchanges usually represent 0 volume candles.
 
+### I'm getting "Price jump between 2 candles detected"
+
+This message is a warning that the candles had a price jump of > 30%.
+This might be a sign that the pair stopped trading, and some token exchange took place (e.g. COCOS in 2021 - where price jumped from 0.0000154 to 0.01621).
+This message is often accompanied by ["Missing data fillup"](#im-getting-missing-data-fillup-messages-in-the-log) - as trading on such pairs is often stopped for some time.
+
 ### I'm getting "Outdated history for pair xxx" in the log
 
 The bot is trying to tell you that it got an outdated last candle (not the last complete candle).

freqtrade/data/history/idatahandler.py

@@ -303,7 +303,7 @@ class IDataHandler(ABC):
             timerange=timerange_startup,
             candle_type=candle_type
         )
-        if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
+        if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True):
             return pairdf
         else:
             enddate = pairdf.iloc[-1]['date']
@@ -323,8 +323,9 @@ class IDataHandler(ABC):
             self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data)
             return pairdf
 
-    def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str,
-                        candle_type: CandleType, warn_no_data: bool):
+    def _check_empty_df(
+            self, pairdf: DataFrame, pair: str, timeframe: str, candle_type: CandleType,
+            warn_no_data: bool, warn_price: bool = False) -> bool:
         """
         Warn on empty dataframe
         """
@@ -335,6 +336,20 @@ class IDataHandler(ABC):
                     "Use `freqtrade download-data` to download the data"
                 )
             return True
+        elif warn_price:
+            candle_price_gap = 0
+            if (candle_type in (CandleType.SPOT, CandleType.FUTURES) and
+                    not pairdf.empty
+                    and 'close' in pairdf.columns and 'open' in pairdf.columns):
+                # Detect gaps between prior close and open
+                gaps = ((pairdf['open'] - pairdf['close'].shift(1)) / pairdf['close'].shift(1))
+                gaps = gaps.dropna()
+                if len(gaps):
+                    candle_price_gap = max(abs(gaps))
+            if candle_price_gap > 0.1:
+                logger.info(f"Price jump in {pair}, {timeframe}, {candle_type} between two candles "
+                            f"of {candle_price_gap:.2%} detected.")
+
         return False
 
     def _validate_pairdata(self, pair, pairdata: DataFrame, timeframe: str,

tests/data/test_datahandler.py

@@ -15,7 +15,7 @@ from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, g
 from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
 from freqtrade.data.history.parquetdatahandler import ParquetDataHandler
 from freqtrade.enums import CandleType, TradingMode
-from tests.conftest import log_has
+from tests.conftest import log_has, log_has_re
 
 
 def test_datahandler_ohlcv_get_pairs(testdatadir):
@@ -154,6 +154,85 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog):
     assert df.columns.equals(df1.columns)
 
 
+def test_datahandler__check_empty_df(testdatadir, caplog):
+    dh = JsonDataHandler(testdatadir)
+    expected_text = r"Price jump in UNITTEST/USDT, 1h, spot between"
+    df = DataFrame([
+        [
+            1511686200000,  # 8:50:00
+            8.794,  # open
+            8.948,  # high
+            8.794,  # low
+            8.88,  # close
+            2255,  # volume (in quote currency)
+        ],
+        [
+            1511686500000,  # 8:55:00
+            8.88,
+            8.942,
+            8.88,
+            8.893,
+            9911,
+        ],
+        [
+            1511687100000,  # 9:05:00
+            8.891,
+            8.893,
+            8.875,
+            8.877,
+            2251
+        ],
+        [
+            1511687400000,  # 9:10:00
+            8.877,
+            8.883,
+            8.895,
+            8.817,
+            123551
+        ]
+    ], columns=['date', 'open', 'high', 'low', 'close', 'volume'])
+
+    dh._check_empty_df(df, 'UNITTEST/USDT', '1h', CandleType.SPOT, True, True)
+    assert not log_has_re(expected_text, caplog)
+    df = DataFrame([
+        [
+            1511686200000,  # 8:50:00
+            8.794,  # open
+            8.948,  # high
+            8.794,  # low
+            8.88,  # close
+            2255,  # volume (in quote currency)
+        ],
+        [
+            1511686500000,  # 8:55:00
+            8.88,
+            8.942,
+            8.88,
+            8.893,
+            9911,
+        ],
+        [
+            1511687100000,  # 9:05:00
+            889.1,   # Price jump by several decimals
+            889.3,
+            887.5,
+            887.7,
+            2251
+        ],
+        [
+            1511687400000,  # 9:10:00
+            8.877,
+            8.883,
+            8.895,
+            8.817,
+            123551
+        ]
+    ], columns=['date', 'open', 'high', 'low', 'close', 'volume'])
+
+    dh._check_empty_df(df, 'UNITTEST/USDT', '1h', CandleType.SPOT, True, True)
+    assert log_has_re(expected_text, caplog)
+
+
 @pytest.mark.parametrize('datahandler', ['feather', 'parquet'])
 def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
     dh = get_datahandler(testdatadir, datahandler)