diff --git a/docs/faq.md b/docs/faq.md index a72268ef9..bcceaf898 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -102,6 +102,12 @@ If this happens for all pairs in the pairlist, this might indicate a recent exch Irrespectively of the reason, Freqtrade will fill up these candles with "empty" candles, where open, high, low and close are set to the previous candle close - and volume is empty. In a chart, this will look like a `_` - and is aligned with how exchanges usually represent 0 volume candles. +### I'm getting "Price jump between 2 candles detected" + +This message is a warning that the candles had a price jump of > 30%. +This might be a sign that the pair stopped trading, and some token exchange took place (e.g. COCOS in 2021 - where price jumped from 0.0000154 to 0.01621). +This message is often accompanied by ["Missing data fillup"](#im-getting-missing-data-fillup-messages-in-the-log) - as trading on such pairs is often stopped for some time. + ### I'm getting "Outdated history for pair xxx" in the log The bot is trying to tell you that it got an outdated last candle (not the last complete candle). diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 80e29f4c0..cbc3f1a34 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -303,7 +303,7 @@ class IDataHandler(ABC): timerange=timerange_startup, candle_type=candle_type ) - if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data): + if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True): return pairdf else: enddate = pairdf.iloc[-1]['date'] @@ -323,8 +323,9 @@ class IDataHandler(ABC): self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data) return pairdf - def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, - candle_type: CandleType, warn_no_data: bool): + def _check_empty_df( + self, pairdf: DataFrame, pair: str, timeframe: str, candle_type: CandleType, + warn_no_data: bool, warn_price: bool = False) -> bool: """ Warn on empty dataframe """ @@ -335,6 +336,20 @@ class IDataHandler(ABC): "Use `freqtrade download-data` to download the data" ) return True + elif warn_price: + candle_price_gap = 0 + if (candle_type in (CandleType.SPOT, CandleType.FUTURES) and + not pairdf.empty + and 'close' in pairdf.columns and 'open' in pairdf.columns): + # Detect gaps between prior close and open + gaps = ((pairdf['open'] - pairdf['close'].shift(1)) / pairdf['close'].shift(1)) + gaps = gaps.dropna() + if len(gaps): + candle_price_gap = max(abs(gaps)) + if candle_price_gap > 0.1: + logger.info(f"Price jump in {pair}, {timeframe}, {candle_type} between two candles " + f"of {candle_price_gap:.2%} detected.") + return False def _validate_pairdata(self, pair, pairdata: DataFrame, timeframe: str, diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 5d6d60f84..67eeda7d0 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -15,7 +15,7 @@ from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, g from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler from freqtrade.data.history.parquetdatahandler import ParquetDataHandler from freqtrade.enums import CandleType, TradingMode -from tests.conftest import log_has +from tests.conftest import log_has, log_has_re def test_datahandler_ohlcv_get_pairs(testdatadir): @@ -154,6 +154,85 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog): assert df.columns.equals(df1.columns) +def test_datahandler__check_empty_df(testdatadir, caplog): + dh = JsonDataHandler(testdatadir) + expected_text = r"Price jump in UNITTEST/USDT, 1h, spot between" + df = DataFrame([ + [ + 1511686200000, # 8:50:00 + 8.794, # open + 8.948, # high + 8.794, # low + 8.88, # close + 2255, # volume (in quote currency) + ], + [ + 1511686500000, # 8:55:00 + 8.88, + 8.942, + 8.88, + 8.893, + 9911, + ], + [ + 1511687100000, # 9:05:00 + 8.891, + 8.893, + 8.875, + 8.877, + 2251 + ], + [ + 1511687400000, # 9:10:00 + 8.877, + 8.883, + 8.895, + 8.817, + 123551 + ] + ], columns=['date', 'open', 'high', 'low', 'close', 'volume']) + + dh._check_empty_df(df, 'UNITTEST/USDT', '1h', CandleType.SPOT, True, True) + assert not log_has_re(expected_text, caplog) + df = DataFrame([ + [ + 1511686200000, # 8:50:00 + 8.794, # open + 8.948, # high + 8.794, # low + 8.88, # close + 2255, # volume (in quote currency) + ], + [ + 1511686500000, # 8:55:00 + 8.88, + 8.942, + 8.88, + 8.893, + 9911, + ], + [ + 1511687100000, # 9:05:00 + 889.1, # Price jump by several decimals + 889.3, + 887.5, + 887.7, + 2251 + ], + [ + 1511687400000, # 9:10:00 + 8.877, + 8.883, + 8.895, + 8.817, + 123551 + ] + ], columns=['date', 'open', 'high', 'low', 'close', 'volume']) + + dh._check_empty_df(df, 'UNITTEST/USDT', '1h', CandleType.SPOT, True, True) + assert log_has_re(expected_text, caplog) + + @pytest.mark.parametrize('datahandler', ['feather', 'parquet']) def test_datahandler_trades_not_supported(datahandler, testdatadir, ): dh = get_datahandler(testdatadir, datahandler)