From 0ff7a0771d1b5ca038a37b7e07368005abadae72 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 22 Oct 2022 08:37:30 +0200 Subject: [PATCH] Move price_jump_warn to dataloading it's not relevant for live data, and should only run when loading data from disk. --- freqtrade/data/converter.py | 8 --- freqtrade/data/history/idatahandler.py | 14 +++++ tests/data/test_converter.py | 9 ++- tests/data/test_datahandler.py | 81 +++++++++++++++++++++++++- 4 files changed, 98 insertions(+), 14 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 7f7e79445..98ed15489 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -119,14 +119,6 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) else: # Don't be verbose if only a small amount is missing logger.debug(message) - candle_price_gap = 0 - if not df.empty and 'close' in df.columns: - returns = df['close'].pct_change().dropna() - if len(returns): - candle_price_gap = max(abs(returns)) - if candle_price_gap > 0.3: - logger.info(f"Price jump in {pair} between two candles of {candle_price_gap:.2%} detected.") - return df diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 80e29f4c0..19d753043 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -335,6 +335,20 @@ class IDataHandler(ABC): "Use `freqtrade download-data` to download the data" ) return True + else: + candle_price_gap = 0 + if (candle_type in (CandleType.SPOT, CandleType.FUTURES) and + not pairdf.empty + and 'close' in pairdf.columns and 'open' in pairdf.columns): + # Detect gaps between prior close and open + gaps = ((pairdf['open'] - pairdf['close'].shift(1)) / pairdf['close'].shift(1)) + gaps = gaps.dropna() + if len(gaps): + candle_price_gap = max(abs(gaps)) + if candle_price_gap > 0.1: + logger.info(f"Price jump in {pair} between two candles of " + f"{candle_price_gap:.2%} detected.") + return False def _validate_pairdata(self, pair, pairdata: DataFrame, timeframe: str, diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 83429bb38..f74383d15 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -80,10 +80,10 @@ def test_ohlcv_fill_up_missing_data2(caplog): ticks = [ [ 1511686200000, # 8:50:00 - 8.794e-07, # open - 8.948e-07, # high - 8.794e-07, # low - 8.88e-07, # close + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close 2255, # volume (in quote currency) ], [ @@ -118,7 +118,6 @@ def test_ohlcv_fill_up_missing_data2(caplog): assert len(data) == 3 caplog.set_level(logging.DEBUG) data2 = ohlcv_fill_up_missing_data(data, timeframe, "UNITTEST/BTC") - assert log_has_re(r"Price jump in .* between two candles .* detected\.", caplog) assert len(data2) == 4 # 3rd candle has been filled row = data2.loc[2, :] diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 5d6d60f84..4b1f9634c 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -15,7 +15,7 @@ from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, g from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler from freqtrade.data.history.parquetdatahandler import ParquetDataHandler from freqtrade.enums import CandleType, TradingMode -from tests.conftest import log_has +from tests.conftest import log_has, log_has_re def test_datahandler_ohlcv_get_pairs(testdatadir): @@ -154,6 +154,85 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog): assert df.columns.equals(df1.columns) +def test_datahandler__check_empty_df(testdatadir, caplog): + dh = JsonDataHandler(testdatadir) + expected_text = r"Price jump in UNITTEST/USDT between" + df = DataFrame([ + [ + 1511686200000, # 8:50:00 + 8.794, # open + 8.948, # high + 8.794, # low + 8.88, # close + 2255, # volume (in quote currency) + ], + [ + 1511686500000, # 8:55:00 + 8.88, + 8.942, + 8.88, + 8.893, + 9911, + ], + [ + 1511687100000, # 9:05:00 + 8.891, + 8.893, + 8.875, + 8.877, + 2251 + ], + [ + 1511687400000, # 9:10:00 + 8.877, + 8.883, + 8.895, + 8.817, + 123551 + ] + ], columns=['date', 'open', 'high', 'low', 'close', 'volume']) + + dh._check_empty_df(df, 'UNITTEST/USDT', '1h', CandleType.SPOT, True) + assert not log_has_re(expected_text, caplog) + df = DataFrame([ + [ + 1511686200000, # 8:50:00 + 8.794, # open + 8.948, # high + 8.794, # low + 8.88, # close + 2255, # volume (in quote currency) + ], + [ + 1511686500000, # 8:55:00 + 8.88, + 8.942, + 8.88, + 8.893, + 9911, + ], + [ + 1511687100000, # 9:05:00 + 889.1, # Price jump by several decimals + 889.3, + 887.5, + 887.7, + 2251 + ], + [ + 1511687400000, # 9:10:00 + 8.877, + 8.883, + 8.895, + 8.817, + 123551 + ] + ], columns=['date', 'open', 'high', 'low', 'close', 'volume']) + + dh._check_empty_df(df, 'UNITTEST/USDT', '1h', CandleType.SPOT, True) + assert log_has_re(expected_text, caplog) + + @pytest.mark.parametrize('datahandler', ['feather', 'parquet']) def test_datahandler_trades_not_supported(datahandler, testdatadir, ): dh = get_datahandler(testdatadir, datahandler)