From dc256684686f41c46527a88cdcc75496bd165612 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 4 Jan 2023 11:41:06 +0100 Subject: [PATCH 1/4] handle data gaps between FreqAI and DP better --- freqtrade/freqai/data_drawer.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 848fb20eb..d8e08ed2d 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -645,12 +645,20 @@ class FreqaiDataDrawer: + 1 ) except IndexError: + index = -1 + if history_data[pair][tf].iloc[-1]['date'] < df_dp['date'].iloc[0]: + index = 0 + else: + index = -1 logger.warning( - f"Unable to update pair history for {pair}. " - "If this does not resolve itself after 1 additional candle, " - "please report the error to #freqai discord channel" + f"No common dates in historical data and dataprovider for {pair}. " + f"Appending dataprovider to historical data (full? {not bool(index)})" + "but please be aware that there is likely a gap in the historical " + "data.\n" + f"Historical data ends at {history_data[pair][tf].iloc[-1]['date']} " + f"while dataprovider starts at {df_dp['date'].iloc[0]} and" + f"ends at {df_dp['date'].iloc[0]}." ) - return history_data[pair][tf] = pd.concat( [ From 3cbe51c3caf50070bbe74e09d23c6161893d3cb7 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 4 Jan 2023 13:58:25 +0100 Subject: [PATCH 2/4] remove duplicated line --- freqtrade/freqai/data_drawer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index d8e08ed2d..1317ba6d3 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -645,7 +645,6 @@ class FreqaiDataDrawer: + 1 ) except IndexError: - index = -1 if history_data[pair][tf].iloc[-1]['date'] < df_dp['date'].iloc[0]: index = 0 else: From 986bc63e54f9c6953a6bb2064e0fa892eff01339 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 21 Feb 2023 21:23:58 +0100 Subject: [PATCH 3/4] raise OperationalException if latest historical data candle is older than earliest dataprovider candle --- freqtrade/freqai/data_drawer.py | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 1317ba6d3..883b9d94b 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -627,12 +627,12 @@ class FreqaiDataDrawer: for pair in dk.all_pairs: for tf in feat_params.get("include_timeframes"): - + hist_df = history_data[pair][tf] # check if newest candle is already appended df_dp = strategy.dp.get_pair_dataframe(pair, tf) if len(df_dp.index) == 0: continue - if str(history_data[pair][tf].iloc[-1]["date"]) == str( + if str(hist_df.iloc[-1]["date"]) == str( df_dp.iloc[-1:]["date"].iloc[-1] ): continue @@ -640,28 +640,30 @@ class FreqaiDataDrawer: try: index = ( df_dp.loc[ - df_dp["date"] == history_data[pair][tf].iloc[-1]["date"] + df_dp["date"] == hist_df.iloc[-1]["date"] ].index[0] + 1 ) except IndexError: - if history_data[pair][tf].iloc[-1]['date'] < df_dp['date'].iloc[0]: - index = 0 + if hist_df.iloc[-1]['date'] < df_dp['date'].iloc[0]: + raise OperationalException("In memory historical data is older than " + f"oldest DataProvider candle for {pair} on " + f"timeframe {tf}") else: index = -1 - logger.warning( - f"No common dates in historical data and dataprovider for {pair}. " - f"Appending dataprovider to historical data (full? {not bool(index)})" - "but please be aware that there is likely a gap in the historical " - "data.\n" - f"Historical data ends at {history_data[pair][tf].iloc[-1]['date']} " - f"while dataprovider starts at {df_dp['date'].iloc[0]} and" - f"ends at {df_dp['date'].iloc[0]}." - ) + logger.warning( + f"No common dates in historical data and dataprovider for {pair}. " + f"Appending latest dataprovider candle to historical data " + "but please be aware that there is likely a gap in the historical " + "data. \n" + f"Historical data ends at {hist_df.iloc[-1]['date']} " + f"while dataprovider starts at {df_dp['date'].iloc[0]} and" + f"ends at {df_dp['date'].iloc[0]}." + ) - history_data[pair][tf] = pd.concat( + hist_df = pd.concat( [ - history_data[pair][tf], + hist_df, df_dp.iloc[index:], ], ignore_index=True, From e88bb4e05ca7cdd17656fe6aa118631c828e501d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Feb 2023 15:09:25 +0100 Subject: [PATCH 4/4] Revert small change - otherwise the data is never updated. --- freqtrade/freqai/data_drawer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 883b9d94b..356100aea 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -661,7 +661,7 @@ class FreqaiDataDrawer: f"ends at {df_dp['date'].iloc[0]}." ) - hist_df = pd.concat( + history_data[pair][tf] = pd.concat( [ hist_df, df_dp.iloc[index:],