Improve external_candle aggregation

freqtrade/data/dataprovider.py

@@ -9,7 +9,7 @@ from collections import deque
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Tuple
 
-from pandas import DataFrame, concat, to_timedelta
+from pandas import DataFrame, to_timedelta
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe
@@ -191,30 +191,13 @@ class DataProvider:
 
         existing_df, _ = self.__producer_pairs_df[producer_name][pair_key]
 
-        # Handle overlapping candles
-        old_candles = existing_df[
-            ~existing_df['date'].isin(
-                dataframe['date']
-            )
-        ]
-        overlapping_candles = existing_df[
-            existing_df['date'].isin(
-                dataframe['date']
-            )
-        ]
-        new_candles = dataframe[
-            ~dataframe['date'].isin(
-                existing_df['date']
-            )
-        ]
-
-        if overlapping_candles:
-            existing_df = concat([old_candles, overlapping_candles], axis=0)
-
         # CHECK FOR MISSING CANDLES
         timeframe_delta = to_timedelta(timeframe)  # Convert the timeframe to a timedelta for pandas
         local_last = existing_df.iloc[-1]['date']  # We want the last date from our copy
-        incoming_first = new_candles.iloc[0]['date']  # We want the first date from the incoming
+        incoming_first = dataframe.iloc[0]['date']  # We want the first date from the incoming
+
+        # Remove existing candles that are newer than the incoming first candle
+        existing_df1 = existing_df[existing_df['date'] < incoming_first]
 
         candle_difference = (incoming_first - local_last) / timeframe_delta
 
@@ -225,8 +208,10 @@ class DataProvider:
         # so return False and candle_difference.
         if candle_difference > 1:
             return (False, candle_difference)
-
-        appended_df = append_candles_to_dataframe(existing_df, dataframe)
+        if existing_df1.empty:
+            appended_df = dataframe
+        else:
+            appended_df = append_candles_to_dataframe(existing_df1, dataframe)
 
         # Everything is good, we appended
         self.__producer_pairs_df[producer_name][pair_key] = appended_df, last_analyzed