use new channel apis in emc, extend analyzed df to include list of dates for candles

freqtrade/data/dataprovider.py

@@ -9,7 +9,7 @@ from collections import deque
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Tuple
 
-from pandas import DataFrame
+from pandas import DataFrame, concat, date_range
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe
@@ -120,7 +120,7 @@ class DataProvider:
                     'type': RPCMessageType.ANALYZED_DF,
                     'data': {
                         'key': pair_key,
-                        'df': dataframe,
+                        'df': dataframe.tail(1),
                         'la': datetime.now(timezone.utc)
                     }
                 }
@@ -157,6 +157,80 @@ class DataProvider:
         self.__producer_pairs_df[producer_name][pair_key] = (dataframe, _last_analyzed)
         logger.debug(f"External DataFrame for {pair_key} from {producer_name} added.")
 
+    def _add_external_candle(
+        self,
+        pair: str,
+        dataframe: DataFrame,
+        last_analyzed: datetime,
+        timeframe: str,
+        candle_type: CandleType,
+        producer_name: str = "default"
+    ) -> Tuple[bool, Optional[List[str]]]:
+        """
+        Append a candle to the existing external dataframe
+
+        :param pair: pair to get the data for
+        :param timeframe: Timeframe to get data for
+        :param candle_type: Any of the enum CandleType (must match trading mode!)
+        :returns: A tuple with a boolean value signifying if the candle was correctly appended,
+                  and a list of datetimes missing from the candle if it finds some.
+                  Will return false if has no data for `producer_name`.
+                  Will return false if no existing data for (pair, timeframe, candle_type).
+                  Will return false if there's missing candles, and a list of datetimes of
+                  the missing candles.
+        """
+        pair_key = (pair, timeframe, candle_type)
+
+        if producer_name not in self.__producer_pairs_df:
+            # We don't have data from this producer yet,
+            # so we can't append a candle
+            return (False, None)
+
+        if pair_key not in self.__producer_pairs_df[producer_name]:
+            # We don't have data for this pair_key,
+            # so we can't append a candle
+            return (False, None)
+
+        # CHECK FOR MISSING CANDLES
+
+        existing_df, _ = self.__producer_pairs_df[producer_name][pair_key]
+        appended_df = self._append_candle_to_dataframe(existing_df, dataframe)
+
+        # Everything is good, we appended
+        self.__producer_pairs_df[producer_name][pair_key] = appended_df, last_analyzed
+        return (True, None)
+
+    def _append_candle_to_dataframe(self, existing: DataFrame, new: DataFrame) -> DataFrame:
+        """
+        Append the `new` dataframe to the `existing` dataframe
+
+        :param existing: The full dataframe you want appended to
+        :param new: The new dataframe containing the data you want appended
+        :returns: The dataframe with the new data in it
+        """
+        if existing.iloc[-1]['date'] != new.iloc[-1]['date']:
+            existing = concat([existing, new])
+
+        # Only keep the last 1000 candles in memory
+        # TODO: Do this better
+        existing = existing[-1000:] if len(existing) > 1000 else existing
+
+        return existing
+
+    def _is_missing_candles(self, dataframe: DataFrame) -> bool:
+        """
+        Check if the dataframe is missing any candles
+
+        :param dataframe: The DataFrame to check
+        """
+        logger.info(dataframe.index)
+        return len(
+            date_range(
+                dataframe.index.min(),
+                dataframe.index.max()
+            ).difference(dataframe.index)
+        ) > 0
+
     def get_producer_df(
         self,
         pair: str,