From 4cbb3341d7160e21a55b86738100a1f49bfc7a6b Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Fri, 25 Nov 2022 19:04:51 -0700 Subject: [PATCH] change how missing candles will be handled --- freqtrade/data/dataprovider.py | 35 +++++----------------- freqtrade/rpc/external_message_consumer.py | 4 +-- freqtrade/rpc/rpc.py | 13 ++++---- 3 files changed, 15 insertions(+), 37 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index d6eb217a8..07999fc90 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -7,9 +7,9 @@ Common Interface for bot and strategy to access data. import logging from collections import deque from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union -from pandas import DataFrame, concat, date_range +from pandas import DataFrame, concat from freqtrade.configuration import TimeRange from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe @@ -165,40 +165,36 @@ class DataProvider: timeframe: str, candle_type: CandleType, producer_name: str = "default" - ) -> Tuple[bool, Optional[List[str]]]: + ) -> Union[bool, int]: """ Append a candle to the existing external dataframe :param pair: pair to get the data for :param timeframe: Timeframe to get data for :param candle_type: Any of the enum CandleType (must match trading mode!) - :returns: A tuple with a boolean value signifying if the candle was correctly appended, - and a list of datetimes missing from the candle if it finds some. - Will return false if has no data for `producer_name`. - Will return false if no existing data for (pair, timeframe, candle_type). - Will return false if there's missing candles, and a list of datetimes of - the missing candles. + :returns: False if the candle could not be appended, or the int number of missing candles. """ pair_key = (pair, timeframe, candle_type) if producer_name not in self.__producer_pairs_df: # We don't have data from this producer yet, # so we can't append a candle - return (False, None) + return False if pair_key not in self.__producer_pairs_df[producer_name]: # We don't have data for this pair_key, # so we can't append a candle - return (False, None) + return False # CHECK FOR MISSING CANDLES + # return int existing_df, _ = self.__producer_pairs_df[producer_name][pair_key] appended_df = self._append_candle_to_dataframe(existing_df, dataframe) # Everything is good, we appended self.__producer_pairs_df[producer_name][pair_key] = appended_df, last_analyzed - return (True, None) + return True def _append_candle_to_dataframe(self, existing: DataFrame, new: DataFrame) -> DataFrame: """ @@ -212,25 +208,10 @@ class DataProvider: existing = concat([existing, new]) # Only keep the last 1000 candles in memory - # TODO: Do this better existing = existing[-1000:] if len(existing) > 1000 else existing return existing - def _is_missing_candles(self, dataframe: DataFrame) -> bool: - """ - Check if the dataframe is missing any candles - - :param dataframe: The DataFrame to check - """ - logger.info(dataframe.index) - return len( - date_range( - dataframe.index.min(), - dataframe.index.max() - ).difference(dataframe.index) - ) > 0 - def get_producer_df( self, pair: str, diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index 24731ef4f..231642142 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -388,8 +388,8 @@ class ExternalMessageConsumer: producer_name=producer_name ) - elif len(df) == 1: - # This is just a single candle + elif len(df) < 999: + # This is n single candles # Have dataprovider append it to # the full datafame. If it can't, # request the missing candles diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8b23d33e7..2452a61b8 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1062,31 +1062,28 @@ class RPC: self, pair: str, timeframe: str, - limit: Optional[Union[int, List[str]]] = None + limit: Optional[int] = None ) -> Tuple[DataFrame, datetime]: """ Get the dataframe and last analyze from the dataprovider :param pair: The pair to get :param timeframe: The timeframe of data to get - :param limit: If an integer, limits the size of dataframe - If a list of string date times, only returns those candles + :param limit: The amount of candles in the dataframe """ _data, last_analyzed = self._freqtrade.dataprovider.get_analyzed_dataframe( pair, timeframe) _data = _data.copy() - if limit and isinstance(limit, int): + if limit: _data = _data.iloc[-limit:] - elif limit and isinstance(limit, str): - _data = _data.iloc[_data['date'].isin(limit)] return _data, last_analyzed def _ws_all_analysed_dataframes( self, pairlist: List[str], - limit: Optional[Union[int, List[str]]] = None + limit: Optional[int] = None ) -> Generator[Dict[str, Any], None, None]: """ Get the analysed dataframes of each pair in the pairlist. @@ -1113,7 +1110,7 @@ class RPC: def _ws_request_analyzed_df( self, pair: Optional[str], - limit: Optional[Union[int, List[str]]] = None, + limit: Optional[int] = None, ): """ Historical Analyzed Dataframes for WebSocket """ pairlist = [pair] if pair else self._freqtrade.active_pair_whitelist