update overlapping candle handling, move append to misc

This commit is contained in:
Timothy Pogue 2022-12-06 16:00:28 -07:00
parent 0602479f7d
commit 6717dff19b
2 changed files with 40 additions and 26 deletions

View File

@ -17,6 +17,7 @@ from freqtrade.data.history import load_pair_history
from freqtrade.enums import CandleType, RPCMessageType, RunMode from freqtrade.enums import CandleType, RPCMessageType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange, timeframe_to_seconds from freqtrade.exchange import Exchange, timeframe_to_seconds
from freqtrade.misc import append_candles_to_dataframe
from freqtrade.rpc import RPCManager from freqtrade.rpc import RPCManager
from freqtrade.util import PeriodicCache from freqtrade.util import PeriodicCache
@ -190,18 +191,30 @@ class DataProvider:
existing_df, la = self.__producer_pairs_df[producer_name][pair_key] existing_df, la = self.__producer_pairs_df[producer_name][pair_key]
# Iterate over any overlapping candles and update the values # Handle overlapping candles
for idx, candle in dataframe.iterrows(): old_candles = existing_df[
existing_df.iloc[ ~existing_df['date'].isin(
existing_df['date'] == candle['date'] dataframe['date']
] = candle )
]
overlapping_candles = existing_df[
existing_df['date'].isin(
dataframe['date']
)
]
new_candles = dataframe[
~dataframe['date'].isin(
existing_df['date']
)
]
existing_df.reset_index(drop=True, inplace=True) if overlapping_candles:
existing_df = concat([old_candles, overlapping_candles], axis=0)
# CHECK FOR MISSING CANDLES # CHECK FOR MISSING CANDLES
timeframe_delta = to_timedelta(timeframe) # Convert the timeframe to a timedelta for pandas timeframe_delta = to_timedelta(timeframe) # Convert the timeframe to a timedelta for pandas
local_last = existing_df.iloc[-1]['date'] # We want the last date from our copy of data local_last = existing_df.iloc[-1]['date'] # We want the last date from our copy
incoming_first = dataframe.iloc[0]['date'] # We want the first date from the incoming data incoming_first = new_candles.iloc[0]['date'] # We want the first date from the incoming
candle_difference = (incoming_first - local_last) / timeframe_delta candle_difference = (incoming_first - local_last) / timeframe_delta
@ -213,29 +226,12 @@ class DataProvider:
if candle_difference > 1: if candle_difference > 1:
return (False, candle_difference) return (False, candle_difference)
appended_df = self._append_candle_to_dataframe(existing_df, dataframe) appended_df = append_candles_to_dataframe(existing_df, dataframe)
# Everything is good, we appended # Everything is good, we appended
self.__producer_pairs_df[producer_name][pair_key] = appended_df, last_analyzed self.__producer_pairs_df[producer_name][pair_key] = appended_df, last_analyzed
return (True, 0) return (True, 0)
def _append_candle_to_dataframe(self, existing: DataFrame, new: DataFrame) -> DataFrame:
"""
Append the `new` dataframe to the `existing` dataframe
:param existing: The full dataframe you want appended to
:param new: The new dataframe containing the data you want appended
:returns: The dataframe with the new data in it
"""
if existing.iloc[-1]['date'] != new.iloc[-1]['date']:
existing = concat([existing, new])
# Only keep the last 1500 candles in memory
existing = existing[-1500:] if len(existing) > 1500 else existing
existing.reset_index(drop=True, inplace=True)
return existing
def get_producer_df( def get_producer_df(
self, self,
pair: str, pair: str,

View File

@ -301,3 +301,21 @@ def remove_entry_exit_signals(dataframe: pd.DataFrame):
dataframe[SignalTagType.EXIT_TAG.value] = None dataframe[SignalTagType.EXIT_TAG.value] = None
return dataframe return dataframe
def append_candles_to_dataframe(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
"""
Append the `right` dataframe to the `left` dataframe
:param left: The full dataframe you want appended to
:param right: The new dataframe containing the data you want appended
:returns: The dataframe with the right data in it
"""
if left.iloc[-1]['date'] != right.iloc[-1]['date']:
left = pd.concat([left, right])
# Only keep the last 1500 candles in memory
left = left[-1500:] if len(left) > 1500 else left
left.reset_index(drop=True, inplace=True)
return left