start tracking the current candle in FreqAI, add robustness to corr_df caching and inference timer, add test for cache corr_df

This commit is contained in:
robcaulk 2022-11-02 19:32:22 +01:00
parent 2ed04916ae
commit 255eb71270
3 changed files with 21 additions and 4 deletions

View File

@ -98,6 +98,7 @@ class FreqaiDataDrawer:
"model_filename": "", "trained_timestamp": 0, "model_filename": "", "trained_timestamp": 0,
"data_path": "", "extras": {}} "data_path": "", "extras": {}}
self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {}
self.current_candle: datetime = datetime.fromtimestamp(637887600)
def update_metric_tracker(self, metric: str, value: float, pair: str) -> None: def update_metric_tracker(self, metric: str, value: float, pair: str) -> None:
""" """
@ -636,6 +637,8 @@ class FreqaiDataDrawer:
axis=0, axis=0,
) )
self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date']
def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None: def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None:
""" """
Load pair histories for all whitelist and corr_pairlist pairs. Load pair histories for all whitelist and corr_pairlist pairs.

View File

@ -75,7 +75,7 @@ class IFreqaiModel(ABC):
if self.keras and self.ft_params.get("DI_threshold", 0): if self.keras and self.ft_params.get("DI_threshold", 0):
self.ft_params["DI_threshold"] = 0 self.ft_params["DI_threshold"] = 0
logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") logger.warning("DI threshold is not configured for Keras models yet. Deactivating.")
self.CONV_WIDTH = self.freqai_info.get("conv_width", 2) self.CONV_WIDTH = self.freqai_info.get("conv_width", 1)
if self.ft_params.get("inlier_metric_window", 0): if self.ft_params.get("inlier_metric_window", 0):
self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
self.pair_it = 0 self.pair_it = 0
@ -93,7 +93,7 @@ class IFreqaiModel(ABC):
# get_corr_dataframes is controlling the caching of corr_dataframes # get_corr_dataframes is controlling the caching of corr_dataframes
# for improved performance. Careful with this boolean. # for improved performance. Careful with this boolean.
self.get_corr_dataframes: bool = True self.get_corr_dataframes: bool = True
self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
self._threads: List[threading.Thread] = [] self._threads: List[threading.Thread] = []
self._stop_event = threading.Event() self._stop_event = threading.Event()
@ -339,6 +339,7 @@ class IFreqaiModel(ABC):
if self.dd.historic_data: if self.dd.historic_data:
self.dd.update_historic_data(strategy, dk) self.dd.update_historic_data(strategy, dk)
logger.debug(f'Updating historic data on pair {metadata["pair"]}') logger.debug(f'Updating historic data on pair {metadata["pair"]}')
self.track_current_candle()
if not self.follow_mode: if not self.follow_mode:
@ -683,8 +684,8 @@ class IFreqaiModel(ABC):
" avoid blinding open trades and degrading performance.") " avoid blinding open trades and degrading performance.")
self.pair_it = 0 self.pair_it = 0
self.inference_time = 0 self.inference_time = 0
if self.corr_pairlist: # if self.corr_pairlist:
self.get_corr_dataframes = True # self.get_corr_dataframes = True
return return
def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''):
@ -766,6 +767,18 @@ class IFreqaiModel(ABC):
return dataframe return dataframe
def track_current_candle(self):
"""
Checks if the latest candle appended by the datadrawer is
equivalent to the latest candle seen by FreqAI. If not, it
asks to refresh the cached corr_dfs, and resets the pair
counter.
"""
if self.dd.current_candle > self.current_candle:
self.get_corr_dataframes = True
self.pair_it = 0
self.current_candle = self.dd.current_candle
# Following methods which are overridden by user made prediction models. # Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModel.py for an example. # See freqai/prediction_models/CatboostPredictionModel.py for an example.

View File

@ -192,6 +192,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
df = freqai.cache_corr_pairlist_dfs(df, freqai.dk)
for i in range(5): for i in range(5):
df[f'%-constant_{i}'] = i df[f'%-constant_{i}'] = i
# df.loc[:, f'%-constant_{i}'] = i # df.loc[:, f'%-constant_{i}'] = i