diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 59d72e337..5c3bbf90c 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -61,7 +61,7 @@ The FreqAI strategy requires including the following lines of code in the standa """ Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User controls the indicators - passed to the training/prediction by prepending indicators with `'%-' + coin ` + passed to the training/prediction by prepending indicators with `'%-' + pair ` (see convention below). I.e. user should not prepend any supporting metrics (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the model. @@ -69,20 +69,17 @@ The FreqAI strategy requires including the following lines of code in the standa :param df: strategy dataframe which will receive merges from informatives :param tf: timeframe of the dataframe which will modify the feature names :param informative: the dataframe associated with the informative pair - :param coin: the name of the coin which will modify the feature names. """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) # first loop is automatically duplicating indicators for time periods for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t) indicators = [col for col in informative if col.startswith("%")] # This loop duplicates and shifts all indicators to add a sense of recency to data @@ -134,7 +131,7 @@ Notice also the location of the labels under `if set_generalized_indicators:` at (as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`): ```python - def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin="", set_generalized_indicators=False): + def populate_any_indicators(self, pair, df, tf, informative=None, set_generalized_indicators=False): ... diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index b7c23aa60..3462955cc 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -2,7 +2,10 @@ ## Defining the features -Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%`, while labels/targets are prepended with `&`. +Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`. + +!!! Note + Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If you decide *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode. Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles." @@ -15,7 +18,7 @@ It is advisable to start from the template `populate_any_indicators()` in the so """ Function designed to automatically generate, name, and merge features from user-indicated timeframes in the configuration file. The user controls the indicators - passed to the training/prediction by prepending indicators with `'%-' + coin ` + passed to the training/prediction by prepending indicators with `'%-' + pair ` (see convention below). I.e., the user should not prepend any supporting metrics (e.g., bb_lowerband below) with % unless they explicitly want to pass that metric to the model. @@ -23,37 +26,34 @@ It is advisable to start from the template `populate_any_indicators()` in the so :param df: strategy dataframe which will receive merges from informatives :param tf: timeframe of the dataframe which will modify the feature names :param informative: the dataframe associated with the informative pair - :param coin: the name of the coin which will modify the feature names. """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) # first loop is automatically duplicating indicators for time periods for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t) bollinger = qtpylib.bollinger_bands( qtpylib.typical_price(informative), window=t, stds=2.2 ) - informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{coin}bb_width-period_{t}"] = ( - informative[f"{coin}bb_upperband-period_{t}"] - - informative[f"{coin}bb_lowerband-period_{t}"] - ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + informative[f"%-{pair}bb_width-period_{t}"] = ( + informative[f"{pair}bb_upperband-period_{t}"] + - informative[f"{pair}bb_lowerband-period_{t}"] + ) / informative[f"{pair}bb_middleband-period_{t}"] + informative[f"%-{pair}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] ) - informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative[f"%-{pair}relative_volume-period_{t}"] = ( informative["volume"] / informative["volume"].rolling(t).mean() ) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f09a216a2..f0e24dd80 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1137,6 +1137,51 @@ class FreqaiDataKitchen: if pair not in self.all_pairs: self.all_pairs.append(pair) + def extract_corr_pair_columns_from_populated_indicators( + self, + dataframe: DataFrame + ) -> Dict[str, DataFrame]: + """ + Find the columns of the dataframe corresponding to the corr_pairlist, save them + in a dictionary to be reused and attached to other pairs. + + :param dataframe: fully populated dataframe (current pair + corr_pairs) + :return: corr_dataframes, dictionary of dataframes to be attached + to other pairs in same candle. + """ + corr_dataframes: Dict[str, DataFrame] = {} + pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + + for pair in pairs: + valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] + pair_cols = [col for col in dataframe.columns if + any(substr in col for substr in valid_strs)] + pair_cols.insert(0, 'date') + corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) + + return corr_dataframes + + def attach_corr_pair_columns(self, dataframe: DataFrame, + corr_dataframes: Dict[str, DataFrame], + current_pair: str) -> DataFrame: + """ + Attach the existing corr_pair dataframes to the current pair dataframe before training + + :param dataframe: current pair strategy dataframe, indicators populated already + :param corr_dataframes: dictionary of saved dataframes from earlier in the same candle + :param current_pair: current pair to which we will attach corr pair dataframe + :return: + :dataframe: current pair dataframe of populated indicators, concatenated with corr_pairs + ready for training + """ + pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + + for pair in pairs: + if current_pair != pair: + dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date') + + return dataframe + def use_strategy_to_populate_indicators( self, strategy: IStrategy, @@ -1144,6 +1189,7 @@ class FreqaiDataKitchen: base_dataframes: dict = {}, pair: str = "", prediction_dataframe: DataFrame = pd.DataFrame(), + do_corr_pairs: bool = True, ) -> DataFrame: """ Use the user defined strategy for populating indicators during retrain @@ -1153,15 +1199,15 @@ class FreqaiDataKitchen: :param base_dataframes: dict = dict containing the current pair dataframes (for user defined timeframes) :param metadata: dict = strategy furnished pair metadata - :returns: + :return: dataframe: DataFrame = dataframe containing populated indicators """ # for prediction dataframe creation, we let dataprovider handle everything in the strategy # so we create empty dictionaries, which allows us to pass None to # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe. - tfs = self.freqai_config["feature_parameters"].get("include_timeframes") - pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") + pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) if not prediction_dataframe.empty: dataframe = prediction_dataframe.copy() for tf in tfs: @@ -1184,15 +1230,18 @@ class FreqaiDataKitchen: informative=base_dataframes[tf], set_generalized_indicators=sgi ) - if pairs: - for i in pairs: - if pair in i: - continue # dont repeat anything from whitelist + + # ensure corr pairs are always last + for corr_pair in pairs: + if pair == corr_pair: + continue # dont repeat anything from whitelist + for tf in tfs: + if pairs and do_corr_pairs: dataframe = strategy.populate_any_indicators( - i, + corr_pair, dataframe.copy(), tf, - informative=corr_dataframes[i][tf] + informative=corr_dataframes[corr_pair][tf] ) self.get_unique_classes_from_labels(dataframe) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 6cb7f79f0..dcf902954 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -5,7 +5,6 @@ from abc import ABC, abstractmethod from collections import deque from datetime import datetime, timezone from pathlib import Path -from threading import Lock from typing import Any, Dict, List, Literal, Tuple import numpy as np @@ -71,6 +70,7 @@ class IFreqaiModel(ABC): self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] + self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", []) self.keras: bool = self.freqai_info.get("keras", False) if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 @@ -82,9 +82,6 @@ class IFreqaiModel(ABC): self.pair_it_train = 0 self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) self.train_queue = self._set_train_queue() - self.last_trade_database_summary: DataFrame = {} - self.current_trade_database_summary: DataFrame = {} - self.analysis_lock = Lock() self.inference_time: float = 0 self.train_time: float = 0 self.begin_time: float = 0 @@ -92,6 +89,10 @@ class IFreqaiModel(ABC): self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) self.continual_learning = self.freqai_info.get('continual_learning', False) self.plot_features = self.ft_params.get("plot_feature_importances", 0) + self.corr_dataframes: Dict[str, DataFrame] = {} + # get_corr_dataframes is controlling the caching of corr_dataframes + # for improved performance. Careful with this boolean. + self.get_corr_dataframes: bool = True self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() @@ -364,10 +365,10 @@ class IFreqaiModel(ABC): # load the model and associated data into the data kitchen self.model = self.dd.load_data(metadata["pair"], dk) - with self.analysis_lock: - dataframe = self.dk.use_strategy_to_populate_indicators( - strategy, prediction_dataframe=dataframe, pair=metadata["pair"] - ) + dataframe = dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe, pair=metadata["pair"], + do_corr_pairs=self.get_corr_dataframes + ) if not self.model: logger.warning( @@ -376,6 +377,9 @@ class IFreqaiModel(ABC): self.dd.return_null_values_to_strategy(dataframe, dk) return dk + if self.corr_pairlist: + dataframe = self.cache_corr_pairlist_dfs(dataframe, dk) + dk.find_labels(dataframe) self.build_strategy_return_arrays(dataframe, dk, metadata["pair"], trained_timestamp) @@ -559,10 +563,9 @@ class IFreqaiModel(ABC): data_load_timerange, pair, dk ) - with self.analysis_lock: - unfiltered_dataframe = dk.use_strategy_to_populate_indicators( - strategy, corr_dataframes, base_dataframes, pair - ) + unfiltered_dataframe = dk.use_strategy_to_populate_indicators( + strategy, corr_dataframes, base_dataframes, pair + ) unfiltered_dataframe = dk.slice_dataframe(new_trained_timerange, unfiltered_dataframe) @@ -680,6 +683,8 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 + if self.corr_pairlist: + self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): @@ -738,6 +743,29 @@ class IFreqaiModel(ABC): f'Best approximation queue: {best_queue}') return best_queue + def cache_corr_pairlist_dfs(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: + """ + Cache the corr_pairlist dfs to speed up performance for subsequent pairs during the + current candle. + :param dataframe: strategy fed dataframe + :param dk: datakitchen object for current asset + :return: dataframe to attach/extract cached corr_pair dfs to/from. + """ + + if self.get_corr_dataframes: + self.corr_dataframes = dk.extract_corr_pair_columns_from_populated_indicators(dataframe) + if not self.corr_dataframes: + logger.warning("Couldn't cache corr_pair dataframes for improved performance. " + "Consider ensuring that the full coin/stake, e.g. XYZ/USD, " + "is included in the column names when you are creating features " + "in `populate_any_indicators()`.") + self.get_corr_dataframes = not bool(self.corr_dataframes) + else: + dataframe = dk.attach_corr_pair_columns( + dataframe, self.corr_dataframes, dk.pair) + + return dataframe + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/templates/FreqaiExampleHybridStrategy.py b/freqtrade/templates/FreqaiExampleHybridStrategy.py index 593a6062b..26335956f 100644 --- a/freqtrade/templates/FreqaiExampleHybridStrategy.py +++ b/freqtrade/templates/FreqaiExampleHybridStrategy.py @@ -110,8 +110,6 @@ class FreqaiExampleHybridStrategy(IStrategy): :param informative: the dataframe associated with the informative pair """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) @@ -119,13 +117,13 @@ class FreqaiExampleHybridStrategy(IStrategy): for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) - informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) - informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) - informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) - informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) + informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + informative[f"%-{pair}relative_volume-period_{t}"] = ( informative["volume"] / informative["volume"].rolling(t).mean() ) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index d58d61025..fc39b0ab4 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -53,7 +53,7 @@ class FreqaiExampleStrategy(IStrategy): """ Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User controls the indicators - passed to the training/prediction by prepending indicators with `'%-' + coin ` + passed to the training/prediction by prepending indicators with `f'%-{pair}` (see convention below). I.e. user should not prepend any supporting metrics (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the model. @@ -63,8 +63,6 @@ class FreqaiExampleStrategy(IStrategy): :param informative: the dataframe associated with the informative pair """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) @@ -72,36 +70,36 @@ class FreqaiExampleStrategy(IStrategy): for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) - informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) - informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) + informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) bollinger = qtpylib.bollinger_bands( qtpylib.typical_price(informative), window=t, stds=2.2 ) - informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{coin}bb_width-period_{t}"] = ( - informative[f"{coin}bb_upperband-period_{t}"] - - informative[f"{coin}bb_lowerband-period_{t}"] - ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + informative[f"%-{pair}bb_width-period_{t}"] = ( + informative[f"{pair}bb_upperband-period_{t}"] + - informative[f"{pair}bb_lowerband-period_{t}"] + ) / informative[f"{pair}bb_middleband-period_{t}"] + informative[f"%-{pair}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] ) - informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) - informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative[f"%-{pair}relative_volume-period_{t}"] = ( informative["volume"] / informative["volume"].rolling(t).mean() ) - informative[f"%-{coin}pct-change"] = informative["close"].pct_change() - informative[f"%-{coin}raw_volume"] = informative["volume"] - informative[f"%-{coin}raw_price"] = informative["close"] + informative[f"%-{pair}pct-change"] = informative["close"].pct_change() + informative[f"%-{pair}raw_volume"] = informative["volume"] + informative[f"%-{pair}raw_price"] = informative["close"] indicators = [col for col in informative if col.startswith("%")] # This loop duplicates and shifts all indicators to add a sense of recency to data