From a9db668082c42532ae73f65dbbad5fe41d9d6a68 Mon Sep 17 00:00:00 2001 From: rcaulk Date: Thu, 20 Oct 2022 16:30:32 +0200 Subject: [PATCH 1/6] avoid redundant indicator population for corr_pair list --- freqtrade/freqai/data_kitchen.py | 55 ++++++++++++++++++++++++++-- freqtrade/freqai/freqai_interface.py | 20 +++++++--- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 52261d341..8e3e22af1 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1126,6 +1126,49 @@ class FreqaiDataKitchen: if pair not in self.all_pairs: self.all_pairs.append(pair) + def extract_corr_pair_columns_from_populated_indicators( + self, + dataframe: DataFrame + ) -> Dict[str, DataFrame]: + """ + Find the columns of the dataframe corresponding to the corr_pairlist, save them + in a dictionary to be reused and attached to other pairs. + :params: + :dataframe: fully populated dataframe (current pair + corr_pairs) + :return: + :corr_dataframes: dictionary of dataframes to be attached to other pairs in same candle. + """ + corr_dataframes: Dict[str, DataFrame] = {} + pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + + for pair in pairs: + coin = pair.split('/')[0] + pair_cols = [col for col in dataframe.columns if coin in col] + corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) + + return corr_dataframes + + def attach_corr_pair_columns(self, dataframe: DataFrame, + corr_dataframes: Dict[str, DataFrame], + current_pair: str) -> DataFrame: + """ + Attach the existing corr_pair dataframes to the current pair dataframe before training + :params: + :dataframe: current pair strategy dataframe, indicators populated already + :corr_dataframes: dictionary of saved dataframes from earlier in the same candle + :current_pair: current pair to which we will attach corr pair dataframe + :return: + :dataframe: current pair dataframe of populated indicators, concatenated with corr_pairs + ready for training + """ + pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + + for pair in pairs: + if current_pair not in pair: + dataframe = pd.concat([dataframe, corr_dataframes[pair]], axis=1) + + return dataframe + def use_strategy_to_populate_indicators( self, strategy: IStrategy, @@ -1133,6 +1176,7 @@ class FreqaiDataKitchen: base_dataframes: dict = {}, pair: str = "", prediction_dataframe: DataFrame = pd.DataFrame(), + do_corr_pairs: bool = True, ) -> DataFrame: """ Use the user defined strategy for populating indicators during retrain @@ -1173,10 +1217,13 @@ class FreqaiDataKitchen: informative=base_dataframes[tf], set_generalized_indicators=sgi ) - if pairs: - for i in pairs: - if pair in i: - continue # dont repeat anything from whitelist + + # ensure corr pairs are always last + for i in pairs: + if pair in i: + continue # dont repeat anything from whitelist + for tf in tfs: + if pairs and do_corr_pairs: dataframe = strategy.populate_any_indicators( i, dataframe.copy(), diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index db0d4c379..d5591a03c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -83,8 +83,6 @@ class IFreqaiModel(ABC): self.pair_it_train = 0 self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) self.train_queue = self._set_train_queue() - self.last_trade_database_summary: DataFrame = {} - self.current_trade_database_summary: DataFrame = {} self.analysis_lock = Lock() self.inference_time: float = 0 self.train_time: float = 0 @@ -93,6 +91,8 @@ class IFreqaiModel(ABC): self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) self.continual_learning = self.freqai_info.get('continual_learning', False) self.plot_features = self.ft_params.get("plot_feature_importances", 0) + self.corr_dataframes: Dict[str, DataFrame] = {} + self.get_corr_dataframes: bool = True self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() @@ -363,10 +363,10 @@ class IFreqaiModel(ABC): # load the model and associated data into the data kitchen self.model = self.dd.load_data(metadata["pair"], dk) - with self.analysis_lock: - dataframe = self.dk.use_strategy_to_populate_indicators( - strategy, prediction_dataframe=dataframe, pair=metadata["pair"] - ) + dataframe = dk.use_strategy_to_populate_indicators( + strategy, prediction_dataframe=dataframe, pair=metadata["pair"], + do_corr_pairs=self.get_corr_dataframes + ) if not self.model: logger.warning( @@ -375,6 +375,13 @@ class IFreqaiModel(ABC): self.dd.return_null_values_to_strategy(dataframe, dk) return dk + if self.get_corr_dataframes: + self.corr_dataframes = dk.extract_corr_pair_columns_from_populated_indicators(dataframe) + self.get_corr_dataframes = False + else: + dataframe = dk.attach_corr_pair_columns( + dataframe, self.corr_dataframes, metadata["pair"]) + dk.find_labels(dataframe) self.build_strategy_return_arrays(dataframe, dk, metadata["pair"], trained_timestamp) @@ -680,6 +687,7 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 + self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): From 650bb8b7d77c3b61b5f9527dd490c6786a5fb3de Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 29 Oct 2022 22:26:49 +0200 Subject: [PATCH 2/6] ensure full pair string is used for caching dataframes. If not, revert to old behavior. Update docs. --- docs/freqai-configuration.md | 13 +++--- docs/freqai-feature-engineering.md | 36 ++++++++--------- freqtrade/freqai/data_kitchen.py | 40 ++++++++++--------- freqtrade/freqai/freqai_interface.py | 35 ++++++++++++---- freqtrade/templates/FreqaiExampleStrategy.py | 42 ++++++++++---------- 5 files changed, 92 insertions(+), 74 deletions(-) diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index d162fe373..7005072df 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -61,7 +61,7 @@ The FreqAI strategy requires including the following lines of code in the standa """ Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User controls the indicators - passed to the training/prediction by prepending indicators with `'%-' + coin ` + passed to the training/prediction by prepending indicators with `'%-' + pair ` (see convention below). I.e. user should not prepend any supporting metrics (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the model. @@ -69,20 +69,17 @@ The FreqAI strategy requires including the following lines of code in the standa :param df: strategy dataframe which will receive merges from informatives :param tf: timeframe of the dataframe which will modify the feature names :param informative: the dataframe associated with the informative pair - :param coin: the name of the coin which will modify the feature names. """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) # first loop is automatically duplicating indicators for time periods for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t) indicators = [col for col in informative if col.startswith("%")] # This loop duplicates and shifts all indicators to add a sense of recency to data @@ -134,7 +131,7 @@ Notice also the location of the labels under `if set_generalized_indicators:` at (as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`): ```python - def populate_any_indicators(self, metadata, pair, df, tf, informative=None, coin="", set_generalized_indicators=False): + def populate_any_indicators(self, pair, df, tf, informative=None, set_generalized_indicators=False): ... diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index b7c23aa60..8a1af69b7 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -2,7 +2,10 @@ ## Defining the features -Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%`, while labels/targets are prepended with `&`. +Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`. + +!!! Note + Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If users elect *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode. Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles." @@ -15,7 +18,7 @@ It is advisable to start from the template `populate_any_indicators()` in the so """ Function designed to automatically generate, name, and merge features from user-indicated timeframes in the configuration file. The user controls the indicators - passed to the training/prediction by prepending indicators with `'%-' + coin ` + passed to the training/prediction by prepending indicators with `'%-' + pair ` (see convention below). I.e., the user should not prepend any supporting metrics (e.g., bb_lowerband below) with % unless they explicitly want to pass that metric to the model. @@ -23,37 +26,34 @@ It is advisable to start from the template `populate_any_indicators()` in the so :param df: strategy dataframe which will receive merges from informatives :param tf: timeframe of the dataframe which will modify the feature names :param informative: the dataframe associated with the informative pair - :param coin: the name of the coin which will modify the feature names. """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) # first loop is automatically duplicating indicators for time periods for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t) bollinger = qtpylib.bollinger_bands( qtpylib.typical_price(informative), window=t, stds=2.2 ) - informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{coin}bb_width-period_{t}"] = ( - informative[f"{coin}bb_upperband-period_{t}"] - - informative[f"{coin}bb_lowerband-period_{t}"] - ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + informative[f"%-{pair}bb_width-period_{t}"] = ( + informative[f"{pair}bb_upperband-period_{t}"] + - informative[f"{pair}bb_lowerband-period_{t}"] + ) / informative[f"{pair}bb_middleband-period_{t}"] + informative[f"%-{pair}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] ) - informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative[f"%-{pair}relative_volume-period_{t}"] = ( informative["volume"] / informative["volume"].rolling(t).mean() ) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 8e3e22af1..6983e7e54 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1133,17 +1133,19 @@ class FreqaiDataKitchen: """ Find the columns of the dataframe corresponding to the corr_pairlist, save them in a dictionary to be reused and attached to other pairs. - :params: - :dataframe: fully populated dataframe (current pair + corr_pairs) - :return: - :corr_dataframes: dictionary of dataframes to be attached to other pairs in same candle. + + :param dataframe: fully populated dataframe (current pair + corr_pairs) + :return: corr_dataframes, dictionary of dataframes to be attached + to other pairs in same candle. """ corr_dataframes: Dict[str, DataFrame] = {} pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: - coin = pair.split('/')[0] - pair_cols = [col for col in dataframe.columns if coin in col] + valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] + pair_cols = [col for col in dataframe.columns if + any(substr in col for substr in valid_strs)] + pair_cols.insert(0, 'date') corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) return corr_dataframes @@ -1153,10 +1155,10 @@ class FreqaiDataKitchen: current_pair: str) -> DataFrame: """ Attach the existing corr_pair dataframes to the current pair dataframe before training - :params: - :dataframe: current pair strategy dataframe, indicators populated already - :corr_dataframes: dictionary of saved dataframes from earlier in the same candle - :current_pair: current pair to which we will attach corr pair dataframe + + :param dataframe: current pair strategy dataframe, indicators populated already + :param corr_dataframes: dictionary of saved dataframes from earlier in the same candle + :param current_pair: current pair to which we will attach corr pair dataframe :return: :dataframe: current pair dataframe of populated indicators, concatenated with corr_pairs ready for training @@ -1164,8 +1166,8 @@ class FreqaiDataKitchen: pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: - if current_pair not in pair: - dataframe = pd.concat([dataframe, corr_dataframes[pair]], axis=1) + if current_pair != pair: + dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date') return dataframe @@ -1186,15 +1188,15 @@ class FreqaiDataKitchen: :param base_dataframes: dict = dict containing the current pair dataframes (for user defined timeframes) :param metadata: dict = strategy furnished pair metadata - :returns: + :return: dataframe: DataFrame = dataframe containing populated indicators """ # for prediction dataframe creation, we let dataprovider handle everything in the strategy # so we create empty dictionaries, which allows us to pass None to # `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe. - tfs = self.freqai_config["feature_parameters"].get("include_timeframes") - pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) + tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") + pairs: List[str] = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) if not prediction_dataframe.empty: dataframe = prediction_dataframe.copy() for tf in tfs: @@ -1219,16 +1221,16 @@ class FreqaiDataKitchen: ) # ensure corr pairs are always last - for i in pairs: - if pair in i: + for corr_pair in pairs: + if pair == corr_pair: continue # dont repeat anything from whitelist for tf in tfs: if pairs and do_corr_pairs: dataframe = strategy.populate_any_indicators( - i, + corr_pair, dataframe.copy(), tf, - informative=corr_dataframes[i][tf] + informative=corr_dataframes[corr_pair][tf] ) self.get_unique_classes_from_labels(dataframe) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index d5591a03c..4f1ae029c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -72,6 +72,7 @@ class IFreqaiModel(ABC): self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] + self.corr_pairlist = self.ft_params.get("include_corr_pairlist", []) self.keras: bool = self.freqai_info.get("keras", False) if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 @@ -375,12 +376,8 @@ class IFreqaiModel(ABC): self.dd.return_null_values_to_strategy(dataframe, dk) return dk - if self.get_corr_dataframes: - self.corr_dataframes = dk.extract_corr_pair_columns_from_populated_indicators(dataframe) - self.get_corr_dataframes = False - else: - dataframe = dk.attach_corr_pair_columns( - dataframe, self.corr_dataframes, metadata["pair"]) + if self.corr_pairlist: + dataframe = self.cache_corr_pairlist_dfs(dataframe, dk) dk.find_labels(dataframe) @@ -687,7 +684,8 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 - self.get_corr_dataframes = True + if self.corr_pairlist: + self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): @@ -746,6 +744,29 @@ class IFreqaiModel(ABC): f'Best approximation queue: {best_queue}') return best_queue + def cache_corr_pairlist_dfs(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: + """ + Cache the corr_pairlist dfs to speed up performance for subsequent pairs during the + current candle. + :param dataframe: strategy fed dataframe + :param dk: datakitchen object for current asset + :return: dataframe to attach/extract cached corr_pair dfs to/from. + """ + + if self.get_corr_dataframes: + self.corr_dataframes = dk.extract_corr_pair_columns_from_populated_indicators(dataframe) + if not self.corr_dataframes: + logger.warning("Couldn't cache corr_pair dataframes for improved performance. " + "Consider ensuring that the full coin/stake, e.g. XYZ/USD, " + "is included in the column names when you are creating features " + "in `populate_any_indicators()`.") + self.get_corr_dataframes = not bool(self.corr_dataframes) + else: + dataframe = dk.attach_corr_pair_columns( + dataframe, self.corr_dataframes, dk.pair) + + return dataframe + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index d58d61025..fc39b0ab4 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -53,7 +53,7 @@ class FreqaiExampleStrategy(IStrategy): """ Function designed to automatically generate, name and merge features from user indicated timeframes in the configuration file. User controls the indicators - passed to the training/prediction by prepending indicators with `'%-' + coin ` + passed to the training/prediction by prepending indicators with `f'%-{pair}` (see convention below). I.e. user should not prepend any supporting metrics (e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the model. @@ -63,8 +63,6 @@ class FreqaiExampleStrategy(IStrategy): :param informative: the dataframe associated with the informative pair """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) @@ -72,36 +70,36 @@ class FreqaiExampleStrategy(IStrategy): for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) - informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) - informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) + informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) bollinger = qtpylib.bollinger_bands( qtpylib.typical_price(informative), window=t, stds=2.2 ) - informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{coin}bb_width-period_{t}"] = ( - informative[f"{coin}bb_upperband-period_{t}"] - - informative[f"{coin}bb_lowerband-period_{t}"] - ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + informative[f"%-{pair}bb_width-period_{t}"] = ( + informative[f"{pair}bb_upperband-period_{t}"] + - informative[f"{pair}bb_lowerband-period_{t}"] + ) / informative[f"{pair}bb_middleband-period_{t}"] + informative[f"%-{pair}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] ) - informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) - informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative[f"%-{pair}relative_volume-period_{t}"] = ( informative["volume"] / informative["volume"].rolling(t).mean() ) - informative[f"%-{coin}pct-change"] = informative["close"].pct_change() - informative[f"%-{coin}raw_volume"] = informative["volume"] - informative[f"%-{coin}raw_price"] = informative["close"] + informative[f"%-{pair}pct-change"] = informative["close"].pct_change() + informative[f"%-{pair}raw_volume"] = informative["volume"] + informative[f"%-{pair}raw_price"] = informative["close"] indicators = [col for col in informative if col.startswith("%")] # This loop duplicates and shifts all indicators to add a sense of recency to data From 391c3f56f722d45a25f843d914a0f4698f2e578a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 Oct 2022 13:28:01 +0100 Subject: [PATCH 3/6] Add typehint to corr_pairlist --- docs/freqai-feature-engineering.md | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index 8a1af69b7..26335d01b 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -2,7 +2,7 @@ ## Defining the features -Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`. +Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`. !!! Note Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If users elect *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode. diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 4f1ae029c..94683733c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -72,7 +72,7 @@ class IFreqaiModel(ABC): self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] - self.corr_pairlist = self.ft_params.get("include_corr_pairlist", []) + self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", []) self.keras: bool = self.freqai_info.get("keras", False) if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 From d59a7fa2f98db9fd0abde5165e852af19a656b82 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 30 Oct 2022 17:07:33 +0100 Subject: [PATCH 4/6] remove analysis_lock and realign example hybrid strat --- freqtrade/freqai/freqai_interface.py | 9 +++------ .../templates/FreqaiHybridExampleStrategy.py | 16 +++++++--------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 94683733c..a52a267a3 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -6,7 +6,6 @@ from abc import ABC, abstractmethod from collections import deque from datetime import datetime, timezone from pathlib import Path -from threading import Lock from typing import Any, Dict, List, Literal, Tuple import numpy as np @@ -84,7 +83,6 @@ class IFreqaiModel(ABC): self.pair_it_train = 0 self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) self.train_queue = self._set_train_queue() - self.analysis_lock = Lock() self.inference_time: float = 0 self.train_time: float = 0 self.begin_time: float = 0 @@ -563,10 +561,9 @@ class IFreqaiModel(ABC): data_load_timerange, pair, dk ) - with self.analysis_lock: - unfiltered_dataframe = dk.use_strategy_to_populate_indicators( - strategy, corr_dataframes, base_dataframes, pair - ) + unfiltered_dataframe = dk.use_strategy_to_populate_indicators( + strategy, corr_dataframes, base_dataframes, pair + ) unfiltered_dataframe = dk.slice_dataframe(new_trained_timerange, unfiltered_dataframe) diff --git a/freqtrade/templates/FreqaiHybridExampleStrategy.py b/freqtrade/templates/FreqaiHybridExampleStrategy.py index 593a6062b..26335956f 100644 --- a/freqtrade/templates/FreqaiHybridExampleStrategy.py +++ b/freqtrade/templates/FreqaiHybridExampleStrategy.py @@ -110,8 +110,6 @@ class FreqaiExampleHybridStrategy(IStrategy): :param informative: the dataframe associated with the informative pair """ - coin = pair.split('/')[0] - if informative is None: informative = self.dp.get_pair_dataframe(pair, tf) @@ -119,13 +117,13 @@ class FreqaiExampleHybridStrategy(IStrategy): for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) - informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) - informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) - informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) - informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) + informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + informative[f"%-{pair}relative_volume-period_{t}"] = ( informative["volume"] / informative["volume"].rolling(t).mean() ) From 7b880a969a1caa27fc56607064c0da0a8d03aa4a Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 31 Oct 2022 18:13:48 +0100 Subject: [PATCH 5/6] change elect to decide --- docs/freqai-feature-engineering.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index 26335d01b..3462955cc 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -5,7 +5,7 @@ Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`. !!! Note - Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If users elect *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode. + Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If you decide *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode. Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles." From 97df232ac6594baa0033de43de549567eaef9c92 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 31 Oct 2022 18:18:00 +0100 Subject: [PATCH 6/6] add a warning to __init__ for get_corr_dataframes --- freqtrade/freqai/freqai_interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 43a410c09..dcf902954 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -90,6 +90,8 @@ class IFreqaiModel(ABC): self.continual_learning = self.freqai_info.get('continual_learning', False) self.plot_features = self.ft_params.get("plot_feature_importances", 0) self.corr_dataframes: Dict[str, DataFrame] = {} + # get_corr_dataframes is controlling the caching of corr_dataframes + # for improved performance. Careful with this boolean. self.get_corr_dataframes: bool = True self._threads: List[threading.Thread] = []