From 5da60b718dee0f60772c081b7406af982555bb13 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 4 Feb 2023 13:47:11 +0100 Subject: [PATCH] pass metadata dictionary to feature_engineering_* and set_freqai_targets functions. Add doc --- docs/freqai-feature-engineering.md | 29 ++++++++++++++++++++ freqtrade/freqai/data_kitchen.py | 14 ++++++---- freqtrade/templates/FreqaiExampleStrategy.py | 16 +++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/docs/freqai-feature-engineering.md b/docs/freqai-feature-engineering.md index 6c8c5bb46..759c81289 100644 --- a/docs/freqai-feature-engineering.md +++ b/docs/freqai-feature-engineering.md @@ -28,6 +28,10 @@ It is advisable to start from the template `feature_engineering_*` functions in All features must be prepended with `%` to be recognized by FreqAI internals. + Access metadata such as the current pair/timeframe/period with: + + `metadata["pair"]` `metadata["tf"]` `metadata["period"]` + :param df: strategy dataframe which will receive the features :param period: period of the indicator - usage example: dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period) @@ -75,6 +79,10 @@ It is advisable to start from the template `feature_engineering_*` functions in Features defined here will *not* be automatically duplicated on user defined `indicator_periods_candles` + Access metadata such as the current pair/timeframe with: + + `metadata["pair"]` `metadata["tf"]` + All features must be prepended with `%` to be recognized by FreqAI internals. :param df: strategy dataframe which will receive the features @@ -98,6 +106,10 @@ It is advisable to start from the template `feature_engineering_*` functions in This function is a good place for any feature that should not be auto-expanded upon (e.g. day of the week). + Access metadata such as the current pair with: + + `metadata["pair"]` + All features must be prepended with `%` to be recognized by FreqAI internals. :param df: strategy dataframe which will receive the features @@ -113,6 +125,10 @@ It is advisable to start from the template `feature_engineering_*` functions in Required function to set the targets for the model. All targets must be prepended with `&` to be recognized by the FreqAI internals. + Access metadata such as the current pair with: + + `metadata["pair"]` + :param df: strategy dataframe which will receive the targets usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"] """ @@ -161,6 +177,19 @@ You can ask for each of the defined features to be included also for informative In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` $= 3 * 3 * 3 * 2 * 2 = 108$. + + ### Gain finer control over `feature_engineering_*` functions with `metadata` + + All `feature_engineering_*` and `set_freqai_targets()` functions are passed a `metadata` dictionary which contains information about the `pair`, `tf` (timeframe), and `period` that FreqAI is automating for feature building. As such, a user can use `metadata` inside `feature_engineering_*` functions as criteria for blocking/reserving features for certain timeframes, periods, pairs etc. + + ```py +def feature_engineering_expand_all(self, dataframe, period, **kwargs): + if metadata["tf"] == "1h": + dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period) +``` + +This will block `ta.ROC()` from being added to any timeframes other than `"1h"`. + ### Returning additional info from training Important metrics can be returned to the strategy at the end of each model training by assigning them to `dk.data['extra_returns_per_train']['my_new_value'] = XYZ` inside the custom prediction model class. diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 6f4a8c2b3..361a4146f 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1247,17 +1247,21 @@ class FreqaiDataKitchen: tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes") for tf in tfs: + metadata = {"pair": pair, "tf": tf} informative_df = self.get_pair_data_for_features( pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs) informative_copy = informative_df.copy() for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]: + metadata["period"] = t df_features = strategy.feature_engineering_expand_all( - informative_copy.copy(), t) + informative_copy.copy(), t, metadata=metadata) suffix = f"{t}" informative_df = self.merge_features(informative_df, df_features, tf, tf, suffix) - generic_df = strategy.feature_engineering_expand_basic(informative_copy.copy()) + metadata.pop("period") + generic_df = strategy.feature_engineering_expand_basic( + informative_copy.copy(), metadata=metadata) suffix = "gen" informative_df = self.merge_features(informative_df, generic_df, tf, tf, suffix) @@ -1326,8 +1330,8 @@ class FreqaiDataKitchen: "include_corr_pairlist", []) dataframe = self.populate_features(dataframe.copy(), pair, strategy, corr_dataframes, base_dataframes) - - dataframe = strategy.feature_engineering_standard(dataframe.copy()) + metadata = {"pair": pair} + dataframe = strategy.feature_engineering_standard(dataframe.copy(), metadata=metadata) # ensure corr pairs are always last for corr_pair in corr_pairs: if pair == corr_pair: @@ -1336,7 +1340,7 @@ class FreqaiDataKitchen: dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy, corr_dataframes, base_dataframes, True) - dataframe = strategy.set_freqai_targets(dataframe.copy()) + dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata) self.get_unique_classes_from_labels(dataframe) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 8e34d733e..a4e86fbf9 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -58,6 +58,10 @@ class FreqaiExampleStrategy(IStrategy): All features must be prepended with `%` to be recognized by FreqAI internals. + Access metadata such as the current pair/timeframe/period with: + + `metadata["pair"]` `metadata["tf"]` `metadata["period"]` + More details on how these config defined parameters accelerate feature engineering in the documentation at: @@ -114,6 +118,10 @@ class FreqaiExampleStrategy(IStrategy): All features must be prepended with `%` to be recognized by FreqAI internals. + Access metadata such as the current pair/timeframe with: + + `metadata["pair"]` `metadata["tf"]` + More details on how these config defined parameters accelerate feature engineering in the documentation at: @@ -144,6 +152,10 @@ class FreqaiExampleStrategy(IStrategy): All features must be prepended with `%` to be recognized by FreqAI internals. + Access metadata such as the current pair with: + + `metadata["pair"]` + More details about feature engineering available: https://www.freqtrade.io/en/latest/freqai-feature-engineering @@ -161,6 +173,10 @@ class FreqaiExampleStrategy(IStrategy): Required function to set the targets for the model. All targets must be prepended with `&` to be recognized by the FreqAI internals. + Access metadata such as the current pair with: + + `metadata["pair"]` + More details about feature engineering available: https://www.freqtrade.io/en/latest/freqai-feature-engineering