From 3a9ec76c919d619756328b9f12ec10256b64e42c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 14 Aug 2022 17:19:50 +0200 Subject: [PATCH] Move "freqai.lock" to backend to simplify user interface --- docs/freqai.md | 184 +++++++++--------- freqtrade/freqai/data_kitchen.py | 36 ++-- freqtrade/templates/FreqaiExampleStrategy.py | 177 +++++++++-------- .../strategy/strats/freqai_test_classifier.py | 69 ++++--- .../strats/freqai_test_multimodel_strat.py | 105 +++++----- tests/strategy/strats/freqai_test_strat.py | 83 ++++---- 6 files changed, 323 insertions(+), 331 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index 6e81d9f28..6631c271e 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -177,8 +177,6 @@ The user interface is isolated to the typical config file. A typical FreqAI conf Features are added by the user inside the `populate_any_indicators()` method of the strategy by prepending indicators with `%` and labels are added by prepending `&`. There are some important components/structures that the user *must* include when building their feature set. -As shown below, `with self.freqai.lock:` must be used to ensure thread safety - especially when using third -party libraries for indicator construction such as TA-lib. Another structure to consider is the location of the labels at the bottom of the example function (below `if set_generalized_indicators:`). This is where the user will add single features and labels to their feature set to avoid duplication from various configuration parameters which multiply the feature set such as `include_timeframes`. @@ -203,69 +201,68 @@ various configuration parameters which multiply the feature set such as `include coint = pair.split('/')[0] - with self.freqai.lock: - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) - bollinger = qtpylib.bollinger_bands( - qtpylib.typical_price(informative), window=t, stds=2.2 - ) - informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(informative), window=t, stds=2.2 + ) + informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{coin}bb_width-period_{t}"] = ( - informative[f"{coin}bb_upperband-period_{t}"] - - informative[f"{coin}bb_lowerband-period_{t}"] - ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] - ) + informative[f"%-{coin}bb_width-period_{t}"] = ( + informative[f"{coin}bb_upperband-period_{t}"] + - informative[f"{coin}bb_lowerband-period_{t}"] + ) / informative[f"{coin}bb_middleband-period_{t}"] + informative[f"%-{coin}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + ) - informative[f"%-{coin}relative_volume-period_{t}"] = ( - informative["volume"] / informative["volume"].rolling(t).mean() - ) + informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative["volume"] / informative["volume"].rolling(t).mean() + ) - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times - if set_generalized_indicators: - df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 - df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # user adds targets here by prepending them with &- (see convention below) - # If user wishes to use multiple targets, a multioutput prediction model - # needs to be used such as templates/CatboostPredictionMultiModel.py - df["&-s_close"] = ( - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .mean() - / df["close"] - - 1 - ) + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) return df ``` @@ -430,48 +427,47 @@ The FreqAI strategy requires the user to include the following lines of code in coin = pair.split('/')[0] - with self.freqai.lock: - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times - if set_generalized_indicators: + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: - # user adds targets here by prepending them with &- (see convention below) - # If user wishes to use multiple targets, a multioutput prediction model - # needs to be used such as templates/CatboostPredictionMultiModel.py - df["&-s_close"] = ( - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .mean() - / df["close"] - - 1 - ) + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) return df diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 2ba8c5ecf..2378dcdd8 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -961,23 +961,25 @@ class FreqaiDataKitchen: for tf in tfs: if tf == tfs[-1]: sgi = True # doing this last allows user to use all tf raw prices in labels - dataframe = strategy.populate_any_indicators( - pair, - dataframe.copy(), - tf, - informative=base_dataframes[tf], - set_generalized_indicators=sgi - ) - if pairs: - for i in pairs: - if pair in i: - continue # dont repeat anything from whitelist - dataframe = strategy.populate_any_indicators( - i, - dataframe.copy(), - tf, - informative=corr_dataframes[i][tf] - ) + with strategy.freqai.lock: + dataframe = strategy.populate_any_indicators( + pair, + dataframe.copy(), + tf, + informative=base_dataframes[tf], + set_generalized_indicators=sgi + ) + with strategy.freqai.lock: + if pairs: + for i in pairs: + if pair in i: + continue # dont repeat anything from whitelist + dataframe = strategy.populate_any_indicators( + i, + dataframe.copy(), + tf, + informative=corr_dataframes[i][tf] + ) self.get_unique_classes_from_labels(dataframe) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index d8584d5f9..5810e7881 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -82,99 +82,98 @@ class FreqaiExampleStrategy(IStrategy): coin = pair.split('/')[0] - with self.freqai.lock: - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) - informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) - informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - bollinger = qtpylib.bollinger_bands( - qtpylib.typical_price(informative), window=t, stds=2.2 - ) - informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] - informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] - informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(informative), window=t, stds=2.2 + ) + informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] - informative[f"%-{coin}bb_width-period_{t}"] = ( - informative[f"{coin}bb_upperband-period_{t}"] - - informative[f"{coin}bb_lowerband-period_{t}"] - ) / informative[f"{coin}bb_middleband-period_{t}"] - informative[f"%-{coin}close-bb_lower-period_{t}"] = ( - informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] - ) + informative[f"%-{coin}bb_width-period_{t}"] = ( + informative[f"{coin}bb_upperband-period_{t}"] + - informative[f"{coin}bb_lowerband-period_{t}"] + ) / informative[f"{coin}bb_middleband-period_{t}"] + informative[f"%-{coin}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + ) - informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) - informative[f"%-{coin}relative_volume-period_{t}"] = ( - informative["volume"] / informative["volume"].rolling(t).mean() - ) + informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative["volume"] / informative["volume"].rolling(t).mean() + ) - informative[f"%-{coin}pct-change"] = informative["close"].pct_change() - informative[f"%-{coin}raw_volume"] = informative["volume"] - informative[f"%-{coin}raw_price"] = informative["close"] + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times - if set_generalized_indicators: - df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 - df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # user adds targets here by prepending them with &- (see convention below) - df["&-s_close"] = ( - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .mean() - / df["close"] - - 1 - ) + # user adds targets here by prepending them with &- (see convention below) + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) - # Classifiers are typically set up with strings as targets: - # df['&s-up_or_down'] = np.where( df["close"].shift(-100) > - # df["close"], 'up', 'down') + # Classifiers are typically set up with strings as targets: + # df['&s-up_or_down'] = np.where( df["close"].shift(-100) > + # df["close"], 'up', 'down') - # If user wishes to use multiple targets, they can add more by - # appending more columns with '&'. User should keep in mind that multi targets - # requires a multioutput prediction model such as - # templates/CatboostPredictionMultiModel.py, + # If user wishes to use multiple targets, they can add more by + # appending more columns with '&'. User should keep in mind that multi targets + # requires a multioutput prediction model such as + # templates/CatboostPredictionMultiModel.py, - # df["&-s_range"] = ( - # df["close"] - # .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - # .max() - # - - # df["close"] - # .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - # .min() - # ) + # df["&-s_range"] = ( + # df["close"] + # .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + # .max() + # - + # df["close"] + # .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + # .min() + # ) return df @@ -252,12 +251,11 @@ class FreqaiExampleStrategy(IStrategy): "prediction" + entry_tag not in pair_dict[pair] or pair_dict[pair]['extras']["prediction" + entry_tag] == 0 ): - with self.freqai.lock: - pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"]) - if not follow_mode: - self.freqai.dd.save_drawer_to_disk() - else: - self.freqai.dd.save_follower_dict_to_disk() + pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"]) + if not follow_mode: + self.freqai.dd.save_drawer_to_disk() + else: + self.freqai.dd.save_follower_dict_to_disk() roi_price = pair_dict[pair]['extras']["prediction" + entry_tag] roi_time = self.max_roi_time_long.value @@ -296,12 +294,11 @@ class FreqaiExampleStrategy(IStrategy): else: pair_dict = self.freqai.dd.follower_dict - with self.freqai.lock: - pair_dict[pair]['extras']["prediction" + entry_tag] = 0 - if not follow_mode: - self.freqai.dd.save_drawer_to_disk() - else: - self.freqai.dd.save_follower_dict_to_disk() + pair_dict[pair]['extras']["prediction" + entry_tag] = 0 + if not follow_mode: + self.freqai.dd.save_drawer_to_disk() + else: + self.freqai.dd.save_follower_dict_to_disk() return True diff --git a/tests/strategy/strats/freqai_test_classifier.py b/tests/strategy/strats/freqai_test_classifier.py index 0a2ce793a..a1e8cb6bf 100644 --- a/tests/strategy/strats/freqai_test_classifier.py +++ b/tests/strategy/strats/freqai_test_classifier.py @@ -63,48 +63,47 @@ class freqai_test_classifier(IStrategy): coin = pair.split('/')[0] - with self.freqai.lock: - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) - informative[f"%-{coin}pct-change"] = informative["close"].pct_change() - informative[f"%-{coin}raw_volume"] = informative["volume"] - informative[f"%-{coin}raw_price"] = informative["close"] + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times - if set_generalized_indicators: - df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 - df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # user adds targets here by prepending them with &- (see convention below) - # If user wishes to use multiple targets, a multioutput prediction model - # needs to be used such as templates/CatboostPredictionMultiModel.py - df['&s-up_or_down'] = np.where(df["close"].shift(-100) > df["close"], 'up', 'down') + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df['&s-up_or_down'] = np.where(df["close"].shift(-100) > df["close"], 'up', 'down') return df diff --git a/tests/strategy/strats/freqai_test_multimodel_strat.py b/tests/strategy/strats/freqai_test_multimodel_strat.py index 4f8877f14..cd3327da9 100644 --- a/tests/strategy/strats/freqai_test_multimodel_strat.py +++ b/tests/strategy/strats/freqai_test_multimodel_strat.py @@ -62,67 +62,66 @@ class freqai_test_multimodel_strat(IStrategy): coin = pair.split('/')[0] - with self.freqai.lock: - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) - informative[f"%-{coin}pct-change"] = informative["close"].pct_change() - informative[f"%-{coin}raw_volume"] = informative["volume"] - informative[f"%-{coin}raw_price"] = informative["close"] + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times - if set_generalized_indicators: - df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 - df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # user adds targets here by prepending them with &- (see convention below) - # If user wishes to use multiple targets, a multioutput prediction model - # needs to be used such as templates/CatboostPredictionMultiModel.py - df["&-s_close"] = ( - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .mean() - / df["close"] - - 1 - ) + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) - df["&-s_range"] = ( - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .max() - - - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .min() - ) + df["&-s_range"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .max() + - + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .min() + ) return df diff --git a/tests/strategy/strats/freqai_test_strat.py b/tests/strategy/strats/freqai_test_strat.py index e239a30b0..792a3952f 100644 --- a/tests/strategy/strats/freqai_test_strat.py +++ b/tests/strategy/strats/freqai_test_strat.py @@ -62,55 +62,54 @@ class freqai_test_strat(IStrategy): coin = pair.split('/')[0] - with self.freqai.lock: - if informative is None: - informative = self.dp.get_pair_dataframe(pair, tf) + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) - # first loop is automatically duplicating indicators for time periods - for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: - t = int(t) - informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) - informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t) - informative[f"%-{coin}pct-change"] = informative["close"].pct_change() - informative[f"%-{coin}raw_volume"] = informative["volume"] - informative[f"%-{coin}raw_price"] = informative["close"] + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] - indicators = [col for col in informative if col.startswith("%")] - # This loop duplicates and shifts all indicators to add a sense of recency to data - for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): - if n == 0: - continue - informative_shift = informative[indicators].shift(n) - informative_shift = informative_shift.add_suffix("_shift-" + str(n)) - informative = pd.concat((informative, informative_shift), axis=1) + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) - df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) - skip_columns = [ - (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] - ] - df = df.drop(columns=skip_columns) + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) - # Add generalized indicators here (because in live, it will call this - # function to populate indicators during training). Notice how we ensure not to - # add them multiple times - if set_generalized_indicators: - df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 - df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + # Add generalized indicators here (because in live, it will call this + # function to populate indicators during training). Notice how we ensure not to + # add them multiple times + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 - # user adds targets here by prepending them with &- (see convention below) - # If user wishes to use multiple targets, a multioutput prediction model - # needs to be used such as templates/CatboostPredictionMultiModel.py - df["&-s_close"] = ( - df["close"] - .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) - .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) - .mean() - / df["close"] - - 1 - ) + # user adds targets here by prepending them with &- (see convention below) + # If user wishes to use multiple targets, a multioutput prediction model + # needs to be used such as templates/CatboostPredictionMultiModel.py + df["&-s_close"] = ( + df["close"] + .shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) + .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) + .mean() + / df["close"] + - 1 + ) return df