add spice_rack to FreqAI

freqtrade/freqai/data_drawer.py

@@ -490,7 +490,7 @@ class FreqaiDataDrawer:
                 f"Unable to load model, ensure model exists at " f"{dk.data_path} "
             )
 
-        if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
+        if self.config["freqai"]["feature_parameters"].get("principal_component_analysis", False):
             dk.pca = cloudpickle.load(
                 open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
             )

freqtrade/freqai/data_kitchen.py

@@ -98,6 +98,7 @@ class FreqaiDataKitchen:
         self.train_dates: DataFrame = pd.DataFrame()
         self.unique_classes: Dict[str, list] = {}
         self.unique_class_list: list = []
+        self.spice_dataframe: DataFrame = None
 
     def set_paths(
         self,
@@ -1267,3 +1268,11 @@ class FreqaiDataKitchen:
                 f"Could not find backtesting prediction file at {path_to_predictionfile}"
             )
         return file_exists
+
+    def spice_extractor(self, indicator: str, dataframe: DataFrame) -> npt.NDArray:
+        if indicator in dataframe:
+            return np.array(dataframe[indicator])
+        else:
+            logger.warning(f'User asked spice_rack for {indicator}, '
+                           f'but it is not available. Returning 0s')
+            return np.zeros(len(dataframe.index))

freqtrade/freqai/freqai_interface.py

@@ -89,7 +89,7 @@ class IFreqaiModel(ABC):
         self.begin_time_train: float = 0
         self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
         self.continual_learning = self.freqai_info.get('continual_learning', False)
-
+        self.spice_rack_open: bool = False
         self._threads: List[threading.Thread] = []
         self._stop_event = threading.Event()
 
@@ -138,7 +138,7 @@ class IFreqaiModel(ABC):
             dk = self.start_backtesting(dataframe, metadata, self.dk)
 
         dataframe = dk.remove_features_from_df(dk.return_dataframe)
-        self.clean_up()
+        # self.clean_up()
         if self.live:
             self.inference_timer('stop')
         return dataframe
@@ -685,6 +685,18 @@ class IFreqaiModel(ABC):
 
         return init_model
 
+    def spice_rack(self, indicator: str, dataframe: DataFrame,
+                   metadata: dict, strategy: IStrategy) -> NDArray:
+        if not self.spice_rack_open:
+            dataframe = self.start(dataframe, metadata, strategy)
+            self.dk.spice_dataframe = dataframe
+            self.spice_rack_open = True
+            return self.dk.spice_extractor(indicator, dataframe)
+        else:
+            return self.dk.spice_extractor(indicator, self.dk.spice_dataframe)
+
+    def close_spice_rack(self):
+        self.spice_rack_open = False
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.
 

freqtrade/freqai/spice_rack/lightgbm_config.json

@@ -0,0 +1,35 @@
+{
+
+    "freqai": {
+        "enabled": true,
+        "purge_old_models": true,
+        "train_period_days": 4,
+        "identifier": "uniqe-id",
+        "feature_parameters": {
+            "include_timeframes": [
+                "3m",
+                "15m",
+                "1h"
+            ],
+            "include_corr_pairlist": [
+                "BTC/USDT",
+                "ETH/USDT"
+            ],
+            "label_period_candles": 20,
+            "include_shifted_candles": 2,
+            "DI_threshold": 0.9,
+            "weight_factor": 0.9,
+            "indicator_periods_candles": [
+                10,
+                20
+            ]
+        },
+        "data_split_parameters": {
+            "test_size": 0,
+            "random_state": 1
+        },
+        "model_training_parameters": {
+            "n_estimators": 800
+        }
+    }
+}

freqtrade/freqai/utils.py

@@ -1,6 +1,13 @@
 import logging
 from datetime import datetime, timezone
 
+import numpy as np
+# for spice rack
+import pandas as pd
+import talib.abstract as ta
+from scipy.signal import argrelextrema
+from technical import qtpylib
+
 from freqtrade.configuration import TimeRange
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
@@ -8,6 +15,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.exchange import market_is_active
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
+from freqtrade.strategy import merge_informative_pair
 
 
 logger = logging.getLogger(__name__)
@@ -85,6 +93,83 @@ def get_required_data_timerange(
     return data_load_timerange
 
 
+def auto_populate_any_indicators(
+    self, pair, df, tf, informative=None, set_generalized_indicators=False
+):
+    """
+    This is a premade `populate_any_indicators()` function which is set in
+    the user strategy is they enable `freqai_spice_rack: true` in their
+    configuration file.
+    """
+
+    coin = pair.split('/')[0]
+
+    if informative is None:
+        informative = self.dp.get_pair_dataframe(pair, tf)
+
+    # first loop is automatically duplicating indicators for time periods
+    for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
+
+        t = int(t)
+        informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
+        informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
+        informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
+        informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
+        informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
+
+        bollinger = qtpylib.bollinger_bands(
+            qtpylib.typical_price(informative), window=t, stds=2.2
+        )
+        informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
+        informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
+        informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
+
+        informative[f"%-{coin}bb_width-period_{t}"] = (
+            informative[f"{coin}bb_upperband-period_{t}"]
+            - informative[f"{coin}bb_lowerband-period_{t}"]
+        ) / informative[f"{coin}bb_middleband-period_{t}"]
+        informative[f"%-{coin}close-bb_lower-period_{t}"] = (
+            informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
+        )
+
+        informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
+
+        informative[f"%-{coin}relative_volume-period_{t}"] = (
+            informative["volume"] / informative["volume"].rolling(t).mean()
+        )
+
+    informative[f"%-{coin}pct-change"] = informative["close"].pct_change()
+    informative[f"%-{coin}raw_volume"] = informative["volume"]
+    informative[f"%-{coin}raw_price"] = informative["close"]
+
+    indicators = [col for col in informative if col.startswith("%")]
+    # This loop duplicates and shifts all indicators to add a sense of recency to data
+    for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
+        if n == 0:
+            continue
+        informative_shift = informative[indicators].shift(n)
+        informative_shift = informative_shift.add_suffix("_shift-" + str(n))
+        informative = pd.concat((informative, informative_shift), axis=1)
+
+    df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
+    skip_columns = [
+        (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
+    ]
+    df = df.drop(columns=skip_columns)
+    if set_generalized_indicators:
+        df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
+        df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
+        df["&s-minima"] = 0
+        df["&s-maxima"] = 0
+        min_peaks = argrelextrema(df["close"].values, np.less, order=80)
+        max_peaks = argrelextrema(df["close"].values, np.greater, order=80)
+        for mp in min_peaks[0]:
+            df.at[mp, "&s-minima"] = 1
+        for mp in max_peaks[0]:
+            df.at[mp, "&s-maxima"] = 1
+
+    return df
+
 # Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
 # def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
 #     """