add spice_rack to FreqAI

docs/freqai.md

@@ -807,3 +807,85 @@ Code review, software architecture brainstorming:
 Beta testing and bug reporting:
 @bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm,
 Juha Nykänen @suikula, Wagner Costa @wagnercosta
+
+## Using the `spice_rack`
+
+<!-- Dont forget this section during the doc reorg! -->
+
+The `spice_rack` is aimed at users who do not wish to deal with setting up `FreqAI` confgs, but instead prefer to interact with `FreqAI` similar to a `talib` indicator. In this case, the user can instead simply add two keys to their config:
+
+```json
+    "freqai_spice_rack": true, 
+    "freqai_identifier": "spicey-id",
+```
+
+Which tells `FreqAI` to set up a pre-set `FreqAI` instance automatically under the hood with preset parameters. Now the user can access a suite of custom `FreqAI` supercharged indicators inside their strategy:
+
+```python
+        dataframe['dissimilarity_index'] = self.freqai.spice_rack(
+            'DI_values', dataframe, metadata, self)
+        dataframe['maxima'] = self.freqai.spice_rack(
+            '&s-maxima', dataframe, metadata, self)
+        dataframe['minima'] = self.freqai.spice_rack(
+            '&s-minima', dataframe, metadata, self)
+        self.freqai.close_spice_rack()  # user must close the spicerack
+```
+
+Users can then use these columns, concert with all their own additional indicators added to `populate_indicators` in their entry/exit criteria and strategy callback methods the same way as any typical indicator. For example:
+
+```python
+    def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
+
+        df.loc[
+            (
+                (df['dissimilarity_index'] < 1) &
+                (df['minima'] > 0.1)
+            ),
+            'enter_long'] = 1
+
+        df.loc[
+            (
+                (df['dissimilarity_index'] < 1) &
+                (df['maxima'] > 0.1)
+            ),
+            'enter_short'] = 1
+
+        return df
+
+    def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
+
+        df.loc[
+            (
+                (df['dissimilarity_index'] < 1) &
+                (df['maxima'] > 0.1) 
+            ),
+
+            'exit_long'] = 1
+
+        df.loc[
+            (
+
+                (df['dissimilarity_index'] < 1) &
+                (df['minima'] > 0.1)
+            ),
+            'exit_short'] = 1
+
+        return df
+```
+
+The user does need to ensure their `informative_pairs()` contains the following (users can add their own `informative_pair` needs to the bottom of this template):
+
+```python
+    def informative_pairs(self):
+        whitelist_pairs = self.dp.current_whitelist()
+        corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"]
+        informative_pairs = []
+        for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]:
+            for pair in whitelist_pairs:
+                informative_pairs.append((pair, tf))
+            for pair in corr_pairs:
+                if pair in whitelist_pairs:
+                    continue  # avoid duplication
+                informative_pairs.append((pair, tf))
+        return informative_pairs
+```

freqtrade/freqai/data_drawer.py

@@ -490,7 +490,7 @@ class FreqaiDataDrawer:
                 f"Unable to load model, ensure model exists at " f"{dk.data_path} "
             )
 
-        if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
+        if self.config["freqai"]["feature_parameters"].get("principal_component_analysis", False):
             dk.pca = cloudpickle.load(
                 open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
             )

freqtrade/freqai/data_kitchen.py

@@ -98,6 +98,7 @@ class FreqaiDataKitchen:
         self.train_dates: DataFrame = pd.DataFrame()
         self.unique_classes: Dict[str, list] = {}
         self.unique_class_list: list = []
+        self.spice_dataframe: DataFrame = None
 
     def set_paths(
         self,
@@ -1267,3 +1268,11 @@ class FreqaiDataKitchen:
                 f"Could not find backtesting prediction file at {path_to_predictionfile}"
             )
         return file_exists
+
+    def spice_extractor(self, indicator: str, dataframe: DataFrame) -> npt.NDArray:
+        if indicator in dataframe:
+            return np.array(dataframe[indicator])
+        else:
+            logger.warning(f'User asked spice_rack for {indicator}, '
+                           f'but it is not available. Returning 0s')
+            return np.zeros(len(dataframe.index))

freqtrade/freqai/freqai_interface.py

@@ -89,7 +89,7 @@ class IFreqaiModel(ABC):
         self.begin_time_train: float = 0
         self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
         self.continual_learning = self.freqai_info.get('continual_learning', False)
-
+        self.spice_rack_open: bool = False
         self._threads: List[threading.Thread] = []
         self._stop_event = threading.Event()
 
@@ -138,7 +138,7 @@ class IFreqaiModel(ABC):
             dk = self.start_backtesting(dataframe, metadata, self.dk)
 
         dataframe = dk.remove_features_from_df(dk.return_dataframe)
-        self.clean_up()
+        # self.clean_up()
         if self.live:
             self.inference_timer('stop')
         return dataframe
@@ -685,6 +685,18 @@ class IFreqaiModel(ABC):
 
         return init_model
 
+    def spice_rack(self, indicator: str, dataframe: DataFrame,
+                   metadata: dict, strategy: IStrategy) -> NDArray:
+        if not self.spice_rack_open:
+            dataframe = self.start(dataframe, metadata, strategy)
+            self.dk.spice_dataframe = dataframe
+            self.spice_rack_open = True
+            return self.dk.spice_extractor(indicator, dataframe)
+        else:
+            return self.dk.spice_extractor(indicator, self.dk.spice_dataframe)
+
+    def close_spice_rack(self):
+        self.spice_rack_open = False
     # Following methods which are overridden by user made prediction models.
     # See freqai/prediction_models/CatboostPredictionModel.py for an example.
 

freqtrade/freqai/spice_rack/lightgbm_config.json

@@ -0,0 +1,35 @@
+{
+
+    "freqai": {
+        "enabled": true,
+        "purge_old_models": true,
+        "train_period_days": 4,
+        "identifier": "uniqe-id",
+        "feature_parameters": {
+            "include_timeframes": [
+                "3m",
+                "15m",
+                "1h"
+            ],
+            "include_corr_pairlist": [
+                "BTC/USDT",
+                "ETH/USDT"
+            ],
+            "label_period_candles": 20,
+            "include_shifted_candles": 2,
+            "DI_threshold": 0.9,
+            "weight_factor": 0.9,
+            "indicator_periods_candles": [
+                10,
+                20
+            ]
+        },
+        "data_split_parameters": {
+            "test_size": 0,
+            "random_state": 1
+        },
+        "model_training_parameters": {
+            "n_estimators": 800
+        }
+    }
+}

freqtrade/freqai/utils.py

@@ -1,6 +1,13 @@
 import logging
 from datetime import datetime, timezone
 
+import numpy as np
+# for spice rack
+import pandas as pd
+import talib.abstract as ta
+from scipy.signal import argrelextrema
+from technical import qtpylib
+
 from freqtrade.configuration import TimeRange
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
@@ -8,6 +15,7 @@ from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
 from freqtrade.exchange.exchange import market_is_active
 from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
+from freqtrade.strategy import merge_informative_pair
 
 
 logger = logging.getLogger(__name__)
@@ -85,6 +93,83 @@ def get_required_data_timerange(
     return data_load_timerange
 
 
+def auto_populate_any_indicators(
+    self, pair, df, tf, informative=None, set_generalized_indicators=False
+):
+    """
+    This is a premade `populate_any_indicators()` function which is set in
+    the user strategy is they enable `freqai_spice_rack: true` in their
+    configuration file.
+    """
+
+    coin = pair.split('/')[0]
+
+    if informative is None:
+        informative = self.dp.get_pair_dataframe(pair, tf)
+
+    # first loop is automatically duplicating indicators for time periods
+    for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
+
+        t = int(t)
+        informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
+        informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
+        informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
+        informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
+        informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
+
+        bollinger = qtpylib.bollinger_bands(
+            qtpylib.typical_price(informative), window=t, stds=2.2
+        )
+        informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
+        informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
+        informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
+
+        informative[f"%-{coin}bb_width-period_{t}"] = (
+            informative[f"{coin}bb_upperband-period_{t}"]
+            - informative[f"{coin}bb_lowerband-period_{t}"]
+        ) / informative[f"{coin}bb_middleband-period_{t}"]
+        informative[f"%-{coin}close-bb_lower-period_{t}"] = (
+            informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
+        )
+
+        informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
+
+        informative[f"%-{coin}relative_volume-period_{t}"] = (
+            informative["volume"] / informative["volume"].rolling(t).mean()
+        )
+
+    informative[f"%-{coin}pct-change"] = informative["close"].pct_change()
+    informative[f"%-{coin}raw_volume"] = informative["volume"]
+    informative[f"%-{coin}raw_price"] = informative["close"]
+
+    indicators = [col for col in informative if col.startswith("%")]
+    # This loop duplicates and shifts all indicators to add a sense of recency to data
+    for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
+        if n == 0:
+            continue
+        informative_shift = informative[indicators].shift(n)
+        informative_shift = informative_shift.add_suffix("_shift-" + str(n))
+        informative = pd.concat((informative, informative_shift), axis=1)
+
+    df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
+    skip_columns = [
+        (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
+    ]
+    df = df.drop(columns=skip_columns)
+    if set_generalized_indicators:
+        df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
+        df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
+        df["&s-minima"] = 0
+        df["&s-maxima"] = 0
+        min_peaks = argrelextrema(df["close"].values, np.less, order=80)
+        max_peaks = argrelextrema(df["close"].values, np.greater, order=80)
+        for mp in min_peaks[0]:
+            df.at[mp, "&s-minima"] = 1
+        for mp in max_peaks[0]:
+            df.at[mp, "&s-maxima"] = 1
+
+    return df
+
 # Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
 # def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
 #     """

freqtrade/strategy/interface.py

@@ -5,7 +5,7 @@ This module defines the interface to apply for strategies
 import logging
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta, timezone
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import arrow
 from pandas import DataFrame
@@ -145,12 +145,27 @@ class IStrategy(ABC, HyperStrategyMixin):
                 self._ft_informative.append((informative_data, cls_method))
 
     def load_freqAI_model(self) -> None:
-        if self.config.get('freqai', {}).get('enabled', False):
+        spice_rack = self.config.get('freqai_spice_rack', False)
+        if self.config.get('freqai', {}).get('enabled', False) or spice_rack:
+            spice_rack = self.config.get('freqai_spice_rack', False)
+            if spice_rack:
+                self.config = self.setup_freqai_spice_rack(self.config)
             # Import here to avoid importing this if freqAI is disabled
             from freqtrade.freqai.utils import download_all_data_for_training
             from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
             self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
-            self.freqai_info = self.config["freqai"]
+
+            if spice_rack:
+                import types
+
+                from freqtrade.freqai.utils import auto_populate_any_indicators
+                self.populate_any_indicators = types.MethodType(  # type: ignore
+                        auto_populate_any_indicators, self)
+                # funcType = type(IStrategy.populate_any_indicators)
+                # self.populate_any_indicators = funcType(self.freqai.auto_populate_any_indicators,
+                #                                         self, self.populate_any_indicators)
+
+                self.freqai_info = self.config["freqai"]
 
             # download the desired data in dry/live
             if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE):
@@ -160,6 +175,7 @@ class IStrategy(ABC, HyperStrategyMixin):
                     "already on disk."
                 )
                 download_all_data_for_training(self.dp, self.config)
+
         else:
             # Gracious failures if freqAI is disabled but "start" is called.
             class DummyClass():
@@ -173,6 +189,17 @@ class IStrategy(ABC, HyperStrategyMixin):
 
             self.freqai = DummyClass()  # type: ignore
 
+    def setup_freqai_spice_rack(self, config: dict) -> Dict[str, Any]:
+        import json
+        from pathlib import Path
+        with open(Path('freqtrade') / 'freqai' / 'spice_rack'
+                  / 'lightgbm_config.json') as json_file:
+            freqai_config = json.load(json_file)
+            config['freqai'] = freqai_config['freqai']
+            config['freqai']['identifier'] = config['freqai_identifier']
+        config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'})
+        return config
+
     def ft_bot_start(self, **kwargs) -> None:
         """
         Strategy init - runs after dataprovider has been added.