From b2094900091969cae209fe6b450f23ec7337e763 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 15 Sep 2022 23:26:43 +0200 Subject: [PATCH] add spice_rack to FreqAI --- docs/freqai.md | 82 ++++++++++++++++++ freqtrade/freqai/data_drawer.py | 2 +- freqtrade/freqai/data_kitchen.py | 9 ++ freqtrade/freqai/freqai_interface.py | 16 +++- .../freqai/spice_rack/lightgbm_config.json | 35 ++++++++ freqtrade/freqai/utils.py | 85 +++++++++++++++++++ freqtrade/strategy/interface.py | 33 ++++++- 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 freqtrade/freqai/spice_rack/lightgbm_config.json diff --git a/docs/freqai.md b/docs/freqai.md index 5f523f58a..1493ac1b4 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -807,3 +807,85 @@ Code review, software architecture brainstorming: Beta testing and bug reporting: @bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm, Juha Nykänen @suikula, Wagner Costa @wagnercosta + +## Using the `spice_rack` + + + +The `spice_rack` is aimed at users who do not wish to deal with setting up `FreqAI` confgs, but instead prefer to interact with `FreqAI` similar to a `talib` indicator. In this case, the user can instead simply add two keys to their config: + +```json + "freqai_spice_rack": true, + "freqai_identifier": "spicey-id", +``` + +Which tells `FreqAI` to set up a pre-set `FreqAI` instance automatically under the hood with preset parameters. Now the user can access a suite of custom `FreqAI` supercharged indicators inside their strategy: + +```python + dataframe['dissimilarity_index'] = self.freqai.spice_rack( + 'DI_values', dataframe, metadata, self) + dataframe['maxima'] = self.freqai.spice_rack( + '&s-maxima', dataframe, metadata, self) + dataframe['minima'] = self.freqai.spice_rack( + '&s-minima', dataframe, metadata, self) + self.freqai.close_spice_rack() # user must close the spicerack +``` + +Users can then use these columns, concert with all their own additional indicators added to `populate_indicators` in their entry/exit criteria and strategy callback methods the same way as any typical indicator. For example: + +```python + def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + df.loc[ + ( + (df['dissimilarity_index'] < 1) & + (df['minima'] > 0.1) + ), + 'enter_long'] = 1 + + df.loc[ + ( + (df['dissimilarity_index'] < 1) & + (df['maxima'] > 0.1) + ), + 'enter_short'] = 1 + + return df + + def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: + + df.loc[ + ( + (df['dissimilarity_index'] < 1) & + (df['maxima'] > 0.1) + ), + + 'exit_long'] = 1 + + df.loc[ + ( + + (df['dissimilarity_index'] < 1) & + (df['minima'] > 0.1) + ), + 'exit_short'] = 1 + + return df +``` + +The user does need to ensure their `informative_pairs()` contains the following (users can add their own `informative_pair` needs to the bottom of this template): + +```python + def informative_pairs(self): + whitelist_pairs = self.dp.current_whitelist() + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in whitelist_pairs: + informative_pairs.append((pair, tf)) + for pair in corr_pairs: + if pair in whitelist_pairs: + continue # avoid duplication + informative_pairs.append((pair, tf)) + return informative_pairs +``` diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 9eeabef8f..39abcb7a6 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -490,7 +490,7 @@ class FreqaiDataDrawer: f"Unable to load model, ensure model exists at " f"{dk.data_path} " ) - if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]: + if self.config["freqai"]["feature_parameters"].get("principal_component_analysis", False): dk.pca = cloudpickle.load( open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb") ) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index e96a945eb..de8a5c705 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -98,6 +98,7 @@ class FreqaiDataKitchen: self.train_dates: DataFrame = pd.DataFrame() self.unique_classes: Dict[str, list] = {} self.unique_class_list: list = [] + self.spice_dataframe: DataFrame = None def set_paths( self, @@ -1267,3 +1268,11 @@ class FreqaiDataKitchen: f"Could not find backtesting prediction file at {path_to_predictionfile}" ) return file_exists + + def spice_extractor(self, indicator: str, dataframe: DataFrame) -> npt.NDArray: + if indicator in dataframe: + return np.array(dataframe[indicator]) + else: + logger.warning(f'User asked spice_rack for {indicator}, ' + f'but it is not available. Returning 0s') + return np.zeros(len(dataframe.index)) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 78931bed4..5981db92f 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -89,7 +89,7 @@ class IFreqaiModel(ABC): self.begin_time_train: float = 0 self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) self.continual_learning = self.freqai_info.get('continual_learning', False) - + self.spice_rack_open: bool = False self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() @@ -138,7 +138,7 @@ class IFreqaiModel(ABC): dk = self.start_backtesting(dataframe, metadata, self.dk) dataframe = dk.remove_features_from_df(dk.return_dataframe) - self.clean_up() + # self.clean_up() if self.live: self.inference_timer('stop') return dataframe @@ -685,6 +685,18 @@ class IFreqaiModel(ABC): return init_model + def spice_rack(self, indicator: str, dataframe: DataFrame, + metadata: dict, strategy: IStrategy) -> NDArray: + if not self.spice_rack_open: + dataframe = self.start(dataframe, metadata, strategy) + self.dk.spice_dataframe = dataframe + self.spice_rack_open = True + return self.dk.spice_extractor(indicator, dataframe) + else: + return self.dk.spice_extractor(indicator, self.dk.spice_dataframe) + + def close_spice_rack(self): + self.spice_rack_open = False # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/freqai/spice_rack/lightgbm_config.json b/freqtrade/freqai/spice_rack/lightgbm_config.json new file mode 100644 index 000000000..d9e644c92 --- /dev/null +++ b/freqtrade/freqai/spice_rack/lightgbm_config.json @@ -0,0 +1,35 @@ +{ + + "freqai": { + "enabled": true, + "purge_old_models": true, + "train_period_days": 4, + "identifier": "uniqe-id", + "feature_parameters": { + "include_timeframes": [ + "3m", + "15m", + "1h" + ], + "include_corr_pairlist": [ + "BTC/USDT", + "ETH/USDT" + ], + "label_period_candles": 20, + "include_shifted_candles": 2, + "DI_threshold": 0.9, + "weight_factor": 0.9, + "indicator_periods_candles": [ + 10, + 20 + ] + }, + "data_split_parameters": { + "test_size": 0, + "random_state": 1 + }, + "model_training_parameters": { + "n_estimators": 800 + } + } +} diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 6a70f050f..e21321f35 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,6 +1,13 @@ import logging from datetime import datetime, timezone +import numpy as np +# for spice rack +import pandas as pd +import talib.abstract as ta +from scipy.signal import argrelextrema +from technical import qtpylib + from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data @@ -8,6 +15,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.exchange import market_is_active from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist +from freqtrade.strategy import merge_informative_pair logger = logging.getLogger(__name__) @@ -85,6 +93,83 @@ def get_required_data_timerange( return data_load_timerange +def auto_populate_any_indicators( + self, pair, df, tf, informative=None, set_generalized_indicators=False +): + """ + This is a premade `populate_any_indicators()` function which is set in + the user strategy is they enable `freqai_spice_rack: true` in their + configuration file. + """ + + coin = pair.split('/')[0] + + if informative is None: + informative = self.dp.get_pair_dataframe(pair, tf) + + # first loop is automatically duplicating indicators for time periods + for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: + + t = int(t) + informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) + informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) + informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) + informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) + informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) + + bollinger = qtpylib.bollinger_bands( + qtpylib.typical_price(informative), window=t, stds=2.2 + ) + informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] + informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] + informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] + + informative[f"%-{coin}bb_width-period_{t}"] = ( + informative[f"{coin}bb_upperband-period_{t}"] + - informative[f"{coin}bb_lowerband-period_{t}"] + ) / informative[f"{coin}bb_middleband-period_{t}"] + informative[f"%-{coin}close-bb_lower-period_{t}"] = ( + informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] + ) + + informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) + + informative[f"%-{coin}relative_volume-period_{t}"] = ( + informative["volume"] / informative["volume"].rolling(t).mean() + ) + + informative[f"%-{coin}pct-change"] = informative["close"].pct_change() + informative[f"%-{coin}raw_volume"] = informative["volume"] + informative[f"%-{coin}raw_price"] = informative["close"] + + indicators = [col for col in informative if col.startswith("%")] + # This loop duplicates and shifts all indicators to add a sense of recency to data + for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): + if n == 0: + continue + informative_shift = informative[indicators].shift(n) + informative_shift = informative_shift.add_suffix("_shift-" + str(n)) + informative = pd.concat((informative, informative_shift), axis=1) + + df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) + skip_columns = [ + (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] + ] + df = df.drop(columns=skip_columns) + if set_generalized_indicators: + df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 + df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 + df["&s-minima"] = 0 + df["&s-maxima"] = 0 + min_peaks = argrelextrema(df["close"].values, np.less, order=80) + max_peaks = argrelextrema(df["close"].values, np.greater, order=80) + for mp in min_peaks[0]: + df.at[mp, "&s-minima"] = 1 + for mp in max_peaks[0]: + df.at[mp, "&s-maxima"] = 1 + + return df + # Keep below for when we wish to download heterogeneously lengthed data for FreqAI. # def download_all_data_for_training(dp: DataProvider, config: dict) -> None: # """ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9401ebebe..4d0837d8f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -5,7 +5,7 @@ This module defines the interface to apply for strategies import logging from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame @@ -145,12 +145,27 @@ class IStrategy(ABC, HyperStrategyMixin): self._ft_informative.append((informative_data, cls_method)) def load_freqAI_model(self) -> None: - if self.config.get('freqai', {}).get('enabled', False): + spice_rack = self.config.get('freqai_spice_rack', False) + if self.config.get('freqai', {}).get('enabled', False) or spice_rack: + spice_rack = self.config.get('freqai_spice_rack', False) + if spice_rack: + self.config = self.setup_freqai_spice_rack(self.config) # Import here to avoid importing this if freqAI is disabled from freqtrade.freqai.utils import download_all_data_for_training from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) - self.freqai_info = self.config["freqai"] + + if spice_rack: + import types + + from freqtrade.freqai.utils import auto_populate_any_indicators + self.populate_any_indicators = types.MethodType( # type: ignore + auto_populate_any_indicators, self) + # funcType = type(IStrategy.populate_any_indicators) + # self.populate_any_indicators = funcType(self.freqai.auto_populate_any_indicators, + # self, self.populate_any_indicators) + + self.freqai_info = self.config["freqai"] # download the desired data in dry/live if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): @@ -160,6 +175,7 @@ class IStrategy(ABC, HyperStrategyMixin): "already on disk." ) download_all_data_for_training(self.dp, self.config) + else: # Gracious failures if freqAI is disabled but "start" is called. class DummyClass(): @@ -173,6 +189,17 @@ class IStrategy(ABC, HyperStrategyMixin): self.freqai = DummyClass() # type: ignore + def setup_freqai_spice_rack(self, config: dict) -> Dict[str, Any]: + import json + from pathlib import Path + with open(Path('freqtrade') / 'freqai' / 'spice_rack' + / 'lightgbm_config.json') as json_file: + freqai_config = json.load(json_file) + config['freqai'] = freqai_config['freqai'] + config['freqai']['identifier'] = config['freqai_identifier'] + config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'}) + return config + def ft_bot_start(self, **kwargs) -> None: """ Strategy init - runs after dataprovider has been added.