adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add spice_rack to FreqAI

Browse Source
This commit is contained in:
robcaulk 2022-09-15 23:26:43 +02:00
parent 075748b21a
commit b209490009
7 changed files with 256 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
docs/freqai.md
View File

@ -807,3 +807,85 @@ Code review, software architecture brainstorming:
Beta testing and bug reporting:
@bloodhunter4rc, Salah Lamkadem @ikonx, @ken11o2, @longyu, @paranoidandy, @smidelis, @smarm,
Juha Nykänen @suikula, Wagner Costa @wagnercosta
## Using the `spice_rack`
<!-- Dont forget this section during the doc reorg! -->
The `spice_rack` is aimed at users who do not wish to deal with setting up `FreqAI` confgs, but instead prefer to interact with `FreqAI` similar to a `talib` indicator. In this case, the user can instead simply add two keys to their config:
```json
"freqai_spice_rack": true,
"freqai_identifier": "spicey-id",
```
Which tells `FreqAI` to set up a pre-set `FreqAI` instance automatically under the hood with preset parameters. Now the user can access a suite of custom `FreqAI` supercharged indicators inside their strategy:
```python
dataframe['dissimilarity_index'] = self.freqai.spice_rack(
'DI_values', dataframe, metadata, self)
dataframe['maxima'] = self.freqai.spice_rack(
'&s-maxima', dataframe, metadata, self)
dataframe['minima'] = self.freqai.spice_rack(
'&s-minima', dataframe, metadata, self)
self.freqai.close_spice_rack() # user must close the spicerack
```
Users can then use these columns, concert with all their own additional indicators added to `populate_indicators` in their entry/exit criteria and strategy callback methods the same way as any typical indicator. For example:
```python
def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
df.loc[
(
(df['dissimilarity_index'] < 1) &
(df['minima'] > 0.1)
),
'enter_long'] = 1
df.loc[
(
(df['dissimilarity_index'] < 1) &
(df['maxima'] > 0.1)
),
'enter_short'] = 1
return df
def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
df.loc[
(
(df['dissimilarity_index'] < 1) &
(df['maxima'] > 0.1)
),
'exit_long'] = 1
df.loc[
(
(df['dissimilarity_index'] < 1) &
(df['minima'] > 0.1)
),
'exit_short'] = 1
return df
```
The user does need to ensure their `informative_pairs()` contains the following (users can add their own `informative_pair` needs to the bottom of this template):
```python
def informative_pairs(self):
whitelist_pairs = self.dp.current_whitelist()
corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"]
informative_pairs = []
for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]:
for pair in whitelist_pairs:
informative_pairs.append((pair, tf))
for pair in corr_pairs:
if pair in whitelist_pairs:
continue # avoid duplication
informative_pairs.append((pair, tf))
return informative_pairs
```

2
freqtrade/freqai/data_drawer.py
View File

@ -490,7 +490,7 @@ class FreqaiDataDrawer:
f"Unable to load model, ensure model exists at " f"{dk.data_path} "
)
if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
if self.config["freqai"]["feature_parameters"].get("principal_component_analysis", False):
dk.pca = cloudpickle.load(
open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
)

9
freqtrade/freqai/data_kitchen.py
View File

@ -98,6 +98,7 @@ class FreqaiDataKitchen:
self.train_dates: DataFrame = pd.DataFrame()
self.unique_classes: Dict[str, list] = {}
self.unique_class_list: list = []
self.spice_dataframe: DataFrame = None
def set_paths(
self,
@ -1267,3 +1268,11 @@ class FreqaiDataKitchen:
f"Could not find backtesting prediction file at {path_to_predictionfile}"
)
return file_exists
def spice_extractor(self, indicator: str, dataframe: DataFrame) -> npt.NDArray:
if indicator in dataframe:
return np.array(dataframe[indicator])
else:
logger.warning(f'User asked spice_rack for {indicator}, '
f'but it is not available. Returning 0s')
return np.zeros(len(dataframe.index))

16
freqtrade/freqai/freqai_interface.py
View File

@ -89,7 +89,7 @@ class IFreqaiModel(ABC):
self.begin_time_train: float = 0
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
self.continual_learning = self.freqai_info.get('continual_learning', False)
self.spice_rack_open: bool = False
self._threads: List[threading.Thread] = []
self._stop_event = threading.Event()
@ -138,7 +138,7 @@ class IFreqaiModel(ABC):
dk = self.start_backtesting(dataframe, metadata, self.dk)
dataframe = dk.remove_features_from_df(dk.return_dataframe)
self.clean_up()
# self.clean_up()
if self.live:
self.inference_timer('stop')
return dataframe
@ -685,6 +685,18 @@ class IFreqaiModel(ABC):
return init_model
def spice_rack(self, indicator: str, dataframe: DataFrame,
metadata: dict, strategy: IStrategy) -> NDArray:
if not self.spice_rack_open:
dataframe = self.start(dataframe, metadata, strategy)
self.dk.spice_dataframe = dataframe
self.spice_rack_open = True
return self.dk.spice_extractor(indicator, dataframe)
else:
return self.dk.spice_extractor(indicator, self.dk.spice_dataframe)
def close_spice_rack(self):
self.spice_rack_open = False
# Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModel.py for an example.

35
freqtrade/freqai/spice_rack/lightgbm_config.json Normal file
View File

@ -0,0 +1,35 @@
{
"freqai": {
"enabled": true,
"purge_old_models": true,
"train_period_days": 4,
"identifier": "uniqe-id",
"feature_parameters": {
"include_timeframes": [
"3m",
"15m",
"1h"
],
"include_corr_pairlist": [
"BTC/USDT",
"ETH/USDT"
],
"label_period_candles": 20,
"include_shifted_candles": 2,
"DI_threshold": 0.9,
"weight_factor": 0.9,
"indicator_periods_candles": [
10,
20
]
},
"data_split_parameters": {
"test_size": 0,
"random_state": 1
},
"model_training_parameters": {
"n_estimators": 800
}
}
}

85
freqtrade/freqai/utils.py
View File

@ -1,6 +1,13 @@
import logging
from datetime import datetime, timezone
import numpy as np
# for spice rack
import pandas as pd
import talib.abstract as ta
from scipy.signal import argrelextrema
from technical import qtpylib
from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
@ -8,6 +15,7 @@ from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.exchange.exchange import market_is_active
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
from freqtrade.strategy import merge_informative_pair
logger = logging.getLogger(__name__)
@ -85,6 +93,83 @@ def get_required_data_timerange(
return data_load_timerange
def auto_populate_any_indicators(
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
"""
This is a premade `populate_any_indicators()` function which is set in
the user strategy is they enable `freqai_spice_rack: true` in their
configuration file.
"""
coin = pair.split('/')[0]
if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2
)
informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
informative[f"%-{coin}bb_width-period_{t}"] = (
informative[f"{coin}bb_upperband-period_{t}"]
- informative[f"{coin}bb_lowerband-period_{t}"]
) / informative[f"{coin}bb_middleband-period_{t}"]
informative[f"%-{coin}close-bb_lower-period_{t}"] = (
informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
)
informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
informative[f"%-{coin}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean()
)
informative[f"%-{coin}pct-change"] = informative["close"].pct_change()
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
df["&s-minima"] = 0
df["&s-maxima"] = 0
min_peaks = argrelextrema(df["close"].values, np.less, order=80)
max_peaks = argrelextrema(df["close"].values, np.greater, order=80)
for mp in min_peaks[0]:
df.at[mp, "&s-minima"] = 1
for mp in max_peaks[0]:
df.at[mp, "&s-maxima"] = 1
return df
# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
# def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
# """

31
freqtrade/strategy/interface.py
View File

@ -5,7 +5,7 @@ This module defines the interface to apply for strategies
import logging
from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple, Union
import arrow
from pandas import DataFrame
@ -145,11 +145,26 @@ class IStrategy(ABC, HyperStrategyMixin):
self._ft_informative.append((informative_data, cls_method))
def load_freqAI_model(self) -> None:
if self.config.get('freqai', {}).get('enabled', False):
spice_rack = self.config.get('freqai_spice_rack', False)
if self.config.get('freqai', {}).get('enabled', False) or spice_rack:
spice_rack = self.config.get('freqai_spice_rack', False)
if spice_rack:
self.config = self.setup_freqai_spice_rack(self.config)
# Import here to avoid importing this if freqAI is disabled
from freqtrade.freqai.utils import download_all_data_for_training
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
if spice_rack:
import types
from freqtrade.freqai.utils import auto_populate_any_indicators
self.populate_any_indicators = types.MethodType( # type: ignore
auto_populate_any_indicators, self)
# funcType = type(IStrategy.populate_any_indicators)
# self.populate_any_indicators = funcType(self.freqai.auto_populate_any_indicators,
# self, self.populate_any_indicators)
self.freqai_info = self.config["freqai"]
# download the desired data in dry/live
@ -160,6 +175,7 @@ class IStrategy(ABC, HyperStrategyMixin):
"already on disk."
)
download_all_data_for_training(self.dp, self.config)
else:
# Gracious failures if freqAI is disabled but "start" is called.
class DummyClass():
@ -173,6 +189,17 @@ class IStrategy(ABC, HyperStrategyMixin):
self.freqai = DummyClass() # type: ignore
def setup_freqai_spice_rack(self, config: dict) -> Dict[str, Any]:
import json
from pathlib import Path
with open(Path('freqtrade') / 'freqai' / 'spice_rack'
/ 'lightgbm_config.json') as json_file:
freqai_config = json.load(json_file)
config['freqai'] = freqai_config['freqai']
config['freqai']['identifier'] = config['freqai_identifier']
config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'})
return config
def ft_bot_start(self, **kwargs) -> None:
"""
Strategy init - runs after dataprovider has been added.