Compare commits
22 Commits
develop
...
add-spice-
Author | SHA1 | Date | |
---|---|---|---|
|
ecdb466887 | ||
|
011759d1b7 | ||
|
7cdd510cf9 | ||
|
1e5df9611b | ||
|
f3dcbb9736 | ||
|
06f4f2db0a | ||
|
d362332527 | ||
|
760f3f157d | ||
|
c31f322349 | ||
|
aca03e38f6 | ||
|
8b1e5daf22 | ||
|
7b390b8edb | ||
|
91e2a05aff | ||
|
793c54db9d | ||
|
b1e92933f4 | ||
|
12a9fda885 | ||
|
a7312dec03 | ||
|
ff300d5c85 | ||
|
4d93a6b757 | ||
|
dac07c5609 | ||
|
fb2d190865 | ||
|
b209490009 |
71
docs/freqai-spice-rack.md
Normal file
71
docs/freqai-spice-rack.md
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
# Using the `spice_rack`
|
||||||
|
|
||||||
|
!!! Note:
|
||||||
|
`spice_rack` indicators should not be used exclusively for entries and exits, the following example is just a demonstration of syntax. `spice_rack` indicators should **always** be used to support existing strategies.
|
||||||
|
|
||||||
|
The `spice_rack` is aimed at users who do not wish to deal with setting up `FreqAI` confgs, but instead prefer to interact with `FreqAI` similar to a `talib` indicator. In this case, the user can instead simply add two keys to their config:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"freqai_spice_rack": true,
|
||||||
|
"freqai_identifier": "spicey-id",
|
||||||
|
```
|
||||||
|
|
||||||
|
Which tells `FreqAI` to set up a pre-set `FreqAI` instance automatically under the hood with preset parameters. Now the user can access a suite of custom `FreqAI` supercharged indicators inside their strategy by placing the following code into `populate_indicators`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
dataframe['dissimilarity_index'] = self.freqai.spice_rack(
|
||||||
|
'DI_values', dataframe, metadata, self)
|
||||||
|
dataframe['extrema'] = self.freqai.spice_rack(
|
||||||
|
'&s-extrema', dataframe, metadata, self)
|
||||||
|
self.freqai.close_spice_rack() # user must close the spicerack
|
||||||
|
```
|
||||||
|
|
||||||
|
Users can then use these columns in concert with all their own additional indicators added to `populate_indicators` in their entry/exit criteria and strategy callback methods the same way as any typical indicator. For example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
|
||||||
|
|
||||||
|
df.loc[
|
||||||
|
(
|
||||||
|
(df['dissimilarity_index'] < 1) &
|
||||||
|
(df['extrema'] < -0.1)
|
||||||
|
),
|
||||||
|
'enter_long'] = 1
|
||||||
|
|
||||||
|
df.loc[
|
||||||
|
(
|
||||||
|
(df['dissimilarity_index'] < 1) &
|
||||||
|
(df['extrema'] > 0.1)
|
||||||
|
),
|
||||||
|
'enter_short'] = 1
|
||||||
|
|
||||||
|
return df
|
||||||
|
|
||||||
|
def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame:
|
||||||
|
|
||||||
|
df.loc[
|
||||||
|
(
|
||||||
|
(df['dissimilarity_index'] < 1) &
|
||||||
|
(df['extrema'] > 0.1)
|
||||||
|
),
|
||||||
|
|
||||||
|
'exit_long'] = 1
|
||||||
|
|
||||||
|
df.loc[
|
||||||
|
(
|
||||||
|
|
||||||
|
(df['dissimilarity_index'] < 1) &
|
||||||
|
(df['extrema'] < -0.1)
|
||||||
|
),
|
||||||
|
'exit_short'] = 1
|
||||||
|
|
||||||
|
return df
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## Available indicators
|
||||||
|
|
||||||
|
| Parameter | Description |
|
||||||
|
|------------|-------------|
|
||||||
|
| `DI_values` | **Required.** <br> The dissimilarity index of the current candle to the recent candles. More information available [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di) <br> **Datatype:** Floats.
|
||||||
|
| `extrema` | **Required.** <br> A continuous prediction from FreqAI which aims to help predict if the current candle is a maxima or a minma. FreqAI aims for 1 to be a maxima and -1 to be a minima - but the values should typically hover between -0.2 and 0.2. <br> **Datatype:** Floats.
|
@ -11,7 +11,8 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
|
|||||||
refresh_backtest_trades_data)
|
refresh_backtest_trades_data)
|
||||||
from freqtrade.enums import CandleType, RunMode, TradingMode
|
from freqtrade.enums import CandleType, RunMode, TradingMode
|
||||||
from freqtrade.exceptions import OperationalException
|
from freqtrade.exceptions import OperationalException
|
||||||
from freqtrade.exchange import market_is_active, timeframe_to_minutes
|
from freqtrade.exchange import Exchange, market_is_active, timeframe_to_minutes
|
||||||
|
from freqtrade.freqai.utils import setup_freqai_spice_rack
|
||||||
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
|
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
|
||||||
from freqtrade.resolvers import ExchangeResolver
|
from freqtrade.resolvers import ExchangeResolver
|
||||||
|
|
||||||
@ -48,6 +49,10 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
|||||||
|
|
||||||
# Init exchange
|
# Init exchange
|
||||||
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
|
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
|
||||||
|
|
||||||
|
if config.get('freqai_spice_rack', False):
|
||||||
|
config = setup_freqai_spice_rack(config, exchange)
|
||||||
|
|
||||||
markets = [p for p, m in exchange.markets.items() if market_is_active(m)
|
markets = [p for p, m in exchange.markets.items() if market_is_active(m)
|
||||||
or config.get('include_inactive')]
|
or config.get('include_inactive')]
|
||||||
|
|
||||||
@ -63,37 +68,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
|||||||
exchange.validate_timeframes(timeframe)
|
exchange.validate_timeframes(timeframe)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
pairs_not_available = download_trades(exchange, expanded_pairs, config, timerange)
|
||||||
if config.get('download_trades'):
|
|
||||||
if config.get('trading_mode') == 'futures':
|
|
||||||
raise OperationalException("Trade download not supported for futures.")
|
|
||||||
pairs_not_available = refresh_backtest_trades_data(
|
|
||||||
exchange, pairs=expanded_pairs, datadir=config['datadir'],
|
|
||||||
timerange=timerange, new_pairs_days=config['new_pairs_days'],
|
|
||||||
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
|
|
||||||
|
|
||||||
# Convert downloaded trade data to different timeframes
|
|
||||||
convert_trades_to_ohlcv(
|
|
||||||
pairs=expanded_pairs, timeframes=config['timeframes'],
|
|
||||||
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
|
||||||
data_format_ohlcv=config['dataformat_ohlcv'],
|
|
||||||
data_format_trades=config['dataformat_trades'],
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
if not exchange.get_option('ohlcv_has_history', True):
|
|
||||||
raise OperationalException(
|
|
||||||
f"Historic klines not available for {exchange.name}. "
|
|
||||||
"Please use `--dl-trades` instead for this exchange "
|
|
||||||
"(will unfortunately take a long time)."
|
|
||||||
)
|
|
||||||
pairs_not_available = refresh_backtest_ohlcv_data(
|
|
||||||
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
|
|
||||||
datadir=config['datadir'], timerange=timerange,
|
|
||||||
new_pairs_days=config['new_pairs_days'],
|
|
||||||
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
|
|
||||||
trading_mode=config.get('trading_mode', 'spot'),
|
|
||||||
prepend=config.get('prepend_data', False)
|
|
||||||
)
|
|
||||||
|
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
sys.exit("SIGINT received, aborting ...")
|
sys.exit("SIGINT received, aborting ...")
|
||||||
@ -104,6 +79,42 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
|||||||
f"on exchange {exchange.name}.")
|
f"on exchange {exchange.name}.")
|
||||||
|
|
||||||
|
|
||||||
|
def download_trades(exchange: Exchange, expanded_pairs: list,
|
||||||
|
config: Dict[str, Any], timerange: TimeRange) -> list:
|
||||||
|
if config.get('download_trades'):
|
||||||
|
if config.get('trading_mode') == 'futures':
|
||||||
|
raise OperationalException("Trade download not supported for futures.")
|
||||||
|
pairs_not_available = refresh_backtest_trades_data(
|
||||||
|
exchange, pairs=expanded_pairs, datadir=config['datadir'],
|
||||||
|
timerange=timerange, new_pairs_days=config['new_pairs_days'],
|
||||||
|
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
|
||||||
|
|
||||||
|
# Convert downloaded trade data to different timeframes
|
||||||
|
convert_trades_to_ohlcv(
|
||||||
|
pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||||
|
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
||||||
|
data_format_ohlcv=config['dataformat_ohlcv'],
|
||||||
|
data_format_trades=config['dataformat_trades'],
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
if not exchange.get_option('ohlcv_has_history', True):
|
||||||
|
raise OperationalException(
|
||||||
|
f"Historic klines not available for {exchange.name}. "
|
||||||
|
"Please use `--dl-trades` instead for this exchange "
|
||||||
|
"(will unfortunately take a long time)."
|
||||||
|
)
|
||||||
|
pairs_not_available = refresh_backtest_ohlcv_data(
|
||||||
|
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||||
|
datadir=config['datadir'], timerange=timerange,
|
||||||
|
new_pairs_days=config['new_pairs_days'],
|
||||||
|
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
|
||||||
|
trading_mode=config.get('trading_mode', 'spot'),
|
||||||
|
prepend=config.get('prepend_data', False)
|
||||||
|
)
|
||||||
|
|
||||||
|
return pairs_not_available
|
||||||
|
|
||||||
|
|
||||||
def start_convert_trades(args: Dict[str, Any]) -> None:
|
def start_convert_trades(args: Dict[str, Any]) -> None:
|
||||||
|
|
||||||
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
|
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
|
||||||
|
@ -520,7 +520,7 @@ class FreqaiDataDrawer:
|
|||||||
f"Unable to load model, ensure model exists at " f"{dk.data_path} "
|
f"Unable to load model, ensure model exists at " f"{dk.data_path} "
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
|
if self.config["freqai"]["feature_parameters"].get("principal_component_analysis", False):
|
||||||
dk.pca = cloudpickle.load(
|
dk.pca = cloudpickle.load(
|
||||||
open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
|
open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
|
||||||
)
|
)
|
||||||
@ -616,7 +616,6 @@ class FreqaiDataDrawer:
|
|||||||
pairs = self.freqai_info["feature_parameters"].get(
|
pairs = self.freqai_info["feature_parameters"].get(
|
||||||
"include_corr_pairlist", []
|
"include_corr_pairlist", []
|
||||||
)
|
)
|
||||||
|
|
||||||
for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
|
for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
|
||||||
base_dataframes[tf] = dk.slice_dataframe(
|
base_dataframes[tf] = dk.slice_dataframe(
|
||||||
timerange, historic_data[pair][tf]).reset_index(drop=True)
|
timerange, historic_data[pair][tf]).reset_index(drop=True)
|
||||||
|
@ -99,6 +99,7 @@ class FreqaiDataKitchen:
|
|||||||
self.train_dates: DataFrame = pd.DataFrame()
|
self.train_dates: DataFrame = pd.DataFrame()
|
||||||
self.unique_classes: Dict[str, list] = {}
|
self.unique_classes: Dict[str, list] = {}
|
||||||
self.unique_class_list: list = []
|
self.unique_class_list: list = []
|
||||||
|
self.spice_dataframe: DataFrame = None
|
||||||
|
|
||||||
def set_paths(
|
def set_paths(
|
||||||
self,
|
self,
|
||||||
@ -1259,3 +1260,11 @@ class FreqaiDataKitchen:
|
|||||||
f"Could not find backtesting prediction file at {path_to_predictionfile}"
|
f"Could not find backtesting prediction file at {path_to_predictionfile}"
|
||||||
)
|
)
|
||||||
return file_exists
|
return file_exists
|
||||||
|
|
||||||
|
def spice_extractor(self, indicator: str, dataframe: DataFrame) -> npt.NDArray:
|
||||||
|
if indicator in dataframe.columns:
|
||||||
|
return np.array(dataframe[indicator])
|
||||||
|
else:
|
||||||
|
logger.warning(f'User asked spice_rack for {indicator}, '
|
||||||
|
f'but it is not available. Returning 0s')
|
||||||
|
return np.zeros(len(dataframe.index))
|
||||||
|
@ -93,7 +93,7 @@ class IFreqaiModel(ABC):
|
|||||||
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
|
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
|
||||||
self.continual_learning = self.freqai_info.get('continual_learning', False)
|
self.continual_learning = self.freqai_info.get('continual_learning', False)
|
||||||
self.plot_features = self.ft_params.get("plot_feature_importances", 0)
|
self.plot_features = self.ft_params.get("plot_feature_importances", 0)
|
||||||
|
self.spice_rack_open: bool = False
|
||||||
self._threads: List[threading.Thread] = []
|
self._threads: List[threading.Thread] = []
|
||||||
self._stop_event = threading.Event()
|
self._stop_event = threading.Event()
|
||||||
|
|
||||||
@ -142,7 +142,7 @@ class IFreqaiModel(ABC):
|
|||||||
dk = self.start_backtesting(dataframe, metadata, self.dk)
|
dk = self.start_backtesting(dataframe, metadata, self.dk)
|
||||||
|
|
||||||
dataframe = dk.remove_features_from_df(dk.return_dataframe)
|
dataframe = dk.remove_features_from_df(dk.return_dataframe)
|
||||||
self.clean_up()
|
# self.clean_up()
|
||||||
if self.live:
|
if self.live:
|
||||||
self.inference_timer('stop')
|
self.inference_timer('stop')
|
||||||
return dataframe
|
return dataframe
|
||||||
@ -732,6 +732,18 @@ class IFreqaiModel(ABC):
|
|||||||
f'Best approximation queue: {best_queue}')
|
f'Best approximation queue: {best_queue}')
|
||||||
return best_queue
|
return best_queue
|
||||||
|
|
||||||
|
def spice_rack(self, indicator: str, dataframe: DataFrame,
|
||||||
|
metadata: dict, strategy: IStrategy) -> NDArray:
|
||||||
|
if not self.spice_rack_open:
|
||||||
|
dataframe = self.start(dataframe, metadata, strategy)
|
||||||
|
self.dk.spice_dataframe = dataframe
|
||||||
|
self.spice_rack_open = True
|
||||||
|
return self.dk.spice_extractor(indicator, dataframe)
|
||||||
|
else:
|
||||||
|
return self.dk.spice_extractor(indicator, self.dk.spice_dataframe)
|
||||||
|
|
||||||
|
def close_spice_rack(self):
|
||||||
|
self.spice_rack_open = False
|
||||||
# Following methods which are overridden by user made prediction models.
|
# Following methods which are overridden by user made prediction models.
|
||||||
# See freqai/prediction_models/CatboostPredictionModel.py for an example.
|
# See freqai/prediction_models/CatboostPredictionModel.py for an example.
|
||||||
|
|
||||||
|
37
freqtrade/freqai/spice_rack/lightgbm_config.json
Normal file
37
freqtrade/freqai/spice_rack/lightgbm_config.json
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
|
||||||
|
"freqai": {
|
||||||
|
"enabled": true,
|
||||||
|
"purge_old_models": true,
|
||||||
|
"train_period_days": 4,
|
||||||
|
"backtest_period_days": 1,
|
||||||
|
"identifier": "spicy-id",
|
||||||
|
"feature_parameters": {
|
||||||
|
"include_timeframes": [
|
||||||
|
"30m",
|
||||||
|
"1h",
|
||||||
|
"4h"
|
||||||
|
],
|
||||||
|
"include_corr_pairlist": [
|
||||||
|
"BTC/USD",
|
||||||
|
"ETH/USD"
|
||||||
|
],
|
||||||
|
"label_period_candles": 20,
|
||||||
|
"include_shifted_candles": 2,
|
||||||
|
"DI_threshold": 0.9,
|
||||||
|
"weight_factor": 0.9,
|
||||||
|
"principal_component_analysis": true,
|
||||||
|
"indicator_periods_candles": [
|
||||||
|
10,
|
||||||
|
20
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"data_split_parameters": {
|
||||||
|
"test_size": 0,
|
||||||
|
"random_state": 1
|
||||||
|
},
|
||||||
|
"model_training_parameters": {
|
||||||
|
"n_estimators": 800
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,19 +1,24 @@
|
|||||||
import logging
|
import logging
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Any
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
# for spice rack
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
import talib.abstract as ta
|
||||||
|
from scipy.signal import argrelextrema
|
||||||
|
from technical import qtpylib
|
||||||
|
|
||||||
from freqtrade.configuration import TimeRange
|
from freqtrade.configuration import TimeRange
|
||||||
from freqtrade.constants import Config
|
from freqtrade.constants import Config
|
||||||
from freqtrade.data.dataprovider import DataProvider
|
from freqtrade.data.dataprovider import DataProvider
|
||||||
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
|
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
|
||||||
from freqtrade.exceptions import OperationalException
|
from freqtrade.exceptions import OperationalException
|
||||||
from freqtrade.exchange import timeframe_to_seconds
|
from freqtrade.exchange import Exchange, timeframe_to_seconds
|
||||||
from freqtrade.exchange.exchange import market_is_active
|
from freqtrade.exchange.exchange import market_is_active
|
||||||
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
|
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
|
||||||
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
|
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
|
||||||
|
from freqtrade.strategy import merge_informative_pair
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@ -89,6 +94,136 @@ def get_required_data_timerange(config: Config) -> TimeRange:
|
|||||||
return data_load_timerange
|
return data_load_timerange
|
||||||
|
|
||||||
|
|
||||||
|
def auto_populate_any_indicators(
|
||||||
|
self, pair, df, tf, informative=None, set_generalized_indicators=False
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
This is a premade `populate_any_indicators()` function which is set in
|
||||||
|
the user strategy is they enable `freqai_spice_rack: true` in their
|
||||||
|
configuration file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
coin = pair.split('/')[0]
|
||||||
|
|
||||||
|
if informative is None:
|
||||||
|
informative = self.dp.get_pair_dataframe(pair, tf)
|
||||||
|
|
||||||
|
# first loop is automatically duplicating indicators for time periods
|
||||||
|
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
|
||||||
|
|
||||||
|
t = int(t)
|
||||||
|
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
|
||||||
|
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
|
||||||
|
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
|
||||||
|
informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
|
||||||
|
informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
|
||||||
|
|
||||||
|
bollinger = qtpylib.bollinger_bands(
|
||||||
|
qtpylib.typical_price(informative), window=t, stds=2.2
|
||||||
|
)
|
||||||
|
informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"]
|
||||||
|
informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"]
|
||||||
|
informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"]
|
||||||
|
|
||||||
|
informative[f"%-{coin}bb_width-period_{t}"] = (
|
||||||
|
informative[f"{coin}bb_upperband-period_{t}"]
|
||||||
|
- informative[f"{coin}bb_lowerband-period_{t}"]
|
||||||
|
) / informative[f"{coin}bb_middleband-period_{t}"]
|
||||||
|
informative[f"%-{coin}close-bb_lower-period_{t}"] = (
|
||||||
|
informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
|
||||||
|
)
|
||||||
|
|
||||||
|
informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
|
||||||
|
|
||||||
|
informative[f"%-{coin}relative_volume-period_{t}"] = (
|
||||||
|
informative["volume"] / informative["volume"].rolling(t).mean()
|
||||||
|
)
|
||||||
|
|
||||||
|
informative[f"%-{coin}pct-change"] = informative["close"].pct_change()
|
||||||
|
informative[f"%-{coin}raw_volume"] = informative["volume"]
|
||||||
|
informative[f"%-{coin}raw_price"] = informative["close"]
|
||||||
|
|
||||||
|
indicators = [col for col in informative if col.startswith("%")]
|
||||||
|
# This loop duplicates and shifts all indicators to add a sense of recency to data
|
||||||
|
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
|
||||||
|
if n == 0:
|
||||||
|
continue
|
||||||
|
informative_shift = informative[indicators].shift(n)
|
||||||
|
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
|
||||||
|
informative = pd.concat((informative, informative_shift), axis=1)
|
||||||
|
|
||||||
|
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
|
||||||
|
skip_columns = [
|
||||||
|
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
|
||||||
|
]
|
||||||
|
df = df.drop(columns=skip_columns)
|
||||||
|
if set_generalized_indicators:
|
||||||
|
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
|
||||||
|
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
|
||||||
|
df["&s-extrema"] = 0
|
||||||
|
min_peaks = argrelextrema(df["close"].values, np.less, order=80)
|
||||||
|
max_peaks = argrelextrema(df["close"].values, np.greater, order=80)
|
||||||
|
for mp in min_peaks[0]:
|
||||||
|
df.at[mp, "&s-extrema"] = -1
|
||||||
|
for mp in max_peaks[0]:
|
||||||
|
df.at[mp, "&s-extrema"] = 1
|
||||||
|
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def setup_freqai_spice_rack(config: dict, exchange: Optional[Exchange]) -> Dict[str, Any]:
|
||||||
|
import difflib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
auto_config = config.get('freqai_config', 'lightgbm_config.json')
|
||||||
|
with open(Path(__file__).parent / Path('spice_rack') / auto_config) as json_file:
|
||||||
|
freqai_config = json.load(json_file)
|
||||||
|
config['freqai'] = freqai_config['freqai']
|
||||||
|
config['freqai']['identifier'] = config['freqai_identifier']
|
||||||
|
corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist']
|
||||||
|
timeframes = config['freqai']['feature_parameters']['include_timeframes']
|
||||||
|
new_corr_pairs = []
|
||||||
|
new_tfs = []
|
||||||
|
|
||||||
|
if not exchange:
|
||||||
|
logger.warning('No dataprovider available.')
|
||||||
|
config['freqai']['enabled'] = False
|
||||||
|
return config
|
||||||
|
# find the closest pairs to what the default config wants
|
||||||
|
for pair in corr_pairs:
|
||||||
|
closest_pair = difflib.get_close_matches(
|
||||||
|
pair,
|
||||||
|
exchange.markets
|
||||||
|
)
|
||||||
|
if not closest_pair:
|
||||||
|
logger.warning(f'Could not find {pair} in markets, removing from '
|
||||||
|
f'corr_pairlist.')
|
||||||
|
else:
|
||||||
|
closest_pair = closest_pair[0]
|
||||||
|
|
||||||
|
new_corr_pairs.append(closest_pair)
|
||||||
|
logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.')
|
||||||
|
|
||||||
|
# find the closest matching timeframes to what the default config wants
|
||||||
|
if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'):
|
||||||
|
logger.warning('Default spice rack is designed for lower base timeframes (e.g. > '
|
||||||
|
f'15m). But user passed {config["timeframe"]}.')
|
||||||
|
new_tfs.append(config['timeframe'])
|
||||||
|
|
||||||
|
list_tfs = [timeframe_to_seconds(tf) for tf
|
||||||
|
in exchange.timeframes]
|
||||||
|
for tf in timeframes:
|
||||||
|
tf_secs = timeframe_to_seconds(tf)
|
||||||
|
closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs))
|
||||||
|
closest_tf = exchange.timeframes[closest_index]
|
||||||
|
logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.')
|
||||||
|
new_tfs.append(closest_tf)
|
||||||
|
|
||||||
|
config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs})
|
||||||
|
config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs})
|
||||||
|
config.update({"freqaimodel": 'LightGBMRegressor'})
|
||||||
|
return config
|
||||||
|
|
||||||
# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
|
# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
|
||||||
# def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
|
# def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
|
||||||
# """
|
# """
|
||||||
|
@ -89,6 +89,10 @@ class Backtesting:
|
|||||||
self._exchange_name, self.config, load_leverage_tiers=True)
|
self._exchange_name, self.config, load_leverage_tiers=True)
|
||||||
self.dataprovider = DataProvider(self.config, self.exchange)
|
self.dataprovider = DataProvider(self.config, self.exchange)
|
||||||
|
|
||||||
|
if config.get('freqai_spice_rack', False):
|
||||||
|
from freqtrade.freqai.utils import setup_freqai_spice_rack
|
||||||
|
self.config = setup_freqai_spice_rack(self.config, self.exchange)
|
||||||
|
|
||||||
if self.config.get('strategy_list'):
|
if self.config.get('strategy_list'):
|
||||||
if self.config.get('freqai', {}).get('enabled', False):
|
if self.config.get('freqai', {}).get('enabled', False):
|
||||||
logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
|
logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies "
|
||||||
|
@ -146,12 +146,28 @@ class IStrategy(ABC, HyperStrategyMixin):
|
|||||||
self._ft_informative.append((informative_data, cls_method))
|
self._ft_informative.append((informative_data, cls_method))
|
||||||
|
|
||||||
def load_freqAI_model(self) -> None:
|
def load_freqAI_model(self) -> None:
|
||||||
if self.config.get('freqai', {}).get('enabled', False):
|
spice_rack = self.config.get('freqai_spice_rack', False)
|
||||||
|
if self.config.get('freqai', {}).get('enabled', False) or spice_rack:
|
||||||
|
if spice_rack:
|
||||||
|
from freqtrade.freqai.utils import setup_freqai_spice_rack
|
||||||
|
self.config = setup_freqai_spice_rack(self.config, self.dp._exchange)
|
||||||
# Import here to avoid importing this if freqAI is disabled
|
# Import here to avoid importing this if freqAI is disabled
|
||||||
from freqtrade.freqai.utils import download_all_data_for_training
|
from freqtrade.freqai.utils import download_all_data_for_training
|
||||||
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
|
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
|
||||||
self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
|
self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
|
||||||
self.freqai_info = self.config["freqai"]
|
if not self.process_only_new_candles:
|
||||||
|
logger.warning('User set process_only_new_candles to false, '
|
||||||
|
'FreqAI requires true. Changing to true.')
|
||||||
|
self.process_only_new_candles = True
|
||||||
|
|
||||||
|
if spice_rack:
|
||||||
|
import types
|
||||||
|
|
||||||
|
from freqtrade.freqai.utils import auto_populate_any_indicators
|
||||||
|
self.populate_any_indicators = types.MethodType( # type: ignore
|
||||||
|
auto_populate_any_indicators, self)
|
||||||
|
|
||||||
|
self.freqai_info = self.config["freqai"]
|
||||||
|
|
||||||
# download the desired data in dry/live
|
# download the desired data in dry/live
|
||||||
if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE):
|
if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE):
|
||||||
@ -161,6 +177,7 @@ class IStrategy(ABC, HyperStrategyMixin):
|
|||||||
"already on disk."
|
"already on disk."
|
||||||
)
|
)
|
||||||
download_all_data_for_training(self.dp, self.config)
|
download_all_data_for_training(self.dp, self.config)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# Gracious failures if freqAI is disabled but "start" is called.
|
# Gracious failures if freqAI is disabled but "start" is called.
|
||||||
class DummyClass():
|
class DummyClass():
|
||||||
|
@ -29,6 +29,7 @@ nav:
|
|||||||
- Parameter table: freqai-parameter-table.md
|
- Parameter table: freqai-parameter-table.md
|
||||||
- Feature engineering: freqai-feature-engineering.md
|
- Feature engineering: freqai-feature-engineering.md
|
||||||
- Running FreqAI: freqai-running.md
|
- Running FreqAI: freqai-running.md
|
||||||
|
- Spice Rack: freqai-spice-rack.md
|
||||||
- Developer guide: freqai-developers.md
|
- Developer guide: freqai-developers.md
|
||||||
- Short / Leverage: leverage.md
|
- Short / Leverage: leverage.md
|
||||||
- Utility Sub-commands: utils.md
|
- Utility Sub-commands: utils.md
|
||||||
|
@ -158,3 +158,28 @@ def test_make_train_test_datasets(mocker, freqai_conf):
|
|||||||
assert data_dictionary
|
assert data_dictionary
|
||||||
assert len(data_dictionary) == 7
|
assert len(data_dictionary) == 7
|
||||||
assert len(data_dictionary['train_features'].index) == 1916
|
assert len(data_dictionary['train_features'].index) == 1916
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('indicator', [
|
||||||
|
'%-ADArsi-period_10_5m',
|
||||||
|
'doesnt_exist',
|
||||||
|
])
|
||||||
|
def test_spice_extractor(mocker, freqai_conf, indicator, caplog):
|
||||||
|
freqai, unfiltered_dataframe = make_unfiltered_dataframe(mocker, freqai_conf)
|
||||||
|
freqai.dk.find_features(unfiltered_dataframe)
|
||||||
|
|
||||||
|
features_filtered, labels_filtered = freqai.dk.filter_features(
|
||||||
|
unfiltered_dataframe,
|
||||||
|
freqai.dk.training_features_list,
|
||||||
|
freqai.dk.label_list,
|
||||||
|
training_filter=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
vec = freqai.dk.spice_extractor(indicator, features_filtered)
|
||||||
|
if 'doesnt_exist' in indicator:
|
||||||
|
assert log_has_re(
|
||||||
|
"User asked spice_rack for",
|
||||||
|
caplog,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
assert len(vec) == 2860
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
import copy
|
||||||
import platform
|
import platform
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@ -382,6 +383,31 @@ def test_plot_feature_importance(mocker, freqai_conf):
|
|||||||
shutil.rmtree(Path(freqai.dk.full_path))
|
shutil.rmtree(Path(freqai.dk.full_path))
|
||||||
|
|
||||||
|
|
||||||
|
def test_spice_rack(mocker, default_conf, tmpdir, caplog):
|
||||||
|
|
||||||
|
strategy = get_patched_freqai_strategy(mocker, default_conf)
|
||||||
|
exchange = get_patched_exchange(mocker, default_conf)
|
||||||
|
strategy.dp = DataProvider(default_conf, exchange)
|
||||||
|
|
||||||
|
default_conf.update({"freqai_spice_rack": "true"})
|
||||||
|
default_conf.update({"freqai_identifier": "spicy-id"})
|
||||||
|
default_conf["config_files"] = [Path('config_examples', 'config_freqai.example.json')]
|
||||||
|
default_conf["timerange"] = "20180110-20180115"
|
||||||
|
default_conf["datadir"] = Path(default_conf["datadir"])
|
||||||
|
default_conf['exchange'].update({'pair_whitelist':
|
||||||
|
['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC']})
|
||||||
|
default_conf["user_data_dir"] = Path(tmpdir)
|
||||||
|
freqai_conf = copy.deepcopy(default_conf)
|
||||||
|
|
||||||
|
strategy.config = freqai_conf
|
||||||
|
strategy.load_freqAI_model()
|
||||||
|
|
||||||
|
assert log_has_re("Spice rack will use LTC/USD", caplog)
|
||||||
|
assert log_has_re("Spice rack will use 15m", caplog)
|
||||||
|
assert 'freqai' in freqai_conf
|
||||||
|
assert strategy.freqai
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('timeframes,corr_pairs', [
|
@pytest.mark.parametrize('timeframes,corr_pairs', [
|
||||||
(['5m'], ['ADA/BTC', 'DASH/BTC']),
|
(['5m'], ['ADA/BTC', 'DASH/BTC']),
|
||||||
(['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']),
|
(['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']),
|
||||||
|
Loading…
Reference in New Issue
Block a user