Compare commits
	
		
			22 Commits
		
	
	
		
			develop
			...
			add-spice-
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
|  | ecdb466887 | ||
|  | 011759d1b7 | ||
|  | 7cdd510cf9 | ||
|  | 1e5df9611b | ||
|  | f3dcbb9736 | ||
|  | 06f4f2db0a | ||
|  | d362332527 | ||
|  | 760f3f157d | ||
|  | c31f322349 | ||
|  | aca03e38f6 | ||
|  | 8b1e5daf22 | ||
|  | 7b390b8edb | ||
|  | 91e2a05aff | ||
|  | 793c54db9d | ||
|  | b1e92933f4 | ||
|  | 12a9fda885 | ||
|  | a7312dec03 | ||
|  | ff300d5c85 | ||
|  | 4d93a6b757 | ||
|  | dac07c5609 | ||
|  | fb2d190865 | ||
|  | b209490009 | 
							
								
								
									
										71
									
								
								docs/freqai-spice-rack.md
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										71
									
								
								docs/freqai-spice-rack.md
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,71 @@ | |||||||
|  | # Using the `spice_rack` | ||||||
|  |  | ||||||
|  | !!! Note: | ||||||
|  |     `spice_rack` indicators should not be used exclusively for entries and exits, the following example is just a demonstration of syntax. `spice_rack` indicators should **always** be used to support existing strategies. | ||||||
|  |  | ||||||
|  | The `spice_rack` is aimed at users who do not wish to deal with setting up `FreqAI` confgs, but instead prefer to interact with `FreqAI` similar to a `talib` indicator. In this case, the user can instead simply add two keys to their config: | ||||||
|  |  | ||||||
|  | ```json | ||||||
|  |     "freqai_spice_rack": true,  | ||||||
|  |     "freqai_identifier": "spicey-id", | ||||||
|  | ``` | ||||||
|  |  | ||||||
|  | Which tells `FreqAI` to set up a pre-set `FreqAI` instance automatically under the hood with preset parameters. Now the user can access a suite of custom `FreqAI` supercharged indicators inside their strategy by placing the following code into `populate_indicators`: | ||||||
|  |  | ||||||
|  | ```python | ||||||
|  |         dataframe['dissimilarity_index'] = self.freqai.spice_rack( | ||||||
|  |             'DI_values', dataframe, metadata, self) | ||||||
|  |         dataframe['extrema'] = self.freqai.spice_rack( | ||||||
|  |             '&s-extrema', dataframe, metadata, self) | ||||||
|  |         self.freqai.close_spice_rack()  # user must close the spicerack | ||||||
|  | ``` | ||||||
|  |  | ||||||
|  | Users can then use these columns in concert with all their own additional indicators added to `populate_indicators` in their entry/exit criteria and strategy callback methods the same way as any typical indicator. For example: | ||||||
|  |  | ||||||
|  | ```python | ||||||
|  |     def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: | ||||||
|  |  | ||||||
|  |         df.loc[ | ||||||
|  |             ( | ||||||
|  |                 (df['dissimilarity_index'] < 1) & | ||||||
|  |                 (df['extrema'] < -0.1) | ||||||
|  |             ), | ||||||
|  |             'enter_long'] = 1 | ||||||
|  |  | ||||||
|  |         df.loc[ | ||||||
|  |             ( | ||||||
|  |                 (df['dissimilarity_index'] < 1) & | ||||||
|  |                 (df['extrema'] > 0.1) | ||||||
|  |             ), | ||||||
|  |             'enter_short'] = 1 | ||||||
|  |  | ||||||
|  |         return df | ||||||
|  |  | ||||||
|  |     def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: | ||||||
|  |  | ||||||
|  |         df.loc[ | ||||||
|  |             ( | ||||||
|  |                 (df['dissimilarity_index'] < 1) & | ||||||
|  |                 (df['extrema'] > 0.1)  | ||||||
|  |             ), | ||||||
|  |  | ||||||
|  |             'exit_long'] = 1 | ||||||
|  |  | ||||||
|  |         df.loc[ | ||||||
|  |             ( | ||||||
|  |  | ||||||
|  |                 (df['dissimilarity_index'] < 1) & | ||||||
|  |                 (df['extrema'] < -0.1) | ||||||
|  |             ), | ||||||
|  |             'exit_short'] = 1 | ||||||
|  |  | ||||||
|  |         return df | ||||||
|  | ``` | ||||||
|  |  | ||||||
|  |  | ||||||
|  | ## Available indicators | ||||||
|  |  | ||||||
|  | |  Parameter | Description | | ||||||
|  | |------------|-------------| | ||||||
|  | | `DI_values` | **Required.** <br> The dissimilarity index of the current candle to the recent candles. More information available [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di) <br> **Datatype:** Floats. | ||||||
|  | | `extrema` | **Required.** <br> A continuous prediction from FreqAI which aims to help predict if the current candle is a maxima or a minma. FreqAI aims for 1 to be a maxima and -1 to be a minima - but the values should typically hover between -0.2 and 0.2. <br> **Datatype:** Floats. | ||||||
| @@ -11,7 +11,8 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh | |||||||
|                                     refresh_backtest_trades_data) |                                     refresh_backtest_trades_data) | ||||||
| from freqtrade.enums import CandleType, RunMode, TradingMode | from freqtrade.enums import CandleType, RunMode, TradingMode | ||||||
| from freqtrade.exceptions import OperationalException | from freqtrade.exceptions import OperationalException | ||||||
| from freqtrade.exchange import market_is_active, timeframe_to_minutes | from freqtrade.exchange import Exchange, market_is_active, timeframe_to_minutes | ||||||
|  | from freqtrade.freqai.utils import setup_freqai_spice_rack | ||||||
| from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist | from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist | ||||||
| from freqtrade.resolvers import ExchangeResolver | from freqtrade.resolvers import ExchangeResolver | ||||||
|  |  | ||||||
| @@ -48,6 +49,10 @@ def start_download_data(args: Dict[str, Any]) -> None: | |||||||
|  |  | ||||||
|     # Init exchange |     # Init exchange | ||||||
|     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) |     exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) | ||||||
|  |  | ||||||
|  |     if config.get('freqai_spice_rack', False): | ||||||
|  |         config = setup_freqai_spice_rack(config, exchange) | ||||||
|  |  | ||||||
|     markets = [p for p, m in exchange.markets.items() if market_is_active(m) |     markets = [p for p, m in exchange.markets.items() if market_is_active(m) | ||||||
|                or config.get('include_inactive')] |                or config.get('include_inactive')] | ||||||
|  |  | ||||||
| @@ -63,37 +68,7 @@ def start_download_data(args: Dict[str, Any]) -> None: | |||||||
|         exchange.validate_timeframes(timeframe) |         exchange.validate_timeframes(timeframe) | ||||||
|  |  | ||||||
|     try: |     try: | ||||||
|  |         pairs_not_available = download_trades(exchange, expanded_pairs, config, timerange) | ||||||
|         if config.get('download_trades'): |  | ||||||
|             if config.get('trading_mode') == 'futures': |  | ||||||
|                 raise OperationalException("Trade download not supported for futures.") |  | ||||||
|             pairs_not_available = refresh_backtest_trades_data( |  | ||||||
|                 exchange, pairs=expanded_pairs, datadir=config['datadir'], |  | ||||||
|                 timerange=timerange, new_pairs_days=config['new_pairs_days'], |  | ||||||
|                 erase=bool(config.get('erase')), data_format=config['dataformat_trades']) |  | ||||||
|  |  | ||||||
|             # Convert downloaded trade data to different timeframes |  | ||||||
|             convert_trades_to_ohlcv( |  | ||||||
|                 pairs=expanded_pairs, timeframes=config['timeframes'], |  | ||||||
|                 datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), |  | ||||||
|                 data_format_ohlcv=config['dataformat_ohlcv'], |  | ||||||
|                 data_format_trades=config['dataformat_trades'], |  | ||||||
|             ) |  | ||||||
|         else: |  | ||||||
|             if not exchange.get_option('ohlcv_has_history', True): |  | ||||||
|                 raise OperationalException( |  | ||||||
|                     f"Historic klines not available for {exchange.name}. " |  | ||||||
|                     "Please use `--dl-trades` instead for this exchange " |  | ||||||
|                     "(will unfortunately take a long time)." |  | ||||||
|                     ) |  | ||||||
|             pairs_not_available = refresh_backtest_ohlcv_data( |  | ||||||
|                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'], |  | ||||||
|                 datadir=config['datadir'], timerange=timerange, |  | ||||||
|                 new_pairs_days=config['new_pairs_days'], |  | ||||||
|                 erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], |  | ||||||
|                 trading_mode=config.get('trading_mode', 'spot'), |  | ||||||
|                 prepend=config.get('prepend_data', False) |  | ||||||
|             ) |  | ||||||
|  |  | ||||||
|     except KeyboardInterrupt: |     except KeyboardInterrupt: | ||||||
|         sys.exit("SIGINT received, aborting ...") |         sys.exit("SIGINT received, aborting ...") | ||||||
| @@ -104,6 +79,42 @@ def start_download_data(args: Dict[str, Any]) -> None: | |||||||
|                         f"on exchange {exchange.name}.") |                         f"on exchange {exchange.name}.") | ||||||
|  |  | ||||||
|  |  | ||||||
|  | def download_trades(exchange: Exchange, expanded_pairs: list, | ||||||
|  |                     config: Dict[str, Any], timerange: TimeRange) -> list: | ||||||
|  |     if config.get('download_trades'): | ||||||
|  |         if config.get('trading_mode') == 'futures': | ||||||
|  |             raise OperationalException("Trade download not supported for futures.") | ||||||
|  |         pairs_not_available = refresh_backtest_trades_data( | ||||||
|  |             exchange, pairs=expanded_pairs, datadir=config['datadir'], | ||||||
|  |             timerange=timerange, new_pairs_days=config['new_pairs_days'], | ||||||
|  |             erase=bool(config.get('erase')), data_format=config['dataformat_trades']) | ||||||
|  |  | ||||||
|  |         # Convert downloaded trade data to different timeframes | ||||||
|  |         convert_trades_to_ohlcv( | ||||||
|  |             pairs=expanded_pairs, timeframes=config['timeframes'], | ||||||
|  |             datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), | ||||||
|  |             data_format_ohlcv=config['dataformat_ohlcv'], | ||||||
|  |             data_format_trades=config['dataformat_trades'], | ||||||
|  |         ) | ||||||
|  |     else: | ||||||
|  |         if not exchange.get_option('ohlcv_has_history', True): | ||||||
|  |             raise OperationalException( | ||||||
|  |                 f"Historic klines not available for {exchange.name}. " | ||||||
|  |                 "Please use `--dl-trades` instead for this exchange " | ||||||
|  |                 "(will unfortunately take a long time)." | ||||||
|  |                 ) | ||||||
|  |         pairs_not_available = refresh_backtest_ohlcv_data( | ||||||
|  |             exchange, pairs=expanded_pairs, timeframes=config['timeframes'], | ||||||
|  |             datadir=config['datadir'], timerange=timerange, | ||||||
|  |             new_pairs_days=config['new_pairs_days'], | ||||||
|  |             erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], | ||||||
|  |             trading_mode=config.get('trading_mode', 'spot'), | ||||||
|  |             prepend=config.get('prepend_data', False) | ||||||
|  |         ) | ||||||
|  |  | ||||||
|  |     return pairs_not_available | ||||||
|  |  | ||||||
|  |  | ||||||
| def start_convert_trades(args: Dict[str, Any]) -> None: | def start_convert_trades(args: Dict[str, Any]) -> None: | ||||||
|  |  | ||||||
|     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) |     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) | ||||||
|   | |||||||
| @@ -520,7 +520,7 @@ class FreqaiDataDrawer: | |||||||
|                 f"Unable to load model, ensure model exists at " f"{dk.data_path} " |                 f"Unable to load model, ensure model exists at " f"{dk.data_path} " | ||||||
|             ) |             ) | ||||||
|  |  | ||||||
|         if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]: |         if self.config["freqai"]["feature_parameters"].get("principal_component_analysis", False): | ||||||
|             dk.pca = cloudpickle.load( |             dk.pca = cloudpickle.load( | ||||||
|                 open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb") |                 open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb") | ||||||
|             ) |             ) | ||||||
| @@ -616,7 +616,6 @@ class FreqaiDataDrawer: | |||||||
|             pairs = self.freqai_info["feature_parameters"].get( |             pairs = self.freqai_info["feature_parameters"].get( | ||||||
|                 "include_corr_pairlist", [] |                 "include_corr_pairlist", [] | ||||||
|             ) |             ) | ||||||
|  |  | ||||||
|             for tf in self.freqai_info["feature_parameters"].get("include_timeframes"): |             for tf in self.freqai_info["feature_parameters"].get("include_timeframes"): | ||||||
|                 base_dataframes[tf] = dk.slice_dataframe( |                 base_dataframes[tf] = dk.slice_dataframe( | ||||||
|                     timerange, historic_data[pair][tf]).reset_index(drop=True) |                     timerange, historic_data[pair][tf]).reset_index(drop=True) | ||||||
|   | |||||||
| @@ -99,6 +99,7 @@ class FreqaiDataKitchen: | |||||||
|         self.train_dates: DataFrame = pd.DataFrame() |         self.train_dates: DataFrame = pd.DataFrame() | ||||||
|         self.unique_classes: Dict[str, list] = {} |         self.unique_classes: Dict[str, list] = {} | ||||||
|         self.unique_class_list: list = [] |         self.unique_class_list: list = [] | ||||||
|  |         self.spice_dataframe: DataFrame = None | ||||||
|  |  | ||||||
|     def set_paths( |     def set_paths( | ||||||
|         self, |         self, | ||||||
| @@ -1259,3 +1260,11 @@ class FreqaiDataKitchen: | |||||||
|                 f"Could not find backtesting prediction file at {path_to_predictionfile}" |                 f"Could not find backtesting prediction file at {path_to_predictionfile}" | ||||||
|             ) |             ) | ||||||
|         return file_exists |         return file_exists | ||||||
|  |  | ||||||
|  |     def spice_extractor(self, indicator: str, dataframe: DataFrame) -> npt.NDArray: | ||||||
|  |         if indicator in dataframe.columns: | ||||||
|  |             return np.array(dataframe[indicator]) | ||||||
|  |         else: | ||||||
|  |             logger.warning(f'User asked spice_rack for {indicator}, ' | ||||||
|  |                            f'but it is not available. Returning 0s') | ||||||
|  |             return np.zeros(len(dataframe.index)) | ||||||
|   | |||||||
| @@ -93,7 +93,7 @@ class IFreqaiModel(ABC): | |||||||
|         self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) |         self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe']) | ||||||
|         self.continual_learning = self.freqai_info.get('continual_learning', False) |         self.continual_learning = self.freqai_info.get('continual_learning', False) | ||||||
|         self.plot_features = self.ft_params.get("plot_feature_importances", 0) |         self.plot_features = self.ft_params.get("plot_feature_importances", 0) | ||||||
|  |         self.spice_rack_open: bool = False | ||||||
|         self._threads: List[threading.Thread] = [] |         self._threads: List[threading.Thread] = [] | ||||||
|         self._stop_event = threading.Event() |         self._stop_event = threading.Event() | ||||||
|  |  | ||||||
| @@ -142,7 +142,7 @@ class IFreqaiModel(ABC): | |||||||
|             dk = self.start_backtesting(dataframe, metadata, self.dk) |             dk = self.start_backtesting(dataframe, metadata, self.dk) | ||||||
|  |  | ||||||
|         dataframe = dk.remove_features_from_df(dk.return_dataframe) |         dataframe = dk.remove_features_from_df(dk.return_dataframe) | ||||||
|         self.clean_up() |         # self.clean_up() | ||||||
|         if self.live: |         if self.live: | ||||||
|             self.inference_timer('stop') |             self.inference_timer('stop') | ||||||
|         return dataframe |         return dataframe | ||||||
| @@ -732,6 +732,18 @@ class IFreqaiModel(ABC): | |||||||
|                     f'Best approximation queue: {best_queue}') |                     f'Best approximation queue: {best_queue}') | ||||||
|         return best_queue |         return best_queue | ||||||
|  |  | ||||||
|  |     def spice_rack(self, indicator: str, dataframe: DataFrame, | ||||||
|  |                    metadata: dict, strategy: IStrategy) -> NDArray: | ||||||
|  |         if not self.spice_rack_open: | ||||||
|  |             dataframe = self.start(dataframe, metadata, strategy) | ||||||
|  |             self.dk.spice_dataframe = dataframe | ||||||
|  |             self.spice_rack_open = True | ||||||
|  |             return self.dk.spice_extractor(indicator, dataframe) | ||||||
|  |         else: | ||||||
|  |             return self.dk.spice_extractor(indicator, self.dk.spice_dataframe) | ||||||
|  |  | ||||||
|  |     def close_spice_rack(self): | ||||||
|  |         self.spice_rack_open = False | ||||||
|     # Following methods which are overridden by user made prediction models. |     # Following methods which are overridden by user made prediction models. | ||||||
|     # See freqai/prediction_models/CatboostPredictionModel.py for an example. |     # See freqai/prediction_models/CatboostPredictionModel.py for an example. | ||||||
|  |  | ||||||
|   | |||||||
							
								
								
									
										37
									
								
								freqtrade/freqai/spice_rack/lightgbm_config.json
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										37
									
								
								freqtrade/freqai/spice_rack/lightgbm_config.json
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,37 @@ | |||||||
|  | { | ||||||
|  |  | ||||||
|  |     "freqai": { | ||||||
|  |         "enabled": true, | ||||||
|  |         "purge_old_models": true, | ||||||
|  |         "train_period_days": 4, | ||||||
|  |         "backtest_period_days": 1, | ||||||
|  |         "identifier": "spicy-id", | ||||||
|  |         "feature_parameters": { | ||||||
|  |             "include_timeframes": [ | ||||||
|  |                 "30m", | ||||||
|  |                 "1h", | ||||||
|  |                 "4h" | ||||||
|  |             ], | ||||||
|  |             "include_corr_pairlist": [ | ||||||
|  |                 "BTC/USD", | ||||||
|  |                 "ETH/USD" | ||||||
|  |             ], | ||||||
|  |             "label_period_candles": 20, | ||||||
|  |             "include_shifted_candles": 2, | ||||||
|  |             "DI_threshold": 0.9, | ||||||
|  |             "weight_factor": 0.9, | ||||||
|  |             "principal_component_analysis": true, | ||||||
|  |             "indicator_periods_candles": [ | ||||||
|  |                 10, | ||||||
|  |                 20 | ||||||
|  |             ] | ||||||
|  |         }, | ||||||
|  |         "data_split_parameters": { | ||||||
|  |             "test_size": 0, | ||||||
|  |             "random_state": 1 | ||||||
|  |         }, | ||||||
|  |         "model_training_parameters": { | ||||||
|  |             "n_estimators": 800 | ||||||
|  |         } | ||||||
|  |     } | ||||||
|  | } | ||||||
| @@ -1,19 +1,24 @@ | |||||||
| import logging | import logging | ||||||
| from datetime import datetime, timezone | from datetime import datetime, timezone | ||||||
| from typing import Any | from typing import Any, Dict, Optional | ||||||
|  |  | ||||||
| import numpy as np | import numpy as np | ||||||
|  | # for spice rack | ||||||
| import pandas as pd | import pandas as pd | ||||||
|  | import talib.abstract as ta | ||||||
|  | from scipy.signal import argrelextrema | ||||||
|  | from technical import qtpylib | ||||||
|  |  | ||||||
| from freqtrade.configuration import TimeRange | from freqtrade.configuration import TimeRange | ||||||
| from freqtrade.constants import Config | from freqtrade.constants import Config | ||||||
| from freqtrade.data.dataprovider import DataProvider | from freqtrade.data.dataprovider import DataProvider | ||||||
| from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data | from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data | ||||||
| from freqtrade.exceptions import OperationalException | from freqtrade.exceptions import OperationalException | ||||||
| from freqtrade.exchange import timeframe_to_seconds | from freqtrade.exchange import Exchange, timeframe_to_seconds | ||||||
| from freqtrade.exchange.exchange import market_is_active | from freqtrade.exchange.exchange import market_is_active | ||||||
| from freqtrade.freqai.data_kitchen import FreqaiDataKitchen | from freqtrade.freqai.data_kitchen import FreqaiDataKitchen | ||||||
| from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist | from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist | ||||||
|  | from freqtrade.strategy import merge_informative_pair | ||||||
|  |  | ||||||
|  |  | ||||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||||
| @@ -89,6 +94,136 @@ def get_required_data_timerange(config: Config) -> TimeRange: | |||||||
|     return data_load_timerange |     return data_load_timerange | ||||||
|  |  | ||||||
|  |  | ||||||
|  | def auto_populate_any_indicators( | ||||||
|  |     self, pair, df, tf, informative=None, set_generalized_indicators=False | ||||||
|  | ): | ||||||
|  |     """ | ||||||
|  |     This is a premade `populate_any_indicators()` function which is set in | ||||||
|  |     the user strategy is they enable `freqai_spice_rack: true` in their | ||||||
|  |     configuration file. | ||||||
|  |     """ | ||||||
|  |  | ||||||
|  |     coin = pair.split('/')[0] | ||||||
|  |  | ||||||
|  |     if informative is None: | ||||||
|  |         informative = self.dp.get_pair_dataframe(pair, tf) | ||||||
|  |  | ||||||
|  |     # first loop is automatically duplicating indicators for time periods | ||||||
|  |     for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: | ||||||
|  |  | ||||||
|  |         t = int(t) | ||||||
|  |         informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) | ||||||
|  |         informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) | ||||||
|  |         informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) | ||||||
|  |         informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) | ||||||
|  |         informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) | ||||||
|  |  | ||||||
|  |         bollinger = qtpylib.bollinger_bands( | ||||||
|  |             qtpylib.typical_price(informative), window=t, stds=2.2 | ||||||
|  |         ) | ||||||
|  |         informative[f"{coin}bb_lowerband-period_{t}"] = bollinger["lower"] | ||||||
|  |         informative[f"{coin}bb_middleband-period_{t}"] = bollinger["mid"] | ||||||
|  |         informative[f"{coin}bb_upperband-period_{t}"] = bollinger["upper"] | ||||||
|  |  | ||||||
|  |         informative[f"%-{coin}bb_width-period_{t}"] = ( | ||||||
|  |             informative[f"{coin}bb_upperband-period_{t}"] | ||||||
|  |             - informative[f"{coin}bb_lowerband-period_{t}"] | ||||||
|  |         ) / informative[f"{coin}bb_middleband-period_{t}"] | ||||||
|  |         informative[f"%-{coin}close-bb_lower-period_{t}"] = ( | ||||||
|  |             informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"] | ||||||
|  |         ) | ||||||
|  |  | ||||||
|  |         informative[f"%-{coin}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) | ||||||
|  |  | ||||||
|  |         informative[f"%-{coin}relative_volume-period_{t}"] = ( | ||||||
|  |             informative["volume"] / informative["volume"].rolling(t).mean() | ||||||
|  |         ) | ||||||
|  |  | ||||||
|  |     informative[f"%-{coin}pct-change"] = informative["close"].pct_change() | ||||||
|  |     informative[f"%-{coin}raw_volume"] = informative["volume"] | ||||||
|  |     informative[f"%-{coin}raw_price"] = informative["close"] | ||||||
|  |  | ||||||
|  |     indicators = [col for col in informative if col.startswith("%")] | ||||||
|  |     # This loop duplicates and shifts all indicators to add a sense of recency to data | ||||||
|  |     for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): | ||||||
|  |         if n == 0: | ||||||
|  |             continue | ||||||
|  |         informative_shift = informative[indicators].shift(n) | ||||||
|  |         informative_shift = informative_shift.add_suffix("_shift-" + str(n)) | ||||||
|  |         informative = pd.concat((informative, informative_shift), axis=1) | ||||||
|  |  | ||||||
|  |     df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) | ||||||
|  |     skip_columns = [ | ||||||
|  |         (s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] | ||||||
|  |     ] | ||||||
|  |     df = df.drop(columns=skip_columns) | ||||||
|  |     if set_generalized_indicators: | ||||||
|  |         df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7 | ||||||
|  |         df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25 | ||||||
|  |         df["&s-extrema"] = 0 | ||||||
|  |         min_peaks = argrelextrema(df["close"].values, np.less, order=80) | ||||||
|  |         max_peaks = argrelextrema(df["close"].values, np.greater, order=80) | ||||||
|  |         for mp in min_peaks[0]: | ||||||
|  |             df.at[mp, "&s-extrema"] = -1 | ||||||
|  |         for mp in max_peaks[0]: | ||||||
|  |             df.at[mp, "&s-extrema"] = 1 | ||||||
|  |  | ||||||
|  |     return df | ||||||
|  |  | ||||||
|  |  | ||||||
|  | def setup_freqai_spice_rack(config: dict, exchange: Optional[Exchange]) -> Dict[str, Any]: | ||||||
|  |     import difflib | ||||||
|  |     import json | ||||||
|  |     from pathlib import Path | ||||||
|  |     auto_config = config.get('freqai_config', 'lightgbm_config.json') | ||||||
|  |     with open(Path(__file__).parent / Path('spice_rack') / auto_config) as json_file: | ||||||
|  |         freqai_config = json.load(json_file) | ||||||
|  |         config['freqai'] = freqai_config['freqai'] | ||||||
|  |         config['freqai']['identifier'] = config['freqai_identifier'] | ||||||
|  |         corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist'] | ||||||
|  |         timeframes = config['freqai']['feature_parameters']['include_timeframes'] | ||||||
|  |         new_corr_pairs = [] | ||||||
|  |         new_tfs = [] | ||||||
|  |  | ||||||
|  |         if not exchange: | ||||||
|  |             logger.warning('No dataprovider available.') | ||||||
|  |             config['freqai']['enabled'] = False | ||||||
|  |             return config | ||||||
|  |         # find the closest pairs to what the default config wants | ||||||
|  |         for pair in corr_pairs: | ||||||
|  |             closest_pair = difflib.get_close_matches( | ||||||
|  |                                         pair, | ||||||
|  |                                         exchange.markets | ||||||
|  |                                         ) | ||||||
|  |             if not closest_pair: | ||||||
|  |                 logger.warning(f'Could not find {pair} in markets, removing from ' | ||||||
|  |                                f'corr_pairlist.') | ||||||
|  |             else: | ||||||
|  |                 closest_pair = closest_pair[0] | ||||||
|  |  | ||||||
|  |             new_corr_pairs.append(closest_pair) | ||||||
|  |             logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.') | ||||||
|  |  | ||||||
|  |         # find the closest matching timeframes to what the default config wants | ||||||
|  |         if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'): | ||||||
|  |             logger.warning('Default spice rack is designed for lower base timeframes (e.g. > ' | ||||||
|  |                            f'15m). But user passed {config["timeframe"]}.') | ||||||
|  |         new_tfs.append(config['timeframe']) | ||||||
|  |  | ||||||
|  |         list_tfs = [timeframe_to_seconds(tf) for tf | ||||||
|  |                     in exchange.timeframes] | ||||||
|  |         for tf in timeframes: | ||||||
|  |             tf_secs = timeframe_to_seconds(tf) | ||||||
|  |             closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs)) | ||||||
|  |             closest_tf = exchange.timeframes[closest_index] | ||||||
|  |             logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.') | ||||||
|  |             new_tfs.append(closest_tf) | ||||||
|  |  | ||||||
|  |     config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs}) | ||||||
|  |     config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs}) | ||||||
|  |     config.update({"freqaimodel": 'LightGBMRegressor'}) | ||||||
|  |     return config | ||||||
|  |  | ||||||
| # Keep below for when we wish to download heterogeneously lengthed data for FreqAI. | # Keep below for when we wish to download heterogeneously lengthed data for FreqAI. | ||||||
| # def download_all_data_for_training(dp: DataProvider, config: Config) -> None: | # def download_all_data_for_training(dp: DataProvider, config: Config) -> None: | ||||||
| #     """ | #     """ | ||||||
|   | |||||||
| @@ -89,6 +89,10 @@ class Backtesting: | |||||||
|             self._exchange_name, self.config, load_leverage_tiers=True) |             self._exchange_name, self.config, load_leverage_tiers=True) | ||||||
|         self.dataprovider = DataProvider(self.config, self.exchange) |         self.dataprovider = DataProvider(self.config, self.exchange) | ||||||
|  |  | ||||||
|  |         if config.get('freqai_spice_rack', False): | ||||||
|  |             from freqtrade.freqai.utils import setup_freqai_spice_rack | ||||||
|  |             self.config = setup_freqai_spice_rack(self.config, self.exchange) | ||||||
|  |  | ||||||
|         if self.config.get('strategy_list'): |         if self.config.get('strategy_list'): | ||||||
|             if self.config.get('freqai', {}).get('enabled', False): |             if self.config.get('freqai', {}).get('enabled', False): | ||||||
|                 logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies " |                 logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies " | ||||||
|   | |||||||
| @@ -146,12 +146,28 @@ class IStrategy(ABC, HyperStrategyMixin): | |||||||
|                 self._ft_informative.append((informative_data, cls_method)) |                 self._ft_informative.append((informative_data, cls_method)) | ||||||
|  |  | ||||||
|     def load_freqAI_model(self) -> None: |     def load_freqAI_model(self) -> None: | ||||||
|         if self.config.get('freqai', {}).get('enabled', False): |         spice_rack = self.config.get('freqai_spice_rack', False) | ||||||
|  |         if self.config.get('freqai', {}).get('enabled', False) or spice_rack: | ||||||
|  |             if spice_rack: | ||||||
|  |                 from freqtrade.freqai.utils import setup_freqai_spice_rack | ||||||
|  |                 self.config = setup_freqai_spice_rack(self.config, self.dp._exchange) | ||||||
|             # Import here to avoid importing this if freqAI is disabled |             # Import here to avoid importing this if freqAI is disabled | ||||||
|             from freqtrade.freqai.utils import download_all_data_for_training |             from freqtrade.freqai.utils import download_all_data_for_training | ||||||
|             from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver |             from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver | ||||||
|             self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) |             self.freqai = FreqaiModelResolver.load_freqaimodel(self.config) | ||||||
|             self.freqai_info = self.config["freqai"] |             if not self.process_only_new_candles: | ||||||
|  |                 logger.warning('User set process_only_new_candles to false, ' | ||||||
|  |                                'FreqAI requires true. Changing to true.') | ||||||
|  |                 self.process_only_new_candles = True | ||||||
|  |  | ||||||
|  |             if spice_rack: | ||||||
|  |                 import types | ||||||
|  |  | ||||||
|  |                 from freqtrade.freqai.utils import auto_populate_any_indicators | ||||||
|  |                 self.populate_any_indicators = types.MethodType(  # type: ignore | ||||||
|  |                         auto_populate_any_indicators, self) | ||||||
|  |  | ||||||
|  |                 self.freqai_info = self.config["freqai"] | ||||||
|  |  | ||||||
|             # download the desired data in dry/live |             # download the desired data in dry/live | ||||||
|             if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): |             if self.config.get('runmode') in (RunMode.DRY_RUN, RunMode.LIVE): | ||||||
| @@ -161,6 +177,7 @@ class IStrategy(ABC, HyperStrategyMixin): | |||||||
|                     "already on disk." |                     "already on disk." | ||||||
|                 ) |                 ) | ||||||
|                 download_all_data_for_training(self.dp, self.config) |                 download_all_data_for_training(self.dp, self.config) | ||||||
|  |  | ||||||
|         else: |         else: | ||||||
|             # Gracious failures if freqAI is disabled but "start" is called. |             # Gracious failures if freqAI is disabled but "start" is called. | ||||||
|             class DummyClass(): |             class DummyClass(): | ||||||
|   | |||||||
| @@ -29,6 +29,7 @@ nav: | |||||||
|         - Parameter table: freqai-parameter-table.md |         - Parameter table: freqai-parameter-table.md | ||||||
|         - Feature engineering: freqai-feature-engineering.md |         - Feature engineering: freqai-feature-engineering.md | ||||||
|         - Running FreqAI: freqai-running.md |         - Running FreqAI: freqai-running.md | ||||||
|  |         - Spice Rack: freqai-spice-rack.md | ||||||
|         - Developer guide: freqai-developers.md |         - Developer guide: freqai-developers.md | ||||||
|     - Short / Leverage: leverage.md |     - Short / Leverage: leverage.md | ||||||
|     - Utility Sub-commands: utils.md |     - Utility Sub-commands: utils.md | ||||||
|   | |||||||
| @@ -158,3 +158,28 @@ def test_make_train_test_datasets(mocker, freqai_conf): | |||||||
|     assert data_dictionary |     assert data_dictionary | ||||||
|     assert len(data_dictionary) == 7 |     assert len(data_dictionary) == 7 | ||||||
|     assert len(data_dictionary['train_features'].index) == 1916 |     assert len(data_dictionary['train_features'].index) == 1916 | ||||||
|  |  | ||||||
|  |  | ||||||
|  | @pytest.mark.parametrize('indicator', [ | ||||||
|  |     '%-ADArsi-period_10_5m', | ||||||
|  |     'doesnt_exist', | ||||||
|  |     ]) | ||||||
|  | def test_spice_extractor(mocker, freqai_conf, indicator, caplog): | ||||||
|  |     freqai, unfiltered_dataframe = make_unfiltered_dataframe(mocker, freqai_conf) | ||||||
|  |     freqai.dk.find_features(unfiltered_dataframe) | ||||||
|  |  | ||||||
|  |     features_filtered, labels_filtered = freqai.dk.filter_features( | ||||||
|  |             unfiltered_dataframe, | ||||||
|  |             freqai.dk.training_features_list, | ||||||
|  |             freqai.dk.label_list, | ||||||
|  |             training_filter=True, | ||||||
|  |         ) | ||||||
|  |  | ||||||
|  |     vec = freqai.dk.spice_extractor(indicator, features_filtered) | ||||||
|  |     if 'doesnt_exist' in indicator: | ||||||
|  |         assert log_has_re( | ||||||
|  |             "User asked spice_rack for", | ||||||
|  |             caplog, | ||||||
|  |         ) | ||||||
|  |     else: | ||||||
|  |         assert len(vec) == 2860 | ||||||
|   | |||||||
| @@ -1,3 +1,4 @@ | |||||||
|  | import copy | ||||||
| import platform | import platform | ||||||
| import shutil | import shutil | ||||||
| from pathlib import Path | from pathlib import Path | ||||||
| @@ -382,6 +383,31 @@ def test_plot_feature_importance(mocker, freqai_conf): | |||||||
|     shutil.rmtree(Path(freqai.dk.full_path)) |     shutil.rmtree(Path(freqai.dk.full_path)) | ||||||
|  |  | ||||||
|  |  | ||||||
|  | def test_spice_rack(mocker, default_conf, tmpdir, caplog): | ||||||
|  |  | ||||||
|  |     strategy = get_patched_freqai_strategy(mocker, default_conf) | ||||||
|  |     exchange = get_patched_exchange(mocker, default_conf) | ||||||
|  |     strategy.dp = DataProvider(default_conf, exchange) | ||||||
|  |  | ||||||
|  |     default_conf.update({"freqai_spice_rack": "true"}) | ||||||
|  |     default_conf.update({"freqai_identifier": "spicy-id"}) | ||||||
|  |     default_conf["config_files"] = [Path('config_examples', 'config_freqai.example.json')] | ||||||
|  |     default_conf["timerange"] = "20180110-20180115" | ||||||
|  |     default_conf["datadir"] = Path(default_conf["datadir"]) | ||||||
|  |     default_conf['exchange'].update({'pair_whitelist': | ||||||
|  |                                     ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC']}) | ||||||
|  |     default_conf["user_data_dir"] = Path(tmpdir) | ||||||
|  |     freqai_conf = copy.deepcopy(default_conf) | ||||||
|  |  | ||||||
|  |     strategy.config = freqai_conf | ||||||
|  |     strategy.load_freqAI_model() | ||||||
|  |  | ||||||
|  |     assert log_has_re("Spice rack will use LTC/USD", caplog) | ||||||
|  |     assert log_has_re("Spice rack will use 15m", caplog) | ||||||
|  |     assert 'freqai' in freqai_conf | ||||||
|  |     assert strategy.freqai | ||||||
|  |  | ||||||
|  |  | ||||||
| @pytest.mark.parametrize('timeframes,corr_pairs', [ | @pytest.mark.parametrize('timeframes,corr_pairs', [ | ||||||
|     (['5m'], ['ADA/BTC', 'DASH/BTC']), |     (['5m'], ['ADA/BTC', 'DASH/BTC']), | ||||||
|     (['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), |     (['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user