adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' of https://github.com/freqtrade/freqtrade into max-open-trades

Browse Source 
# Conflicts:
#	freqtrade/optimize/backtesting.py
This commit is contained in:
Antonio Della Fortuna 2023-01-11 18:55:57 +01:00
commit 534aa8f7ff
36 changed files with 1361 additions and 707 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
docs/freqai-configuration.md
View File

@ -43,116 +43,113 @@ The FreqAI strategy requires including the following lines of code in the standa
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
# the model will return all labels created by user in `populate_any_indicators` # the model will return all labels created by user in `set_freqai_labels()`
# (& appended targets), an indication of whether or not the prediction should be accepted, # (& appended targets), an indication of whether or not the prediction should be accepted,
# the target mean/std values for each of the labels created by user in # the target mean/std values for each of the labels created by user in
# `populate_any_indicators()` for each training period. # `feature_engineering_*` for each training period.
dataframe = self.freqai.start(dataframe, metadata, self) dataframe = self.freqai.start(dataframe, metadata, self)
return dataframe return dataframe
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
""" """
Function designed to automatically generate, name and merge features *Only functional with FreqAI enabled strategies*
from user indicated timeframes in the configuration file. User controls the indicators This function will automatically expand the defined features on the config defined
passed to the training/prediction by prepending indicators with `'%-' + pair ` `indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and
(see convention below). I.e. user should not prepend any supporting metrics `include_corr_pairs`. In other words, a single feature defined in this function
(e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the will automatically expand to a total of
model. `indicator_periods_candles` * `include_timeframes` * `include_shifted_candles` *
:param pair: pair to be used as informative `include_corr_pairs` numbers of features added to the model.
:param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names All features must be prepended with `%` to be recognized by FreqAI internals.
:param informative: the dataframe associated with the informative pair
:param df: strategy dataframe which will receive the features
:param period: period of the indicator - usage example:
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
""" """
if informative is None: dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
informative = self.dp.get_pair_dataframe(pair, tf) dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period)
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
# first loop is automatically duplicating indicators for time periods return dataframe
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
indicators = [col for col in informative if col.startswith("%")] def feature_engineering_expand_basic(self, dataframe, **kwargs):
# This loop duplicates and shifts all indicators to add a sense of recency to data """
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): *Only functional with FreqAI enabled strategies*
if n == 0: This function will automatically expand the defined features on the config defined
continue `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
informative_shift = informative[indicators].shift(n) In other words, a single feature defined in this function
informative_shift = informative_shift.add_suffix("_shift-" + str(n)) will automatically expand to a total of
informative = pd.concat((informative, informative_shift), axis=1) `include_timeframes` * `include_shifted_candles` * `include_corr_pairs`
numbers of features added to the model.
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) Features defined here will *not* be automatically duplicated on user defined
skip_columns = [ `indicator_periods_candles`
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this All features must be prepended with `%` to be recognized by FreqAI internals.
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
# user adds targets here by prepending them with &- (see convention below) :param df: strategy dataframe which will receive the features
# If user wishes to use multiple targets, a multioutput prediction model dataframe["%-pct-change"] = dataframe["close"].pct_change()
# needs to be used such as templates/CatboostPredictionMultiModel.py dataframe["%-ema-200"] = ta.EMA(dataframe, timeperiod=200)
df["&-s_close"] = ( """
df["close"] dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-raw_volume"] = dataframe["volume"]
dataframe["%-raw_price"] = dataframe["close"]
return dataframe
def feature_engineering_standard(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This optional function will be called once with the dataframe of the base timeframe.
This is the final function to be called, which means that the dataframe entering this
function will contain all the features and columns created by all other
freqai_feature_engineering_* functions.
This function is a good place to do custom exotic feature extractions (e.g. tsfresh).
This function is a good place for any feature that should not be auto-expanded upon
(e.g. day of the week).
All features must be prepended with `%` to be recognized by FreqAI internals.
:param df: strategy dataframe which will receive the features
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
"""
dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
dataframe["%-hour_of_day"] = (dataframe["date"].dt.hour + 1) / 25
return dataframe
def set_freqai_targets(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
dataframe["&-s_close"] = (
dataframe["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean() .mean()
/ df["close"] / dataframe["close"]
- 1 - 1
) )
return df
``` ```
Notice how the `populate_any_indicators()` is where [features](freqai-feature-engineering.md#feature-engineering) and labels/targets are added. A full example strategy is available in `templates/FreqaiExampleStrategy.py`. Notice how the `feature_engineering_*()` is where [features](freqai-feature-engineering.md#feature-engineering) are added. Meanwhile `set_freqai_targets()` adds the labels/targets. A full example strategy is available in `templates/FreqaiExampleStrategy.py`.
Notice also the location of the labels under `if set_generalized_indicators:` at the bottom of the example. This is where single features and labels/targets should be added to the feature set to avoid duplication of them from various configuration parameters that multiply the feature set, such as `include_timeframes`.
!!! Note !!! Note
The `self.freqai.start()` function cannot be called outside the `populate_indicators()`. The `self.freqai.start()` function cannot be called outside the `populate_indicators()`.
!!! Note !!! Note
Features **must** be defined in `populate_any_indicators()`. Defining FreqAI features in `populate_indicators()` Features **must** be defined in `feature_engineering_*()`. Defining FreqAI features in `populate_indicators()`
will cause the algorithm to fail in live/dry mode. In order to add generalized features that are not associated with a specific pair or timeframe, the following structure inside `populate_any_indicators()` should be used will cause the algorithm to fail in live/dry mode. In order to add generalized features that are not associated with a specific pair or timeframe, you should use `feature_engineering_standard()`
(as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`): (as exemplified in `freqtrade/templates/FreqaiExampleStrategy.py`).
```python
def populate_any_indicators(self, pair, df, tf, informative=None, set_generalized_indicators=False):
...
# Add generalized indicators here (because in live, it will call only this function to populate
# indicators for retraining). Notice how we ensure not to add them multiple times by associating
# these generalized indicators to the basepair/timeframe
if set_generalized_indicators:
df['%-day_of_week'] = (df["date"].dt.dayofweek + 1) / 7
df['%-hour_of_day'] = (df['date'].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below)
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py
df["&-s_close"] = (
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean()
/ df["close"]
- 1
)
```
Please see the example script located in `freqtrade/templates/FreqaiExampleStrategy.py` for a full example of `populate_any_indicators()`.
## Important dataframe key patterns ## Important dataframe key patterns
@ -160,11 +157,11 @@ Below are the values you can expect to include/use inside a typical strategy dat
| DataFrame Key | Description | | DataFrame Key | Description |
|------------|-------------| |------------|-------------|
| `df['&*']` | Any dataframe column prepended with `&` in `populate_any_indicators()` is treated as a training target (label) inside FreqAI (typically following the naming convention `&-s*`). For example, to predict the close price 40 candles into the future, you would set `df['&-s_close'] = df['close'].shift(-self.freqai_info["feature_parameters"]["label_period_candles"])` with `"label_period_candles": 40` in the config. FreqAI makes the predictions and gives them back under the same key (`df['&-s_close']`) to be used in `populate_entry/exit_trend()`. <br> **Datatype:** Depends on the output of the model. | `df['&*']` | Any dataframe column prepended with `&` in `set_freqai_targets()` is treated as a training target (label) inside FreqAI (typically following the naming convention `&-s*`). For example, to predict the close price 40 candles into the future, you would set `df['&-s_close'] = df['close'].shift(-self.freqai_info["feature_parameters"]["label_period_candles"])` with `"label_period_candles": 40` in the config. FreqAI makes the predictions and gives them back under the same key (`df['&-s_close']`) to be used in `populate_entry/exit_trend()`. <br> **Datatype:** Depends on the output of the model.
| `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`). <br> **Datatype:** Float. | `df['&*_std/mean']` | Standard deviation and mean values of the defined labels during training (or live tracking with `fit_live_predictions_candles`). Commonly used to understand the rarity of a prediction (use the z-score as shown in `templates/FreqaiExampleStrategy.py` and explained [here](#creating-a-dynamic-target-threshold) to evaluate how often a particular prediction was observed during training or historically with `fit_live_predictions_candles`). <br> **Datatype:** Float.
| `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers()` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -2 and 2. | `df['do_predict']` | Indication of an outlier data point. The return value is integer between -2 and 2, which lets you know if the prediction is trustworthy or not. `do_predict==1` means that the prediction is trustworthy. If the Dissimilarity Index (DI, see details [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di)) of the input data point is above the threshold defined in the config, FreqAI will subtract 1 from `do_predict`, resulting in `do_predict==0`. If `use_SVM_to_remove_outliers()` is active, the Support Vector Machine (SVM, see details [here](freqai-feature-engineering.md#identifying-outliers-using-a-support-vector-machine-svm)) may also detect outliers in training and prediction data. In this case, the SVM will also subtract 1 from `do_predict`. If the input data point was considered an outlier by the SVM but not by the DI, or vice versa, the result will be `do_predict==0`. If both the DI and the SVM considers the input data point to be an outlier, the result will be `do_predict==-1`. As with the SVM, if `use_DBSCAN_to_remove_outliers` is active, DBSCAN (see details [here](freqai-feature-engineering.md#identifying-outliers-with-dbscan)) may also detect outliers and subtract 1 from `do_predict`. Hence, if both the SVM and DBSCAN are active and identify a datapoint that was above the DI threshold as an outlier, the result will be `do_predict==-2`. A particular case is when `do_predict == 2`, which means that the model has expired due to exceeding `expired_hours`. <br> **Datatype:** Integer between -2 and 2.
| `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float. | `df['DI_values']` | Dissimilarity Index (DI) values are proxies for the level of confidence FreqAI has in the prediction. A lower DI means the prediction is close to the training data, i.e., higher prediction confidence. See details about the DI [here](freqai-feature-engineering.md#identifying-outliers-with-the-dissimilarity-index-di). <br> **Datatype:** Float.
| `df['%*']` | Any dataframe column prepended with `%` in `populate_any_indicators()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model. | `df['%*']` | Any dataframe column prepended with `%` in `feature_engineering_*()` is treated as a training feature. For example, you can include the RSI in the training feature set (similar to in `templates/FreqaiExampleStrategy.py`) by setting `df['%-rsi']`. See more details on how this is done [here](freqai-feature-engineering.md). <br> **Note:** Since the number of features prepended with `%` can multiply very quickly (10s of thousands of features are easily engineered using the multiplictative functionality of, e.g., `include_shifted_candles` and `include_timeframes` as described in the [parameter table](freqai-parameter-table.md)), these features are removed from the dataframe that is returned from FreqAI to the strategy. To keep a particular type of feature for plotting purposes, you would prepend it with `%%`. <br> **Datatype:** Depends on the output of the model.
## Setting the `startup_candle_count` ## Setting the `startup_candle_count`

168
docs/freqai-feature-engineering.md
View File

@ -2,96 +2,130 @@
## Defining the features ## Defining the features
Low level feature engineering is performed in the user strategy within a function called `populate_any_indicators()`. That function sets the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. One important syntax rule is that all `base features` string names are prepended with `%-{pair}`, while labels/targets are prepended with `&`. Low level feature engineering is performed in the user strategy within a set of functions called `feature_engineering_*`. These function set the `base features` such as, `RSI`, `MFI`, `EMA`, `SMA`, time of day, volume, etc. The `base features` can be custom indicators or they can be imported from any technical-analysis library that you can find. FreqAI is equipped with a set of functions to simplify rapid large-scale feature engineering:
!!! Note | Function | Description |
Adding the full pair string, e.g. XYZ/USD, in the feature name enables improved performance for dataframe caching on the backend. If you decide *not* to add the full pair string in the feature string, FreqAI will operate in a reduced performance mode. |---------------|-------------|
| `feature_engineering__expand_all()` | This optional function will automatically expand the defined features on the config defined `indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
| `feature_engineering__expand_basic()` | This optional function will automatically expand the defined features on the config defined `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`. Note: this function does *not* expand across `include_periods_candles`.
| `feature_engineering_standard()` | This optional function will be called once with the dataframe of the base timeframe. This is the final function to be called, which means that the dataframe entering this function will contain all the features and columns from the base asset created by the other `feature_engineering_expand` functions. This function is a good place to do custom exotic feature extractions (e.g. tsfresh). This function is also a good place for any feature that should not be auto-expanded upon (e.g. day of the week).
| `set_freqai_targets()` | Required function to set the targets for the model. All targets must be prepended with `&` to be recognized by the FreqAI internals.
Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles." Meanwhile, high level feature engineering is handled within `"feature_parameters":{}` in the FreqAI config. Within this file, it is possible to decide large scale feature expansions on top of the `base_features` such as "including correlated pairs" or "including informative timeframes" or even "including recent candles."
It is advisable to start from the template `populate_any_indicators()` in the source provided example strategy (found in `templates/FreqaiExampleStrategy.py`) to ensure that the feature definitions are following the correct conventions. Here is an example of how to set the indicators and labels in the strategy: It is advisable to start from the template `feature_engineering_*` functions in the source provided example strategy (found in `templates/FreqaiExampleStrategy.py`) to ensure that the feature definitions are following the correct conventions. Here is an example of how to set the indicators and labels in the strategy:
```python ```python
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
""" """
Function designed to automatically generate, name, and merge features *Only functional with FreqAI enabled strategies*
from user-indicated timeframes in the configuration file. The user controls the indicators This function will automatically expand the defined features on the config defined
passed to the training/prediction by prepending indicators with `'%-' + pair ` `indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and
(see convention below). I.e., the user should not prepend any supporting metrics `include_corr_pairs`. In other words, a single feature defined in this function
(e.g., bb_lowerband below) with % unless they explicitly want to pass that metric to the will automatically expand to a total of
model. `indicator_periods_candles` * `include_timeframes` * `include_shifted_candles` *
:param pair: pair to be used as informative `include_corr_pairs` numbers of features added to the model.
:param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names All features must be prepended with `%` to be recognized by FreqAI internals.
:param informative: the dataframe associated with the informative pair
:param df: strategy dataframe which will receive the features
:param period: period of the indicator - usage example:
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
""" """
if informative is None: dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
informative = self.dp.get_pair_dataframe(pair, tf) dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
# first loop is automatically duplicating indicators for time periods dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period)
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
bollinger = qtpylib.bollinger_bands( bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2 qtpylib.typical_price(dataframe), window=period, stds=2.2
) )
informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] dataframe["bb_lowerband-period"] = bollinger["lower"]
informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] dataframe["bb_middleband-period"] = bollinger["mid"]
informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] dataframe["bb_upperband-period"] = bollinger["upper"]
informative[f"%-{pair}bb_width-period_{t}"] = ( dataframe["%-bb_width-period"] = (
informative[f"{pair}bb_upperband-period_{t}"] dataframe["bb_upperband-period"]
- informative[f"{pair}bb_lowerband-period_{t}"] - dataframe["bb_lowerband-period"]
) / informative[f"{pair}bb_middleband-period_{t}"] ) / dataframe["bb_middleband-period"]
informative[f"%-{pair}close-bb_lower-period_{t}"] = ( dataframe["%-close-bb_lower-period"] = (
informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] dataframe["close"] / dataframe["bb_lowerband-period"]
) )
informative[f"%-{pair}relative_volume-period_{t}"] = ( dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period)
informative["volume"] / informative["volume"].rolling(t).mean()
dataframe["%-relative_volume-period"] = (
dataframe["volume"] / dataframe["volume"].rolling(period).mean()
) )
indicators = [col for col in informative if col.startswith("%")] return dataframe
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) def feature_engineering_expand_basic(self, dataframe, **kwargs):
skip_columns = [ """
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] *Only functional with FreqAI enabled strategies*
] This function will automatically expand the defined features on the config defined
df = df.drop(columns=skip_columns) `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
In other words, a single feature defined in this function
will automatically expand to a total of
`include_timeframes` * `include_shifted_candles` * `include_corr_pairs`
numbers of features added to the model.
# Add generalized indicators here (because in live, it will call this Features defined here will *not* be automatically duplicated on user defined
# function to populate indicators during training). Notice how we ensure not to `indicator_periods_candles`
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) All features must be prepended with `%` to be recognized by FreqAI internals.
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py :param df: strategy dataframe which will receive the features
df["&-s_close"] = ( dataframe["%-pct-change"] = dataframe["close"].pct_change()
df["close"] dataframe["%-ema-200"] = ta.EMA(dataframe, timeperiod=200)
"""
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-raw_volume"] = dataframe["volume"]
dataframe["%-raw_price"] = dataframe["close"]
return dataframe
def feature_engineering_standard(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This optional function will be called once with the dataframe of the base timeframe.
This is the final function to be called, which means that the dataframe entering this
function will contain all the features and columns created by all other
freqai_feature_engineering_* functions.
This function is a good place to do custom exotic feature extractions (e.g. tsfresh).
This function is a good place for any feature that should not be auto-expanded upon
(e.g. day of the week).
All features must be prepended with `%` to be recognized by FreqAI internals.
:param df: strategy dataframe which will receive the features
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
"""
dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
dataframe["%-hour_of_day"] = (dataframe["date"].dt.hour + 1) / 25
return dataframe
def set_freqai_targets(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
dataframe["&-s_close"] = (
dataframe["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean() .mean()
/ df["close"] / dataframe["close"]
- 1 - 1
) )
return df return dataframe
``` ```
In the presented example, the user does not wish to pass the `bb_lowerband` as a feature to the model, In the presented example, the user does not wish to pass the `bb_lowerband` as a feature to the model,
@ -118,13 +152,13 @@ After having defined the `base features`, the next step is to expand upon them u
} }
``` ```
The `include_timeframes` in the config above are the timeframes (`tf`) of each call to `populate_any_indicators()` in the strategy. In the presented case, the user is asking for the `5m`, `15m`, and `4h` timeframes of the `rsi`, `mfi`, `roc`, and `bb_width` to be included in the feature set. The `include_timeframes` in the config above are the timeframes (`tf`) of each call to `feature_engineering_expand_*()` in the strategy. In the presented case, the user is asking for the `5m`, `15m`, and `4h` timeframes of the `rsi`, `mfi`, `roc`, and `bb_width` to be included in the feature set.
You can ask for each of the defined features to be included also for informative pairs using the `include_corr_pairlist`. This means that the feature set will include all the features from `populate_any_indicators` on all the `include_timeframes` for each of the correlated pairs defined in the config (`ETH/USD`, `LINK/USD`, and `BNB/USD` in the presented example). You can ask for each of the defined features to be included also for informative pairs using the `include_corr_pairlist`. This means that the feature set will include all the features from `feature_engineering_expand_*()` on all the `include_timeframes` for each of the correlated pairs defined in the config (`ETH/USD`, `LINK/USD`, and `BNB/USD` in the presented example).
`include_shifted_candles` indicates the number of previous candles to include in the feature set. For example, `include_shifted_candles: 2` tells FreqAI to include the past 2 candles for each of the features in the feature set. `include_shifted_candles` indicates the number of previous candles to include in the feature set. For example, `include_shifted_candles: 2` tells FreqAI to include the past 2 candles for each of the features in the feature set.
In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `populate_any_indicators()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles` In total, the number of features the user of the presented example strat has created is: length of `include_timeframes` * no. features in `feature_engineering_expand_*()` * length of `include_corr_pairlist` * no. `include_shifted_candles` * length of `indicator_periods_candles`
$= 3 * 3 * 3 * 2 * 2 = 108$. $= 3 * 3 * 3 * 2 * 2 = 108$.
### Returning additional info from training ### Returning additional info from training

8
docs/freqai-parameter-table.md
View File

@ -29,12 +29,12 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
|------------|-------------| |------------|-------------|
| | **Feature parameters within the `freqai.feature_parameters` sub dictionary** | | **Feature parameters within the `freqai.feature_parameters` sub dictionary**
| `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md). <br> **Datatype:** Dictionary. | `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md). <br> **Datatype:** Dictionary.
| `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base indicators dataset. <br> **Datatype:** List of timeframes (strings). | `include_timeframes` | A list of timeframes that all indicators in `feature_engineering_expand_*()` will be created for. The list is added as features to the base indicators dataset. <br> **Datatype:** List of timeframes (strings).
| `include_corr_pairlist` | A list of correlated coins that FreqAI will add as additional features to all `pair_whitelist` coins. All indicators set in `populate_any_indicators` during feature engineering (see details [here](freqai-feature-engineering.md)) will be created for each correlated coin. The correlated coins features are added to the base indicators dataset. <br> **Datatype:** List of assets (strings). | `include_corr_pairlist` | A list of correlated coins that FreqAI will add as additional features to all `pair_whitelist` coins. All indicators set in `feature_engineering_expand_*()` during feature engineering (see details [here](freqai-feature-engineering.md)) will be created for each correlated coin. The correlated coins features are added to the base indicators dataset. <br> **Datatype:** List of assets (strings).
| `label_period_candles` | Number of candles into the future that the labels are created for. This is used in `populate_any_indicators` (see `templates/FreqaiExampleStrategy.py` for detailed usage). You can create custom labels and choose whether to make use of this parameter or not. <br> **Datatype:** Positive integer. | `label_period_candles` | Number of candles into the future that the labels are created for. This is used in `feature_engineering_expand_all()` (see `templates/FreqaiExampleStrategy.py` for detailed usage). You can create custom labels and choose whether to make use of this parameter or not. <br> **Datatype:** Positive integer.
| `include_shifted_candles` | Add features from previous candles to subsequent candles with the intent of adding historical information. If used, FreqAI will duplicate and shift all features from the `include_shifted_candles` previous candles so that the information is available for the subsequent candle. <br> **Datatype:** Positive integer. | `include_shifted_candles` | Add features from previous candles to subsequent candles with the intent of adding historical information. If used, FreqAI will duplicate and shift all features from the `include_shifted_candles` previous candles so that the information is available for the subsequent candle. <br> **Datatype:** Positive integer.
| `weight_factor` | Weight training data points according to their recency (see details [here](freqai-feature-engineering.md#weighting-features-for-temporal-importance)). <br> **Datatype:** Positive float (typically < 1). | `weight_factor` | Weight training data points according to their recency (see details [here](freqai-feature-engineering.md#weighting-features-for-temporal-importance)). <br> **Datatype:** Positive float (typically < 1).
| `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `populate_any_indicators()` for indicator creation. FreqAI uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN. <br> **Datatype:** Positive integer. | `indicator_max_period_candles` | **No longer used (#7325)**. Replaced by `startup_candle_count` which is set in the [strategy](freqai-configuration.md#building-a-freqai-strategy). `startup_candle_count` is timeframe independent and defines the maximum *period* used in `feature_engineering_*()` for indicator creation. FreqAI uses this parameter together with the maximum timeframe in `include_time_frames` to calculate how many data points to download such that the first data point does not include a NaN. <br> **Datatype:** Positive integer.
| `indicator_periods_candles` | Time periods to calculate indicators for. The indicators are added to the base indicator dataset. <br> **Datatype:** List of positive integers. | `indicator_periods_candles` | Time periods to calculate indicators for. The indicators are added to the base indicator dataset. <br> **Datatype:** List of positive integers.
| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) <br> **Datatype:** Boolean. <br> Default: `False`. | `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) <br> **Datatype:** Boolean. <br> Default: `False`.
| `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. Plot is stored in `user_data/models/<identifier>/sub-train-<COIN>_<timestamp>.html`. <br> **Datatype:** Integer. <br> Default: `0`. | `plot_feature_importances` | Create a feature importance plot for each model for the top/bottom `plot_feature_importances` number of features. Plot is stored in `user_data/models/<identifier>/sub-train-<COIN>_<timestamp>.html`. <br> **Datatype:** Integer. <br> Default: `0`.

52
docs/freqai-reinforcement-learning.md
View File

@ -34,55 +34,25 @@ Setting up and running a Reinforcement Learning model is the same as running a R
freqtrade trade --freqaimodel ReinforcementLearner --strategy MyRLStrategy --config config.json freqtrade trade --freqaimodel ReinforcementLearner --strategy MyRLStrategy --config config.json
``` ```
where `ReinforcementLearner` will use the templated `ReinforcementLearner` from `freqai/prediction_models/ReinforcementLearner` (or a custom user defined one located in `user_data/freqaimodels`). The strategy, on the other hand, follows the same base [feature engineering](freqai-feature-engineering.md) with `populate_any_indicators` as a typical Regressor: where `ReinforcementLearner` will use the templated `ReinforcementLearner` from `freqai/prediction_models/ReinforcementLearner` (or a custom user defined one located in `user_data/freqaimodels`). The strategy, on the other hand, follows the same base [feature engineering](freqai-feature-engineering.md) with `feature_engineering_*` as a typical Regressor. The difference lies in the creation of the targets, Reinforcement Learning doesnt require them. However, FreqAI requires a default (neutral) value to be set in the action column:
```python ```python
def populate_any_indicators( def set_freqai_targets(self, dataframe, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False """
): *Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
if informative is None: More details about feature engineering available:
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods https://www.freqtrade.io/en/latest/freqai-feature-engineering
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, window=t)
# The following raw price values are necessary for RL models
informative[f"%-{pair}raw_close"] = informative["close"]
informative[f"%-{pair}raw_open"] = informative["open"]
informative[f"%-{pair}raw_high"] = informative["high"]
informative[f"%-{pair}raw_low"] = informative["low"]
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
# For RL, there are no direct targets to set. This is filler (neutral) # For RL, there are no direct targets to set. This is filler (neutral)
# until the agent sends an action. # until the agent sends an action.
df["&-action"] = 0 df["&-action"] = 0
return df
``` ```
Most of the function remains the same as for typical Regressors, however, the function above shows how the strategy must pass the raw price data to the agent so that it has access to raw OHLCV in the training environment: Most of the function remains the same as for typical Regressors, however, the function above shows how the strategy must pass the raw price data to the agent so that it has access to raw OHLCV in the training environment:

6
docs/freqai-running.md
View File

@ -67,6 +67,10 @@ Backtesting mode requires [downloading the necessary data](#downloading-data-to-
*want* to retrain a new model with the same config file, you should simply change the `identifier`. *want* to retrain a new model with the same config file, you should simply change the `identifier`.
This way, you can return to using any model you wish by simply specifying the `identifier`. This way, you can return to using any model you wish by simply specifying the `identifier`.
!!! Note
Backtesting calls `set_freqai_targets()` one time for each backtest window (where the number of windows is the full backtest timerange divided by the `backtest_period_days` parameter). Doing this means that the targets simulate dry/live behavior without look ahead bias. However, the definition of the features in `feature_engineering_*()` is performed once on the entire backtest timerange. This means that you should be sure that features do look-ahead into the future.
More details about look-ahead bias can be found in [Common Mistakes](strategy-customization.md#common-mistakes-when-developing-strategies).
--- ---
### Saving prediction data ### Saving prediction data
@ -135,7 +139,7 @@ freqtrade hyperopt --hyperopt-loss SharpeHyperOptLoss --strategy FreqaiExampleSt
`hyperopt` requires you to have the data pre-downloaded in the same fashion as if you were doing [backtesting](#backtesting). In addition, you must consider some restrictions when trying to hyperopt FreqAI strategies: `hyperopt` requires you to have the data pre-downloaded in the same fashion as if you were doing [backtesting](#backtesting). In addition, you must consider some restrictions when trying to hyperopt FreqAI strategies:
- The `--analyze-per-epoch` hyperopt parameter is not compatible with FreqAI. - The `--analyze-per-epoch` hyperopt parameter is not compatible with FreqAI.
- It's not possible to hyperopt indicators in the `populate_any_indicators()` function. This means that you cannot optimize model parameters using hyperopt. Apart from this exception, it is possible to optimize all other [spaces](hyperopt.md#running-hyperopt-with-smaller-search-space). - It's not possible to hyperopt indicators in the `feature_engineering_*()` and `set_freqai_targets()` functions. This means that you cannot optimize model parameters using hyperopt. Apart from this exception, it is possible to optimize all other [spaces](hyperopt.md#running-hyperopt-with-smaller-search-space).
- The backtesting instructions also apply to hyperopt. - The backtesting instructions also apply to hyperopt.
The best method for combining hyperopt and FreqAI is to focus on hyperopting entry/exit thresholds/criteria. You need to focus on hyperopting parameters that are not used in your features. For example, you should not try to hyperopt rolling window lengths in the feature creation, or any part of the FreqAI config which changes predictions. In order to efficiently hyperopt the FreqAI strategy, FreqAI stores predictions as dataframes and reuses them. Hence the requirement to hyperopt entry/exit thresholds/criteria only. The best method for combining hyperopt and FreqAI is to focus on hyperopting entry/exit thresholds/criteria. You need to focus on hyperopting parameters that are not used in your features. For example, you should not try to hyperopt rolling window lengths in the feature creation, or any part of the FreqAI config which changes predictions. In order to efficiently hyperopt the FreqAI strategy, FreqAI stores predictions as dataframes and reuses them. Hence the requirement to hyperopt entry/exit thresholds/criteria only.

2
docs/requirements-docs.txt
View File

@ -1,6 +1,6 @@
markdown==3.3.7 markdown==3.3.7
mkdocs==1.4.2 mkdocs==1.4.2
mkdocs-material==8.5.11 mkdocs-material==9.0.3
mdx_truly_sane_lists==1.3 mdx_truly_sane_lists==1.3
pymdown-extensions==9.9 pymdown-extensions==9.9
jinja2==3.1.2 jinja2==3.1.2

251
docs/strategy_migration.md
View File

@ -477,3 +477,254 @@ after:
"ignore_buying_expired_candle_after": 120 "ignore_buying_expired_candle_after": 120
} }
``` ```
## FreqAI strategy
The `populate_any_indicators()` method has been split into `feature_engineering_expand_all()`, `feature_engineering_expand_basic()`, `feature_engineering_standard()` and`set_freqai_targets()`.
For each new function, the pair (and timeframe where necessary) will be automatically added to the column.
As such, the definition of features becomes much simpler with the new logic.
For a full explanation of each method, please go to the corresponding [freqAI documentation page](freqai-feature-engineering.md#defining-the-features)
``` python linenums="1" hl_lines="12-37 39-42 63-65 67-75"
def populate_any_indicators(
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2
)
informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"]
informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"]
informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"]
informative[f"%-{pair}bb_width-period_{t}"] = (
informative[f"{pair}bb_upperband-period_{t}"]
- informative[f"{pair}bb_lowerband-period_{t}"]
) / informative[f"{pair}bb_middleband-period_{t}"]
informative[f"%-{pair}close-bb_lower-period_{t}"] = (
informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"]
)
informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
informative[f"%-{pair}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean()
) # (1)
informative[f"%-{pair}pct-change"] = informative["close"].pct_change()
informative[f"%-{pair}raw_volume"] = informative["volume"]
informative[f"%-{pair}raw_price"] = informative["close"]
# (2)
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True)
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# (3)
# user adds targets here by prepending them with &- (see convention below)
df["&-s_close"] = (
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean()
/ df["close"]
- 1
) # (4)
return df
```
1. Features - Move to `feature_engineering_expand_all`
2. Basic features, not expanded across `include_periods_candles` - move to`feature_engineering_expand_basic()`.
3. Standard features which should not be expanded - move to `feature_engineering_standard()`.
4. Targets - Move this part to `set_freqai_targets()`.
### freqai - feature engineering expand all
Features will now expand automatically. As such, the expansion loops, as well as the `{pair}` / `{timeframe}` parts will need to be removed.
``` python linenums="1"
def feature_engineering_expand_all(self, dataframe, period, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This function will automatically expand the defined features on the config defined
`indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and
`include_corr_pairs`. In other words, a single feature defined in this function
will automatically expand to a total of
`indicator_periods_candles` * `include_timeframes` * `include_shifted_candles` *
`include_corr_pairs` numbers of features added to the model.
All features must be prepended with `%` to be recognized by FreqAI internals.
More details on how these config defined parameters accelerate feature engineering
in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
:param period: period of the indicator - usage example:
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
"""
dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period)
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(dataframe), window=period, stds=2.2
)
dataframe["bb_lowerband-period"] = bollinger["lower"]
dataframe["bb_middleband-period"] = bollinger["mid"]
dataframe["bb_upperband-period"] = bollinger["upper"]
dataframe["%-bb_width-period"] = (
dataframe["bb_upperband-period"]
- dataframe["bb_lowerband-period"]
) / dataframe["bb_middleband-period"]
dataframe["%-close-bb_lower-period"] = (
dataframe["close"] / dataframe["bb_lowerband-period"]
)
dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period)
dataframe["%-relative_volume-period"] = (
dataframe["volume"] / dataframe["volume"].rolling(period).mean()
)
return dataframe
```
### Freqai - feature engineering basic
Basic features. Make sure to remove the `{pair}` part from your features.
``` python linenums="1"
def feature_engineering_expand_basic(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This function will automatically expand the defined features on the config defined
`include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
In other words, a single feature defined in this function
will automatically expand to a total of
`include_timeframes` * `include_shifted_candles` * `include_corr_pairs`
numbers of features added to the model.
Features defined here will *not* be automatically duplicated on user defined
`indicator_periods_candles`
All features must be prepended with `%` to be recognized by FreqAI internals.
More details on how these config defined parameters accelerate feature engineering
in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-ema-200"] = ta.EMA(dataframe, timeperiod=200)
"""
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-raw_volume"] = dataframe["volume"]
dataframe["%-raw_price"] = dataframe["close"]
return dataframe
```
### FreqAI - feature engineering standard
``` python linenums="1"
def feature_engineering_standard(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This optional function will be called once with the dataframe of the base timeframe.
This is the final function to be called, which means that the dataframe entering this
function will contain all the features and columns created by all other
freqai_feature_engineering_* functions.
This function is a good place to do custom exotic feature extractions (e.g. tsfresh).
This function is a good place for any feature that should not be auto-expanded upon
(e.g. day of the week).
All features must be prepended with `%` to be recognized by FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the features
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
"""
dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
return dataframe
```
### FreqAI - set Targets
Targets now get their own, dedicated method.
``` python linenums="1"
def set_freqai_targets(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
dataframe["&-s_close"] = (
dataframe["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean()
/ dataframe["close"]
- 1
)
return dataframe
```

36
freqtrade/freqai/RL/BaseReinforcementLearningModel.py
View File

@ -280,26 +280,36 @@ class BaseReinforcementLearningModel(IFreqaiModel):
train_df = data_dictionary["train_features"] train_df = data_dictionary["train_features"]
test_df = data_dictionary["test_features"] test_df = data_dictionary["test_features"]
# %-raw_volume_gen_shift-2_ETH/USDT_1h
# price data for model training and evaluation # price data for model training and evaluation
tf = self.config['timeframe'] tf = self.config['timeframe']
ohlc_list = [f'%-{pair}raw_open_{tf}', f'%-{pair}raw_low_{tf}', rename_dict = {'%-raw_open': 'open', '%-raw_low': 'low',
f'%-{pair}raw_high_{tf}', f'%-{pair}raw_close_{tf}'] '%-raw_high': ' high', '%-raw_close': 'close'}
rename_dict = {f'%-{pair}raw_open_{tf}': 'open', f'%-{pair}raw_low_{tf}': 'low', rename_dict_old = {f'%-{pair}raw_open_{tf}': 'open', f'%-{pair}raw_low_{tf}': 'low',
f'%-{pair}raw_high_{tf}': ' high', f'%-{pair}raw_close_{tf}': 'close'} f'%-{pair}raw_high_{tf}': ' high', f'%-{pair}raw_close_{tf}': 'close'}
prices_train = train_df.filter(ohlc_list, axis=1) prices_train = train_df.filter(rename_dict.keys(), axis=1)
if prices_train.empty: prices_train_old = train_df.filter(rename_dict_old.keys(), axis=1)
raise OperationalException('Reinforcement learning module didnt find the raw prices ' if prices_train.empty or not prices_train_old.empty:
'assigned in populate_any_indicators. Please assign them ' if not prices_train_old.empty:
'with:\n' prices_train = prices_train_old
'informative[f"%-{pair}raw_close"] = informative["close"]\n' rename_dict = rename_dict_old
'informative[f"%-{pair}raw_open"] = informative["open"]\n' logger.warning('Reinforcement learning module didnt find the correct raw prices '
'informative[f"%-{pair}raw_high"] = informative["high"]\n' 'assigned in feature_engineering_standard(). '
'informative[f"%-{pair}raw_low"] = informative["low"]\n') 'Please assign them with:\n'
'dataframe["%-raw_close"] = dataframe["close"]\n'
'dataframe["%-raw_open"] = dataframe["open"]\n'
'dataframe["%-raw_high"] = dataframe["high"]\n'
'dataframe["%-raw_low"] = dataframe["low"]\n'
'inside `feature_engineering_standard()')
elif prices_train.empty:
raise OperationalException("No prices found, please follow log warning "
"instructions to correct the strategy.")
prices_train.rename(columns=rename_dict, inplace=True) prices_train.rename(columns=rename_dict, inplace=True)
prices_train.reset_index(drop=True) prices_train.reset_index(drop=True)
prices_test = test_df.filter(ohlc_list, axis=1) prices_test = test_df.filter(rename_dict.keys(), axis=1)
prices_test.rename(columns=rename_dict, inplace=True) prices_test.rename(columns=rename_dict, inplace=True)
prices_test.reset_index(drop=True) prices_test.reset_index(drop=True)

187
freqtrade/freqai/data_kitchen.py
View File

@ -1,4 +1,5 @@
import copy import copy
import inspect
import logging import logging
import shutil import shutil
from datetime import datetime, timezone from datetime import datetime, timezone
@ -23,6 +24,7 @@ from freqtrade.constants import Config
from freqtrade.data.converter import reduce_dataframe_footprint from freqtrade.data.converter import reduce_dataframe_footprint
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange import timeframe_to_seconds
from freqtrade.strategy import merge_informative_pair
from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.interface import IStrategy
@ -1145,9 +1147,9 @@ class FreqaiDataKitchen:
for pair in pairs: for pair in pairs:
pair = pair.replace(':', '') # lightgbm doesnt like colons pair = pair.replace(':', '') # lightgbm doesnt like colons
valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] pair_cols = [col for col in dataframe.columns if col.startswith("%")
pair_cols = [col for col in dataframe.columns if and f"{pair}_" in col]
any(substr in col for substr in valid_strs)]
if pair_cols: if pair_cols:
pair_cols.insert(0, 'date') pair_cols.insert(0, 'date')
corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
@ -1176,6 +1178,103 @@ class FreqaiDataKitchen:
return dataframe return dataframe
def get_pair_data_for_features(self,
pair: str,
tf: str,
strategy: IStrategy,
corr_dataframes: dict = {},
base_dataframes: dict = {},
is_corr_pairs: bool = False) -> DataFrame:
"""
Get the data for the pair. If it's not in the dictionary, get it from the data provider
:param pair: str = pair to get data for
:param tf: str = timeframe to get data for
:param strategy: IStrategy = user defined strategy object
:param corr_dataframes: dict = dict containing the df pair dataframes
(for user defined timeframes)
:param base_dataframes: dict = dict containing the current pair dataframes
(for user defined timeframes)
:param is_corr_pairs: bool = whether the pair is a corr pair or not
:return: dataframe = dataframe containing the pair data
"""
if is_corr_pairs:
dataframe = corr_dataframes[pair][tf]
if not dataframe.empty:
return dataframe
else:
dataframe = strategy.dp.get_pair_dataframe(pair=pair, timeframe=tf)
return dataframe
else:
dataframe = base_dataframes[tf]
if not dataframe.empty:
return dataframe
else:
dataframe = strategy.dp.get_pair_dataframe(pair=pair, timeframe=tf)
return dataframe
def merge_features(self, df_main: DataFrame, df_to_merge: DataFrame,
tf: str, timeframe_inf: str, suffix: str) -> DataFrame:
"""
Merge the features of the dataframe and remove HLCV and date added columns
:param df_main: DataFrame = main dataframe
:param df_to_merge: DataFrame = dataframe to merge
:param tf: str = timeframe of the main dataframe
:param timeframe_inf: str = timeframe of the dataframe to merge
:param suffix: str = suffix to add to the columns of the dataframe to merge
:return: dataframe = merged dataframe
"""
dataframe = merge_informative_pair(df_main, df_to_merge, tf, timeframe_inf=timeframe_inf,
append_timeframe=False, suffix=suffix, ffill=True)
skip_columns = [
(f"{s}_{suffix}") for s in ["date", "open", "high", "low", "close", "volume"]
]
dataframe = dataframe.drop(columns=skip_columns)
return dataframe
def populate_features(self, dataframe: DataFrame, pair: str, strategy: IStrategy,
corr_dataframes: dict, base_dataframes: dict,
is_corr_pairs: bool = False) -> DataFrame:
"""
Use the user defined strategy functions for populating features
:param dataframe: DataFrame = dataframe to populate
:param pair: str = pair to populate
:param strategy: IStrategy = user defined strategy object
:param corr_dataframes: dict = dict containing the df pair dataframes
:param base_dataframes: dict = dict containing the current pair dataframes
:param is_corr_pairs: bool = whether the pair is a corr pair or not
:return: dataframe = populated dataframe
"""
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
for tf in tfs:
informative_df = self.get_pair_data_for_features(
pair, tf, strategy, corr_dataframes, base_dataframes, is_corr_pairs)
informative_copy = informative_df.copy()
for t in self.freqai_config["feature_parameters"]["indicator_periods_candles"]:
df_features = strategy.feature_engineering_expand_all(
informative_copy.copy(), t)
suffix = f"{t}"
informative_df = self.merge_features(informative_df, df_features, tf, tf, suffix)
generic_df = strategy.feature_engineering_expand_basic(informative_copy.copy())
suffix = "gen"
informative_df = self.merge_features(informative_df, generic_df, tf, tf, suffix)
indicators = [col for col in informative_df if col.startswith("%")]
for n in range(self.freqai_config["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
df_shift = informative_df[indicators].shift(n)
df_shift = df_shift.add_suffix("_shift-" + str(n))
informative_df = pd.concat((informative_df, df_shift), axis=1)
dataframe = self.merge_features(dataframe.copy(), informative_df,
self.config["timeframe"], tf, f'{pair}_{tf}')
return dataframe
def use_strategy_to_populate_indicators( def use_strategy_to_populate_indicators(
self, self,
strategy: IStrategy, strategy: IStrategy,
@ -1188,7 +1287,87 @@ class FreqaiDataKitchen:
""" """
Use the user defined strategy for populating indicators during retrain Use the user defined strategy for populating indicators during retrain
:param strategy: IStrategy = user defined strategy object :param strategy: IStrategy = user defined strategy object
:param corr_dataframes: dict = dict containing the informative pair dataframes :param corr_dataframes: dict = dict containing the df pair dataframes
(for user defined timeframes)
:param base_dataframes: dict = dict containing the current pair dataframes
(for user defined timeframes)
:param pair: str = pair to populate
:param prediction_dataframe: DataFrame = dataframe containing the pair data
used for prediction
:param do_corr_pairs: bool = whether to populate corr pairs or not
:return:
dataframe: DataFrame = dataframe containing populated indicators
"""
# this is a hack to check if the user is using the populate_any_indicators function
new_version = inspect.getsource(strategy.populate_any_indicators) == (
inspect.getsource(IStrategy.populate_any_indicators))
if new_version:
tfs: List[str] = self.freqai_config["feature_parameters"].get("include_timeframes")
pairs: List[str] = self.freqai_config["feature_parameters"].get(
"include_corr_pairlist", [])
for tf in tfs:
if tf not in base_dataframes:
base_dataframes[tf] = pd.DataFrame()
for p in pairs:
if p not in corr_dataframes:
corr_dataframes[p] = {}
if tf not in corr_dataframes[p]:
corr_dataframes[p][tf] = pd.DataFrame()
if not prediction_dataframe.empty:
dataframe = prediction_dataframe.copy()
else:
dataframe = base_dataframes[self.config["timeframe"]].copy()
corr_pairs: List[str] = self.freqai_config["feature_parameters"].get(
"include_corr_pairlist", [])
dataframe = self.populate_features(dataframe.copy(), pair, strategy,
corr_dataframes, base_dataframes)
dataframe = strategy.feature_engineering_standard(dataframe.copy())
# ensure corr pairs are always last
for corr_pair in corr_pairs:
if pair == corr_pair:
continue # dont repeat anything from whitelist
if corr_pairs and do_corr_pairs:
dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy,
corr_dataframes, base_dataframes, True)
dataframe = strategy.set_freqai_targets(dataframe.copy())
self.get_unique_classes_from_labels(dataframe)
dataframe = self.remove_special_chars_from_feature_names(dataframe)
if self.config.get('reduce_df_footprint', False):
dataframe = reduce_dataframe_footprint(dataframe)
return dataframe
else:
# the user is using the populate_any_indicators functions which is deprecated
df = self.use_strategy_to_populate_indicators_old_version(
strategy, corr_dataframes, base_dataframes, pair,
prediction_dataframe, do_corr_pairs)
return df
def use_strategy_to_populate_indicators_old_version(
self,
strategy: IStrategy,
corr_dataframes: dict = {},
base_dataframes: dict = {},
pair: str = "",
prediction_dataframe: DataFrame = pd.DataFrame(),
do_corr_pairs: bool = True,
) -> DataFrame:
"""
Use the user defined strategy for populating indicators during retrain
:param strategy: IStrategy = user defined strategy object
:param corr_dataframes: dict = dict containing the df pair dataframes
(for user defined timeframes) (for user defined timeframes)
:param base_dataframes: dict = dict containing the current pair dataframes :param base_dataframes: dict = dict containing the current pair dataframes
(for user defined timeframes) (for user defined timeframes)

70
freqtrade/freqai/freqai_interface.py
View File

@ -1,3 +1,4 @@
import inspect
import logging import logging
import threading import threading
import time import time
@ -106,6 +107,8 @@ class IFreqaiModel(ABC):
self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1) self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
self.can_short = True # overridden in start() with strategy.can_short self.can_short = True # overridden in start() with strategy.can_short
self.warned_deprecated_populate_any_indicators = False
record_params(config, self.full_path) record_params(config, self.full_path)
def __getstate__(self): def __getstate__(self):
@ -136,6 +139,9 @@ class IFreqaiModel(ABC):
self.data_provider = strategy.dp self.data_provider = strategy.dp
self.can_short = strategy.can_short self.can_short = strategy.can_short
# check if the strategy has deprecated populate_any_indicators function
self.check_deprecated_populate_any_indicators(strategy)
if self.live: if self.live:
self.inference_timer('start') self.inference_timer('start')
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
@ -149,12 +155,9 @@ class IFreqaiModel(ABC):
# the concatenated results for the full backtesting period back to the strategy. # the concatenated results for the full backtesting period back to the strategy.
elif not self.follow_mode: elif not self.follow_mode:
self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"]) self.dk = FreqaiDataKitchen(self.config, self.live, metadata["pair"])
dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
)
if not self.config.get("freqai_backtest_live_models", False): if not self.config.get("freqai_backtest_live_models", False):
logger.info(f"Training {len(self.dk.training_timeranges)} timeranges") logger.info(f"Training {len(self.dk.training_timeranges)} timeranges")
dk = self.start_backtesting(dataframe, metadata, self.dk) dk = self.start_backtesting(dataframe, metadata, self.dk, strategy)
dataframe = dk.remove_features_from_df(dk.return_dataframe) dataframe = dk.remove_features_from_df(dk.return_dataframe)
else: else:
logger.info( logger.info(
@ -255,7 +258,7 @@ class IFreqaiModel(ABC):
self.dd.save_metric_tracker_to_disk() self.dd.save_metric_tracker_to_disk()
def start_backtesting( def start_backtesting(
self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen, strategy: IStrategy
) -> FreqaiDataKitchen: ) -> FreqaiDataKitchen:
""" """
The main broad execution for backtesting. For backtesting, each pair enters and then gets The main broad execution for backtesting. For backtesting, each pair enters and then gets
@ -267,19 +270,22 @@ class IFreqaiModel(ABC):
:param dataframe: DataFrame = strategy passed dataframe :param dataframe: DataFrame = strategy passed dataframe
:param metadata: Dict = pair metadata :param metadata: Dict = pair metadata
:param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
:param strategy: Strategy to train on
:return: :return:
FreqaiDataKitchen = Data management/analysis tool associated to present pair only FreqaiDataKitchen = Data management/analysis tool associated to present pair only
""" """
self.pair_it += 1 self.pair_it += 1
train_it = 0 train_it = 0
pair = metadata["pair"]
populate_indicators = True
check_features = True
# Loop enforcing the sliding window training/backtesting paradigm # Loop enforcing the sliding window training/backtesting paradigm
# tr_train is the training time range e.g. 1 historical month # tr_train is the training time range e.g. 1 historical month
# tr_backtest is the backtesting time range e.g. the week directly # tr_backtest is the backtesting time range e.g. the week directly
# following tr_train. Both of these windows slide through the # following tr_train. Both of these windows slide through the
# entire backtest # entire backtest
for tr_train, tr_backtest in zip(dk.training_timeranges, dk.backtesting_timeranges): for tr_train, tr_backtest in zip(dk.training_timeranges, dk.backtesting_timeranges):
pair = metadata["pair"]
(_, _, _) = self.dd.get_pair_dict_info(pair) (_, _, _) = self.dd.get_pair_dict_info(pair)
train_it += 1 train_it += 1
total_trains = len(dk.backtesting_timeranges) total_trains = len(dk.backtesting_timeranges)
@ -301,18 +307,42 @@ class IFreqaiModel(ABC):
dk.set_new_model_names(pair, timestamp_model_id) dk.set_new_model_names(pair, timestamp_model_id)
if dk.check_if_backtest_prediction_is_valid(len_backtest_df): if dk.check_if_backtest_prediction_is_valid(len_backtest_df):
if check_features:
self.dd.load_metadata(dk) self.dd.load_metadata(dk)
dk.find_features(dataframe) dataframe_dummy_features = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe.tail(1), pair=metadata["pair"]
)
dk.find_features(dataframe_dummy_features)
self.check_if_feature_list_matches_strategy(dk) self.check_if_feature_list_matches_strategy(dk)
check_features = False
append_df = dk.get_backtesting_prediction() append_df = dk.get_backtesting_prediction()
dk.append_predictions(append_df) dk.append_predictions(append_df)
else: else:
dataframe_train = dk.slice_dataframe(tr_train, dataframe) if populate_indicators:
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe) dataframe = self.dk.use_strategy_to_populate_indicators(
strategy, prediction_dataframe=dataframe, pair=metadata["pair"]
)
populate_indicators = False
dataframe_base_train = dataframe.loc[dataframe["date"] < tr_train.stopdt, :]
dataframe_base_train = strategy.set_freqai_targets(dataframe_base_train)
dataframe_base_backtest = dataframe.loc[dataframe["date"] < tr_backtest.stopdt, :]
dataframe_base_backtest = strategy.set_freqai_targets(dataframe_base_backtest)
dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train)
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest)
if not self.model_exists(dk): if not self.model_exists(dk):
dk.find_features(dataframe_train) dk.find_features(dataframe_train)
dk.find_labels(dataframe_train) dk.find_labels(dataframe_train)
try:
self.model = self.train(dataframe_train, pair, dk) self.model = self.train(dataframe_train, pair, dk)
except Exception as msg:
logger.warning(
f"Training {pair} raised exception {msg.__class__.__name__}. "
f"Message: {msg}, skipping.")
self.dd.pair_dict[pair]["trained_timestamp"] = int( self.dd.pair_dict[pair]["trained_timestamp"] = int(
tr_train.stopts) tr_train.stopts)
if self.plot_features: if self.plot_features:
@ -349,7 +379,6 @@ class IFreqaiModel(ABC):
:returns: :returns:
dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only
""" """
# update follower # update follower
if self.follow_mode: if self.follow_mode:
self.dd.update_follower_metadata() self.dd.update_follower_metadata()
@ -913,9 +942,28 @@ class IFreqaiModel(ABC):
dk.return_dataframe = dk.return_dataframe.drop(columns=list(columns_to_drop)) dk.return_dataframe = dk.return_dataframe.drop(columns=list(columns_to_drop))
dk.return_dataframe = pd.merge( dk.return_dataframe = pd.merge(
dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred") dk.return_dataframe, saved_dataframe, how='left', left_on='date', right_on="date_pred")
# dk.return_dataframe = dk.return_dataframe[saved_dataframe.columns].fillna(0)
return dk return dk
def check_deprecated_populate_any_indicators(self, strategy: IStrategy):
"""
Check and warn if the deprecated populate_any_indicators function is used.
:param strategy: strategy object
"""
if not self.warned_deprecated_populate_any_indicators:
self.warned_deprecated_populate_any_indicators = True
old_version = inspect.getsource(strategy.populate_any_indicators) != (
inspect.getsource(IStrategy.populate_any_indicators))
if old_version:
logger.warning("DEPRECATION WARNING: "
"You are using the deprecated populate_any_indicators function. "
"This function will raise an error on March 1 2023. "
"Please update your strategy by using "
"the new feature_engineering functions. See \n"
"https://www.freqtrade.io/en/latest/freqai-feature-engineering/"
"for details.")
# Following methods which are overridden by user made prediction models. # Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModel.py for an example. # See freqai/prediction_models/CatboostPredictionModel.py for an example.

7
freqtrade/freqtradebot.py
View File

@ -720,7 +720,7 @@ class FreqtradeBot(LoggingMixin):
time_in_force=time_in_force, time_in_force=time_in_force,
leverage=leverage leverage=leverage
) )
order_obj = Order.parse_from_ccxt_object(order, pair, side) order_obj = Order.parse_from_ccxt_object(order, pair, side, amount, enter_limit_requested)
order_id = order['id'] order_id = order['id']
order_status = order.get('status') order_status = order.get('status')
logger.info(f"Order #{order_id} was created for {pair} and status is {order_status}.") logger.info(f"Order #{order_id} was created for {pair} and status is {order_status}.")
@ -1094,7 +1094,8 @@ class FreqtradeBot(LoggingMixin):
leverage=trade.leverage leverage=trade.leverage
) )
order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss') order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss',
trade.amount, stop_price)
trade.orders.append(order_obj) trade.orders.append(order_obj)
trade.stoploss_order_id = str(stoploss_order['id']) trade.stoploss_order_id = str(stoploss_order['id'])
trade.stoploss_last_update = datetime.now(timezone.utc) trade.stoploss_last_update = datetime.now(timezone.utc)
@ -1595,7 +1596,7 @@ class FreqtradeBot(LoggingMixin):
self.handle_insufficient_funds(trade) self.handle_insufficient_funds(trade)
return False return False
order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side) order_obj = Order.parse_from_ccxt_object(order, trade.pair, trade.exit_side, amount, limit)
trade.orders.append(order_obj) trade.orders.append(order_obj)
trade.open_order_id = order['id'] trade.open_order_id = order['id']

21
freqtrade/optimize/backtesting.py
View File

@ -1052,7 +1052,8 @@ class Backtesting:
def backtest_loop( def backtest_loop(
self, row: Tuple, pair: str, current_time: datetime, end_date: datetime, self, row: Tuple, pair: str, current_time: datetime, end_date: datetime,
open_trade_count_start: int, is_first: bool = True) -> int: open_trade_count_start: int, trade_dir: Optional[LongShort],
is_first: bool = True) -> int:
""" """
NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
@ -1071,7 +1072,6 @@ class Backtesting:
# max_open_trades must be respected # max_open_trades must be respected
# don't open on the last row # don't open on the last row
# We only open trades on the main candle, not on detail candles # We only open trades on the main candle, not on detail candles
trade_dir = self.check_for_trade_entry(row)
if ( if (
(self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0) (self._position_stacking or len(LocalTrade.bt_trades_open_pp[pair]) == 0)
and is_first and is_first
@ -1163,7 +1163,15 @@ class Backtesting:
indexes[pair] = row_index indexes[pair] = row_index
self.dataprovider._set_dataframe_max_index(row_index) self.dataprovider._set_dataframe_max_index(row_index)
current_detail_time: datetime = row[DATE_IDX].to_pydatetime() current_detail_time: datetime = row[DATE_IDX].to_pydatetime()
if self.timeframe_detail and pair in self.detail_data: trade_dir: Optional[LongShort] = self.check_for_trade_entry(row)
if (
(trade_dir is not None or len(LocalTrade.bt_trades_open_pp[pair]) > 0)
and self.timeframe_detail and pair in self.detail_data
):
# Spread out into detail timeframe.
# Should only happen when we are either in a trade for this pair
# or when we got the signal for a new trade.
exit_candle_end = current_detail_time + timedelta(minutes=self.timeframe_min) exit_candle_end = current_detail_time + timedelta(minutes=self.timeframe_min)
detail_data = self.detail_data[pair] detail_data = self.detail_data[pair]
@ -1175,7 +1183,7 @@ class Backtesting:
# Fall back to "regular" data if no detail data was found for this candle # Fall back to "regular" data if no detail data was found for this candle
open_trade_count_start = self.backtest_loop( open_trade_count_start = self.backtest_loop(
row, pair, current_time, end_date, row, pair, current_time, end_date,
open_trade_count_start) open_trade_count_start, trade_dir)
continue continue
detail_data.loc[:, 'enter_long'] = row[LONG_IDX] detail_data.loc[:, 'enter_long'] = row[LONG_IDX]
detail_data.loc[:, 'exit_long'] = row[ELONG_IDX] detail_data.loc[:, 'exit_long'] = row[ELONG_IDX]
@ -1188,12 +1196,13 @@ class Backtesting:
for det_row in detail_data[HEADERS].values.tolist(): for det_row in detail_data[HEADERS].values.tolist():
open_trade_count_start = self.backtest_loop( open_trade_count_start = self.backtest_loop(
det_row, pair, current_time_det, end_date, det_row, pair, current_time_det, end_date,
open_trade_count_start, is_first) open_trade_count_start, trade_dir, is_first)
current_time_det += timedelta(minutes=self.timeframe_detail_min) current_time_det += timedelta(minutes=self.timeframe_detail_min)
is_first = False is_first = False
else: else:
open_trade_count_start = self.backtest_loop( open_trade_count_start = self.backtest_loop(
row, pair, current_time, end_date, open_trade_count_start) row, pair, current_time, end_date,
open_trade_count_start, trade_dir)
# Move time one configured time_interval ahead. # Move time one configured time_interval ahead.
self.progress.increment() self.progress.increment()

13
freqtrade/persistence/migrations.py
View File

@ -214,17 +214,22 @@ def migrate_orders_table(engine, table_back_name: str, cols_order: List):
average = get_column_def(cols_order, 'average', 'null') average = get_column_def(cols_order, 'average', 'null')
stop_price = get_column_def(cols_order, 'stop_price', 'null') stop_price = get_column_def(cols_order, 'stop_price', 'null')
funding_fee = get_column_def(cols_order, 'funding_fee', '0.0') funding_fee = get_column_def(cols_order, 'funding_fee', '0.0')
ft_amount = get_column_def(cols_order, 'ft_amount', 'coalesce(amount, 0.0)')
ft_price = get_column_def(cols_order, 'ft_price', 'coalesce(price, 0.0)')
# sqlite does not support literals for booleans # sqlite does not support literals for booleans
with engine.begin() as connection: with engine.begin() as connection:
connection.execute(text(f""" connection.execute(text(f"""
insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
status, symbol, order_type, side, price, amount, filled, average, remaining, cost, status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
stop_price, order_date, order_filled_date, order_update_date, ft_fee_base, funding_fee) stop_price, order_date, order_filled_date, order_update_date, ft_fee_base, funding_fee,
ft_amount, ft_price
)
select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
status, symbol, order_type, side, price, amount, filled, {average} average, remaining, status, symbol, order_type, side, price, amount, filled, {average} average, remaining,
cost, {stop_price} stop_price, order_date, order_filled_date, cost, {stop_price} stop_price, order_date, order_filled_date,
order_update_date, {ft_fee_base} ft_fee_base, {funding_fee} funding_fee order_update_date, {ft_fee_base} ft_fee_base, {funding_fee} funding_fee,
{ft_amount} ft_amount, {ft_price} ft_price
from {table_back_name} from {table_back_name}
""")) """))
@ -311,8 +316,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
# if ('orders' not in previous_tables # if ('orders' not in previous_tables
# or not has_column(cols_orders, 'funding_fee')): # or not has_column(cols_orders, 'funding_fee')):
migrating = False migrating = False
# if not has_column(cols_orders, 'funding_fee'): # if not has_column(cols_trades, 'max_stake_amount'):
if not has_column(cols_trades, 'max_stake_amount'): if not has_column(cols_orders, 'ft_price'):
migrating = True migrating = True
logger.info(f"Running database migration for trades - " logger.info(f"Running database migration for trades - "
f"backup: {table_back_name}, {order_table_bak_name}") f"backup: {table_back_name}, {order_table_bak_name}")

23
freqtrade/persistence/trade_model.py
View File

@ -49,6 +49,8 @@ class Order(_DECL_BASE):
ft_order_side: str = Column(String(25), nullable=False) ft_order_side: str = Column(String(25), nullable=False)
ft_pair: str = Column(String(25), nullable=False) ft_pair: str = Column(String(25), nullable=False)
ft_is_open = Column(Boolean, nullable=False, default=True, index=True) ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
ft_amount = Column(Float, nullable=False)
ft_price = Column(Float, nullable=False)
order_id: str = Column(String(255), nullable=False, index=True) order_id: str = Column(String(255), nullable=False, index=True)
status = Column(String(255), nullable=True) status = Column(String(255), nullable=True)
@ -82,9 +84,13 @@ class Order(_DECL_BASE):
self.order_filled_date.replace(tzinfo=timezone.utc) if self.order_filled_date else None self.order_filled_date.replace(tzinfo=timezone.utc) if self.order_filled_date else None
) )
@property
def safe_amount(self) -> float:
return self.amount or self.ft_amount
@property @property
def safe_price(self) -> float: def safe_price(self) -> float:
return self.average or self.price or self.stop_price return self.average or self.price or self.stop_price or self.ft_price
@property @property
def safe_filled(self) -> float: def safe_filled(self) -> float:
@ -94,7 +100,7 @@ class Order(_DECL_BASE):
def safe_remaining(self) -> float: def safe_remaining(self) -> float:
return ( return (
self.remaining if self.remaining is not None else self.remaining if self.remaining is not None else
self.amount - (self.filled or 0.0) self.safe_amount - (self.filled or 0.0)
) )
@property @property
@ -227,11 +233,20 @@ class Order(_DECL_BASE):
logger.warning(f"Did not find order for {order}.") logger.warning(f"Did not find order for {order}.")
@staticmethod @staticmethod
def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order': def parse_from_ccxt_object(
order: Dict[str, Any], pair: str, side: str,
amount: Optional[float] = None, price: Optional[float] = None) -> 'Order':
""" """
Parse an order from a ccxt object and return a new order Object. Parse an order from a ccxt object and return a new order Object.
Optional support for overriding amount and price is only used for test simplification.
""" """
o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair) o = Order(
order_id=str(order['id']),
ft_order_side=side,
ft_pair=pair,
ft_amount=amount if amount else order['amount'],
ft_price=price if price else order['price'],
)
o.update_from_ccxt_object(order) o.update_from_ccxt_object(order)
return o return o

93
freqtrade/strategy/interface.py
View File

@ -601,6 +601,7 @@ class IStrategy(ABC, HyperStrategyMixin):
informative: DataFrame = None, informative: DataFrame = None,
set_generalized_indicators: bool = False) -> DataFrame: set_generalized_indicators: bool = False) -> DataFrame:
""" """
DEPRECATED - USE FEATURE ENGINEERING FUNCTIONS INSTEAD
Function designed to automatically generate, name and merge features Function designed to automatically generate, name and merge features
from user indicated timeframes in the configuration file. User can add from user indicated timeframes in the configuration file. User can add
additional features here, but must follow the naming convention. additional features here, but must follow the naming convention.
@ -613,6 +614,98 @@ class IStrategy(ABC, HyperStrategyMixin):
""" """
return df return df
def feature_engineering_expand_all(self, dataframe: DataFrame,
period: int, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This function will automatically expand the defined features on the config defined
`indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and
`include_corr_pairs`. In other words, a single feature defined in this function
will automatically expand to a total of
`indicator_periods_candles` * `include_timeframes` * `include_shifted_candles` *
`include_corr_pairs` numbers of features added to the model.
All features must be prepended with `%` to be recognized by FreqAI internals.
More details on how these config defined parameters accelerate feature engineering
in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
:param period: period of the indicator - usage example:
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
"""
return dataframe
def feature_engineering_expand_basic(self, dataframe: DataFrame, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This function will automatically expand the defined features on the config defined
`include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
In other words, a single feature defined in this function
will automatically expand to a total of
`include_timeframes` * `include_shifted_candles` * `include_corr_pairs`
numbers of features added to the model.
Features defined here will *not* be automatically duplicated on user defined
`indicator_periods_candles`
All features must be prepended with `%` to be recognized by FreqAI internals.
More details on how these config defined parameters accelerate feature engineering
in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-ema-200"] = ta.EMA(dataframe, timeperiod=200)
"""
return dataframe
def feature_engineering_standard(self, dataframe: DataFrame, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This optional function will be called once with the dataframe of the base timeframe.
This is the final function to be called, which means that the dataframe entering this
function will contain all the features and columns created by all other
freqai_feature_engineering_* functions.
This function is a good place to do custom exotic feature extractions (e.g. tsfresh).
This function is a good place for any feature that should not be auto-expanded upon
(e.g. day of the week).
All features must be prepended with `%` to be recognized by FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the features
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
"""
return dataframe
def set_freqai_targets(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
return dataframe
### ###
# END - Intended to be overridden by strategy # END - Intended to be overridden by strategy
### ###

163
freqtrade/templates/FreqaiExampleHybridStrategy.py
View File

@ -95,65 +95,132 @@ class FreqaiExampleHybridStrategy(IStrategy):
short_rsi = IntParameter(low=51, high=100, default=70, space='sell', optimize=True, load=True) short_rsi = IntParameter(low=51, high=100, default=70, space='sell', optimize=True, load=True)
exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True)
# FreqAI required function, user can add or remove indicators, but general structure def feature_engineering_expand_all(self, dataframe, period, **kwargs):
# must stay the same.
def populate_any_indicators(
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
""" """
User feeds these indicators to FreqAI to train a classifier to decide *Only functional with FreqAI enabled strategies*
if the market will go up or down. This function will automatically expand the defined features on the config defined
`indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and
`include_corr_pairs`. In other words, a single feature defined in this function
will automatically expand to a total of
`indicator_periods_candles` * `include_timeframes` * `include_shifted_candles` *
`include_corr_pairs` numbers of features added to the model.
:param pair: pair to be used as informative All features must be prepended with `%` to be recognized by FreqAI internals.
:param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names More details on how these config defined parameters accelerate feature engineering
:param informative: the dataframe associated with the informative pair in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
:param period: period of the indicator - usage example:
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
""" """
if informative is None: dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
informative = self.dp.get_pair_dataframe(pair, tf) dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period)
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
# first loop is automatically duplicating indicators for time periods bollinger = qtpylib.bollinger_bands(
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: qtpylib.typical_price(dataframe), window=period, stds=2.2
)
dataframe["bb_lowerband-period"] = bollinger["lower"]
dataframe["bb_middleband-period"] = bollinger["mid"]
dataframe["bb_upperband-period"] = bollinger["upper"]
t = int(t) dataframe["%-bb_width-period"] = (
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) dataframe["bb_upperband-period"]
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) - dataframe["bb_lowerband-period"]
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t) ) / dataframe["bb_middleband-period"]
informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t) dataframe["%-close-bb_lower-period"] = (
informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t) dataframe["close"] / dataframe["bb_lowerband-period"]
informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t)
informative[f"%-{pair}relative_volume-period_{t}"] = (
informative["volume"] / informative["volume"].rolling(t).mean()
) )
# FreqAI needs the following lines in order to detect features and automatically dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period)
# expand upon them.
indicators = [col for col in informative if col.startswith("%")]
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) dataframe["%-relative_volume-period"] = (
skip_columns = [ dataframe["volume"] / dataframe["volume"].rolling(period).mean()
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"] )
]
df = df.drop(columns=skip_columns)
# User can set the "target" here (in present case it is the return dataframe
# "up" or "down")
if set_generalized_indicators:
# User "looks into the future" here to figure out if the future
# will be "up" or "down". This same column name is available to
# the user
df['&s-up_or_down'] = np.where(df["close"].shift(-50) >
df["close"], 'up', 'down')
return df def feature_engineering_expand_basic(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This function will automatically expand the defined features on the config defined
`include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
In other words, a single feature defined in this function
will automatically expand to a total of
`include_timeframes` * `include_shifted_candles` * `include_corr_pairs`
numbers of features added to the model.
Features defined here will *not* be automatically duplicated on user defined
`indicator_periods_candles`
All features must be prepended with `%` to be recognized by FreqAI internals.
More details on how these config defined parameters accelerate feature engineering
in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-ema-200"] = ta.EMA(dataframe, timeperiod=200)
"""
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-raw_volume"] = dataframe["volume"]
dataframe["%-raw_price"] = dataframe["close"]
return dataframe
def feature_engineering_standard(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This optional function will be called once with the dataframe of the base timeframe.
This is the final function to be called, which means that the dataframe entering this
function will contain all the features and columns created by all other
freqai_feature_engineering_* functions.
This function is a good place to do custom exotic feature extractions (e.g. tsfresh).
This function is a good place for any feature that should not be auto-expanded upon
(e.g. day of the week).
All features must be prepended with `%` to be recognized by FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the features
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
"""
dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
return dataframe
def set_freqai_targets(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
dataframe['&s-up_or_down'] = np.where(dataframe["close"].shift(-50) >
dataframe["close"], 'up', 'down')
return dataframe
# flake8: noqa: C901 # flake8: noqa: C901
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

192
freqtrade/templates/FreqaiExampleStrategy.py
View File

@ -1,12 +1,11 @@
import logging import logging
from functools import reduce from functools import reduce
import pandas as pd
import talib.abstract as ta import talib.abstract as ta
from pandas import DataFrame from pandas import DataFrame
from technical import qtpylib from technical import qtpylib
from freqtrade.strategy import CategoricalParameter, IStrategy, merge_informative_pair from freqtrade.strategy import CategoricalParameter, IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -18,8 +17,8 @@ class FreqaiExampleStrategy(IStrategy):
IFreqaiModel to the strategy. Namely, the user uses: IFreqaiModel to the strategy. Namely, the user uses:
self.freqai.start(dataframe, metadata) self.freqai.start(dataframe, metadata)
to make predictions on their data. populate_any_indicators() automatically to make predictions on their data. feature_engineering_*() automatically
generates the variety of features indicated by the user in the generate the variety of features indicated by the user in the
canonical freqtrade configuration file under config['freqai']. canonical freqtrade configuration file under config['freqai'].
""" """
@ -40,96 +39,141 @@ class FreqaiExampleStrategy(IStrategy):
use_exit_signal = True use_exit_signal = True
# this is the maximum period fed to talib (timeframe independent) # this is the maximum period fed to talib (timeframe independent)
startup_candle_count: int = 40 startup_candle_count: int = 40
can_short = False can_short = True
std_dev_multiplier_buy = CategoricalParameter( std_dev_multiplier_buy = CategoricalParameter(
[0.75, 1, 1.25, 1.5, 1.75], default=1.25, space="buy", optimize=True) [0.75, 1, 1.25, 1.5, 1.75], default=1.25, space="buy", optimize=True)
std_dev_multiplier_sell = CategoricalParameter( std_dev_multiplier_sell = CategoricalParameter(
[0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True) [0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True)
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
""" """
Function designed to automatically generate, name and merge features *Only functional with FreqAI enabled strategies*
from user indicated timeframes in the configuration file. User controls the indicators This function will automatically expand the defined features on the config defined
passed to the training/prediction by prepending indicators with `f'%-{pair}` `indicator_periods_candles`, `include_timeframes`, `include_shifted_candles`, and
(see convention below). I.e. user should not prepend any supporting metrics `include_corr_pairs`. In other words, a single feature defined in this function
(e.g. bb_lowerband below) with % unless they explicitly want to pass that metric to the will automatically expand to a total of
model. `indicator_periods_candles` * `include_timeframes` * `include_shifted_candles` *
:param pair: pair to be used as informative `include_corr_pairs` numbers of features added to the model.
:param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names All features must be prepended with `%` to be recognized by FreqAI internals.
:param informative: the dataframe associated with the informative pair
More details on how these config defined parameters accelerate feature engineering
in the documentation at:
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
:param period: period of the indicator - usage example:
dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
""" """
if informative is None: dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
informative = self.dp.get_pair_dataframe(pair, tf) dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
# first loop is automatically duplicating indicators for time periods dataframe["%-sma-period"] = ta.SMA(dataframe, timeperiod=period)
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]: dataframe["%-ema-period"] = ta.EMA(dataframe, timeperiod=period)
t = int(t)
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{pair}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{pair}adx-period_{t}"] = ta.ADX(informative, timeperiod=t)
informative[f"%-{pair}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{pair}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
bollinger = qtpylib.bollinger_bands( bollinger = qtpylib.bollinger_bands(
qtpylib.typical_price(informative), window=t, stds=2.2 qtpylib.typical_price(dataframe), window=period, stds=2.2
) )
informative[f"{pair}bb_lowerband-period_{t}"] = bollinger["lower"] dataframe["bb_lowerband-period"] = bollinger["lower"]
informative[f"{pair}bb_middleband-period_{t}"] = bollinger["mid"] dataframe["bb_middleband-period"] = bollinger["mid"]
informative[f"{pair}bb_upperband-period_{t}"] = bollinger["upper"] dataframe["bb_upperband-period"] = bollinger["upper"]
informative[f"%-{pair}bb_width-period_{t}"] = ( dataframe["%-bb_width-period"] = (
informative[f"{pair}bb_upperband-period_{t}"] dataframe["bb_upperband-period"]
- informative[f"{pair}bb_lowerband-period_{t}"] - dataframe["bb_lowerband-period"]
) / informative[f"{pair}bb_middleband-period_{t}"] ) / dataframe["bb_middleband-period"]
informative[f"%-{pair}close-bb_lower-period_{t}"] = ( dataframe["%-close-bb_lower-period"] = (
informative["close"] / informative[f"{pair}bb_lowerband-period_{t}"] dataframe["close"] / dataframe["bb_lowerband-period"]
) )
informative[f"%-{pair}roc-period_{t}"] = ta.ROC(informative, timeperiod=t) dataframe["%-roc-period"] = ta.ROC(dataframe, timeperiod=period)
informative[f"%-{pair}relative_volume-period_{t}"] = ( dataframe["%-relative_volume-period"] = (
informative["volume"] / informative["volume"].rolling(t).mean() dataframe["volume"] / dataframe["volume"].rolling(period).mean()
) )
informative[f"%-{pair}pct-change"] = informative["close"].pct_change() return dataframe
informative[f"%-{pair}raw_volume"] = informative["volume"]
informative[f"%-{pair}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")] def feature_engineering_expand_basic(self, dataframe, **kwargs):
# This loop duplicates and shifts all indicators to add a sense of recency to data """
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1): *Only functional with FreqAI enabled strategies*
if n == 0: This function will automatically expand the defined features on the config defined
continue `include_timeframes`, `include_shifted_candles`, and `include_corr_pairs`.
informative_shift = informative[indicators].shift(n) In other words, a single feature defined in this function
informative_shift = informative_shift.add_suffix("_shift-" + str(n)) will automatically expand to a total of
informative = pd.concat((informative, informative_shift), axis=1) `include_timeframes` * `include_shifted_candles` * `include_corr_pairs`
numbers of features added to the model.
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) Features defined here will *not* be automatically duplicated on user defined
skip_columns = [ `indicator_periods_candles`
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this All features must be prepended with `%` to be recognized by FreqAI internals.
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) More details on how these config defined parameters accelerate feature engineering
df["&-s_close"] = ( in the documentation at:
df["close"]
https://www.freqtrade.io/en/latest/freqai-parameter-table/#feature-parameters
https://www.freqtrade.io/en/latest/freqai-feature-engineering/#defining-the-features
:param df: strategy dataframe which will receive the features
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-ema-200"] = ta.EMA(dataframe, timeperiod=200)
"""
dataframe["%-pct-change"] = dataframe["close"].pct_change()
dataframe["%-raw_volume"] = dataframe["volume"]
dataframe["%-raw_price"] = dataframe["close"]
return dataframe
def feature_engineering_standard(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
This optional function will be called once with the dataframe of the base timeframe.
This is the final function to be called, which means that the dataframe entering this
function will contain all the features and columns created by all other
freqai_feature_engineering_* functions.
This function is a good place to do custom exotic feature extractions (e.g. tsfresh).
This function is a good place for any feature that should not be auto-expanded upon
(e.g. day of the week).
All features must be prepended with `%` to be recognized by FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the features
usage example: dataframe["%-day_of_week"] = (dataframe["date"].dt.dayofweek + 1) / 7
"""
dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
return dataframe
def set_freqai_targets(self, dataframe, **kwargs):
"""
*Only functional with FreqAI enabled strategies*
Required function to set the targets for the model.
All targets must be prepended with `&` to be recognized by the FreqAI internals.
More details about feature engineering available:
https://www.freqtrade.io/en/latest/freqai-feature-engineering
:param df: strategy dataframe which will receive the targets
usage example: dataframe["&-target"] = dataframe["close"].shift(-1) / dataframe["close"]
"""
dataframe["&-s_close"] = (
dataframe["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean() .mean()
/ df["close"] / dataframe["close"]
- 1 - 1
) )
@ -155,19 +199,19 @@ class FreqaiExampleStrategy(IStrategy):
# .min() # .min()
# ) # )
return df return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
# All indicators must be populated by populate_any_indicators() for live functionality # All indicators must be populated by feature_engineering_*() functions
# to work correctly.
# the model will return all labels created by user in `populate_any_indicators` # the model will return all labels created by user in `feature_engineering_*`
# (& appended targets), an indication of whether or not the prediction should be accepted, # (& appended targets), an indication of whether or not the prediction should be accepted,
# the target mean/std values for each of the labels created by user in # the target mean/std values for each of the labels created by user in
# `populate_any_indicators()` for each training period. # `set_freqai_targets()` for each training period.
dataframe = self.freqai.start(dataframe, metadata, self) dataframe = self.freqai.start(dataframe, metadata, self)
for val in self.std_dev_multiplier_buy.range: for val in self.std_dev_multiplier_buy.range:
dataframe[f'target_roi_{val}'] = ( dataframe[f'target_roi_{val}'] = (
dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * val dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * val

4
mkdocs.yml
View File

@ -59,7 +59,11 @@ theme:
favicon: "images/logo.png" favicon: "images/logo.png"
custom_dir: "docs/overrides" custom_dir: "docs/overrides"
features: features:
- content.code.annotate
- search.share - search.share
- content.code.copy
- navigation.top
- navigation.footer
palette: palette:
- scheme: default - scheme: default
primary: "blue grey" primary: "blue grey"

2
requirements-dev.txt
View File

@ -20,7 +20,7 @@ isort==5.11.4
# For datetime mocking # For datetime mocking
time-machine==2.9.0 time-machine==2.9.0
# fastapi testing # fastapi testing
httpx==0.23.1 httpx==0.23.3
# Convert jupyter notebooks to markdown documents # Convert jupyter notebooks to markdown documents
nbconvert==7.2.7 nbconvert==7.2.7

2
requirements-freqai.txt
View File

@ -6,6 +6,6 @@
scikit-learn==1.1.3 scikit-learn==1.1.3
joblib==1.2.0 joblib==1.2.0
catboost==1.1.1; platform_machine != 'aarch64' catboost==1.1.1; platform_machine != 'aarch64'
lightgbm==3.3.3 lightgbm==3.3.4
xgboost==1.7.2 xgboost==1.7.2
tensorboard==2.11.0 tensorboard==2.11.0

2
requirements-hyperopt.txt
View File

@ -2,7 +2,7 @@
-r requirements.txt -r requirements.txt
# Required for hyperopt # Required for hyperopt
scipy==1.9.3 scipy==1.10.0
scikit-learn==1.1.3 scikit-learn==1.1.3
scikit-optimize==0.9.0 scikit-optimize==0.9.0
filelock==3.9.0 filelock==3.9.0

8
requirements.txt
View File

@ -2,12 +2,12 @@ numpy==1.24.1
pandas==1.5.2 pandas==1.5.2
pandas-ta==0.3.14b pandas-ta==0.3.14b
ccxt==2.5.46 ccxt==2.5.56
# Pin cryptography for now due to rust build errors with piwheels # Pin cryptography for now due to rust build errors with piwheels
cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.1; platform_machine == 'armv7l'
cryptography==38.0.4; platform_machine != 'armv7l' cryptography==38.0.4; platform_machine != 'armv7l'
aiohttp==3.8.3 aiohttp==3.8.3
SQLAlchemy==1.4.45 SQLAlchemy==1.4.46
python-telegram-bot==13.15 python-telegram-bot==13.15
arrow==1.2.3 arrow==1.2.3
cachetools==4.2.2 cachetools==4.2.2
@ -30,13 +30,13 @@ py_find_1st==1.1.5
# Load ticker files 30% faster # Load ticker files 30% faster
python-rapidjson==1.9 python-rapidjson==1.9
# Properly format api responses # Properly format api responses
orjson==3.8.3 orjson==3.8.4
# Notify systemd # Notify systemd
sdnotify==0.3.2 sdnotify==0.3.2
# API Server # API Server
fastapi==0.88.0 fastapi==0.89.0
pydantic==1.10.4 pydantic==1.10.4
uvicorn==0.20.0 uvicorn==0.20.0
pyjwt==2.6.0 pyjwt==2.6.0

6
tests/conftest.py
View File

@ -2606,6 +2606,8 @@ def open_trade():
ft_order_side='buy', ft_order_side='buy',
ft_pair=trade.pair, ft_pair=trade.pair,
ft_is_open=False, ft_is_open=False,
ft_amount=trade.amount,
ft_price=trade.open_rate,
order_id='123456789', order_id='123456789',
status="closed", status="closed",
symbol=trade.pair, symbol=trade.pair,
@ -2642,6 +2644,8 @@ def open_trade_usdt():
ft_order_side='buy', ft_order_side='buy',
ft_pair=trade.pair, ft_pair=trade.pair,
ft_is_open=False, ft_is_open=False,
ft_amount=trade.amount,
ft_price=trade.open_rate,
order_id='123456789', order_id='123456789',
status="closed", status="closed",
symbol=trade.pair, symbol=trade.pair,
@ -2659,6 +2663,8 @@ def open_trade_usdt():
ft_order_side='exit', ft_order_side='exit',
ft_pair=trade.pair, ft_pair=trade.pair,
ft_is_open=True, ft_is_open=True,
ft_amount=trade.amount,
ft_price=trade.open_rate,
order_id='123456789_exit', order_id='123456789_exit',
status="open", status="open",
symbol=trade.pair, symbol=trade.pair,

4
tests/exchange/test_ccxt_compat.py
View File

@ -77,8 +77,8 @@ EXCHANGES = {
'leverage_in_spot_market': True, 'leverage_in_spot_market': True,
}, },
'huobi': { 'huobi': {
'pair': 'BTC/USDT', 'pair': 'ETH/BTC',
'stake_currency': 'USDT', 'stake_currency': 'BTC',
'hasQuoteVolume': True, 'hasQuoteVolume': True,
'timeframe': '5m', 'timeframe': '5m',
'futures': False, 'futures': False,

4
tests/freqai/test_freqai_datakitchen.py
View File

@ -82,7 +82,7 @@ def test_compute_distances(mocker, freqai_conf):
freqai = make_data_dictionary(mocker, freqai_conf) freqai = make_data_dictionary(mocker, freqai_conf)
freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 1}) freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 1})
avg_mean_dist = freqai.dk.compute_distances() avg_mean_dist = freqai.dk.compute_distances()
assert round(avg_mean_dist, 2) == 1.99 assert round(avg_mean_dist, 2) == 1.98
def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf, caplog): def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf, caplog):
@ -90,7 +90,7 @@ def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf,
freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 0.1}) freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 0.1})
freqai.dk.use_SVM_to_remove_outliers(predict=False) freqai.dk.use_SVM_to_remove_outliers(predict=False)
assert log_has_re( assert log_has_re(
"SVM detected 7.36%", "SVM detected 7.83%",
caplog, caplog,
) )

39
tests/freqai/test_freqai_interface.py
View File

@ -222,6 +222,9 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
if 'test_4ac' in model: if 'test_4ac' in model:
freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models") freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models")
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"indicator_periods_candles": [2]})
strategy = get_patched_freqai_strategy(mocker, freqai_conf) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange) strategy.dp = DataProvider(freqai_conf, exchange)
@ -232,15 +235,14 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk) freqai.dd.load_all_pair_histories(timerange, freqai.dk)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180110-20180130")
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) _, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
df = base_df[freqai_conf["timeframe"]]
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
df = freqai.cache_corr_pairlist_dfs(df, freqai.dk)
for i in range(5): for i in range(5):
df[f'%-constant_{i}'] = i df[f'%-constant_{i}'] = i
metadata = {"pair": "LTC/BTC"} metadata = {"pair": "LTC/BTC"}
freqai.start_backtesting(df, metadata, freqai.dk) freqai.start_backtesting(df, metadata, freqai.dk, strategy)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
assert len(model_folders) == num_files assert len(model_folders) == num_files
@ -261,6 +263,8 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180120-20180124"}) freqai_conf.update({"timerange": "20180120-20180124"})
freqai_conf.get("freqai", {}).update({"backtest_period_days": 0.5}) freqai_conf.get("freqai", {}).update({"backtest_period_days": 0.5})
freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf.get("freqai", {}).update({"save_backtest_models": True})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"indicator_periods_candles": [2]})
strategy = get_patched_freqai_strategy(mocker, freqai_conf) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange) strategy.dp = DataProvider(freqai_conf, exchange)
@ -271,12 +275,11 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf):
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk) freqai.dd.load_all_pair_histories(timerange, freqai.dk)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180110-20180130")
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) _, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
df = base_df[freqai_conf["timeframe"]]
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
metadata = {"pair": "LTC/BTC"} metadata = {"pair": "LTC/BTC"}
freqai.start_backtesting(df, metadata, freqai.dk) freqai.start_backtesting(df, metadata, freqai.dk, strategy)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
assert len(model_folders) == 9 assert len(model_folders) == 9
@ -287,6 +290,8 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf):
def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqai_conf.update({"timerange": "20180120-20180130"}) freqai_conf.update({"timerange": "20180120-20180130"})
freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) freqai_conf.get("freqai", {}).update({"save_backtest_models": True})
freqai_conf.get("freqai", {}).get("feature_parameters", {}).update(
{"indicator_periods_candles": [2]})
strategy = get_patched_freqai_strategy(mocker, freqai_conf) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange) strategy.dp = DataProvider(freqai_conf, exchange)
@ -296,15 +301,14 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqai.dk = FreqaiDataKitchen(freqai_conf) freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk) freqai.dd.load_all_pair_histories(timerange, freqai.dk)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180101-20180130")
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) _, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
df = base_df[freqai_conf["timeframe"]]
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
pair = "ADA/BTC" pair = "ADA/BTC"
metadata = {"pair": pair} metadata = {"pair": pair}
freqai.dk.pair = pair freqai.dk.pair = pair
freqai.start_backtesting(df, metadata, freqai.dk) freqai.start_backtesting(df, metadata, freqai.dk, strategy)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
assert len(model_folders) == 2 assert len(model_folders) == 2
@ -322,14 +326,13 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk) freqai.dd.load_all_pair_histories(timerange, freqai.dk)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180110-20180130")
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) _, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
df = base_df[freqai_conf["timeframe"]]
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
pair = "ADA/BTC" pair = "ADA/BTC"
metadata = {"pair": pair} metadata = {"pair": pair}
freqai.dk.pair = pair freqai.dk.pair = pair
freqai.start_backtesting(df, metadata, freqai.dk) freqai.start_backtesting(df, metadata, freqai.dk, strategy)
assert log_has_re( assert log_has_re(
"Found backtesting prediction file ", "Found backtesting prediction file ",
@ -339,7 +342,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
pair = "ETH/BTC" pair = "ETH/BTC"
metadata = {"pair": pair} metadata = {"pair": pair}
freqai.dk.pair = pair freqai.dk.pair = pair
freqai.start_backtesting(df, metadata, freqai.dk) freqai.start_backtesting(df, metadata, freqai.dk, strategy)
path = (freqai.dd.full_path / freqai.dk.backtest_predictions_folder) path = (freqai.dd.full_path / freqai.dk.backtest_predictions_folder)
prediction_files = [x for x in path.iterdir() if x.is_file()] prediction_files = [x for x in path.iterdir() if x.is_file()]

10
tests/persistence/test_persistence.py
View File

@ -1870,11 +1870,13 @@ def test_get_exit_order_count(fee, is_short):
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
def test_update_order_from_ccxt(caplog): def test_update_order_from_ccxt(caplog):
# Most basic order return (only has orderid) # Most basic order return (only has orderid)
o = Order.parse_from_ccxt_object({'id': '1234'}, 'ADA/USDT', 'buy') o = Order.parse_from_ccxt_object({'id': '1234'}, 'ADA/USDT', 'buy', 20.01, 1234.6)
assert isinstance(o, Order) assert isinstance(o, Order)
assert o.ft_pair == 'ADA/USDT' assert o.ft_pair == 'ADA/USDT'
assert o.ft_order_side == 'buy' assert o.ft_order_side == 'buy'
assert o.order_id == '1234' assert o.order_id == '1234'
assert o.ft_price == 1234.6
assert o.ft_amount == 20.01
assert o.ft_is_open assert o.ft_is_open
ccxt_order = { ccxt_order = {
'id': '1234', 'id': '1234',
@ -1888,13 +1890,15 @@ def test_update_order_from_ccxt(caplog):
'status': 'open', 'status': 'open',
'timestamp': 1599394315123 'timestamp': 1599394315123
} }
o = Order.parse_from_ccxt_object(ccxt_order, 'ADA/USDT', 'buy') o = Order.parse_from_ccxt_object(ccxt_order, 'ADA/USDT', 'buy', 20.01, 1234.6)
assert isinstance(o, Order) assert isinstance(o, Order)
assert o.ft_pair == 'ADA/USDT' assert o.ft_pair == 'ADA/USDT'
assert o.ft_order_side == 'buy' assert o.ft_order_side == 'buy'
assert o.order_id == '1234' assert o.order_id == '1234'
assert o.order_type == 'limit' assert o.order_type == 'limit'
assert o.price == 1234.5 assert o.price == 1234.5
assert o.ft_price == 1234.6
assert o.ft_amount == 20.01
assert o.filled == 9 assert o.filled == 9
assert o.remaining == 11 assert o.remaining == 11
assert o.order_date is not None assert o.order_date is not None
@ -2539,6 +2543,8 @@ def test_recalc_trade_from_orders_dca(data) -> None:
ft_pair=trade.pair, ft_pair=trade.pair,
order_id=f"order_{order[0]}_{idx}", order_id=f"order_{order[0]}_{idx}",
ft_is_open=False, ft_is_open=False,
ft_amount=amount,
ft_price=price,
status="closed", status="closed",
symbol=trade.pair, symbol=trade.pair,
order_type="market", order_type="market",

11
tests/plugins/test_protections.py
View File

@ -39,6 +39,8 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool,
order_id=f'{pair}-{trade.entry_side}-{trade.open_date}', order_id=f'{pair}-{trade.entry_side}-{trade.open_date}',
ft_is_open=False, ft_is_open=False,
ft_pair=pair, ft_pair=pair,
ft_amount=trade.amount,
ft_price=trade.open_rate,
amount=trade.amount, amount=trade.amount,
filled=trade.amount, filled=trade.amount,
remaining=0, remaining=0,
@ -49,16 +51,19 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool,
side=trade.entry_side, side=trade.entry_side,
)) ))
if not is_open: if not is_open:
close_price = open_rate * (2 - profit_rate if is_short else profit_rate)
trade.orders.append(Order( trade.orders.append(Order(
ft_order_side=trade.exit_side, ft_order_side=trade.exit_side,
order_id=f'{pair}-{trade.exit_side}-{trade.close_date}', order_id=f'{pair}-{trade.exit_side}-{trade.close_date}',
ft_is_open=False, ft_is_open=False,
ft_pair=pair, ft_pair=pair,
ft_amount=trade.amount,
ft_price=trade.open_rate,
amount=trade.amount, amount=trade.amount,
filled=trade.amount, filled=trade.amount,
remaining=0, remaining=0,
price=open_rate * (2 - profit_rate if is_short else profit_rate), price=close_price,
average=open_rate * (2 - profit_rate if is_short else profit_rate), average=close_price,
status="closed", status="closed",
order_type="market", order_type="market",
side=trade.exit_side, side=trade.exit_side,
@ -66,7 +71,7 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool,
trade.recalc_open_trade_value() trade.recalc_open_trade_value()
if not is_open: if not is_open:
trade.close(open_rate * (2 - profit_rate if is_short else profit_rate)) trade.close(close_price)
trade.exit_reason = exit_reason trade.exit_reason = exit_reason
Trade.query.session.add(trade) Trade.query.session.add(trade)

41
tests/rpc/test_rpc_apiserver.py
View File

@ -1,8 +1,6 @@
""" """
Unit test file for rpc/api_server.py Unit test file for rpc/api_server.py
""" """
import json
import logging import logging
import time import time
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
@ -68,22 +66,23 @@ def botclient(default_conf, mocker):
ApiServer.shutdown() ApiServer.shutdown()
def client_post(client, url, data={}): def client_post(client: TestClient, url, data={}):
return client.post(url, return client.post(url,
content=data, json=data,
headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS),
'Origin': 'http://example.com', 'Origin': 'http://example.com',
'content-type': 'application/json' 'content-type': 'application/json'
}) })
def client_get(client, url): def client_get(client: TestClient, url):
# Add fake Origin to ensure CORS kicks in # Add fake Origin to ensure CORS kicks in
return client.get(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), return client.get(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS),
'Origin': 'http://example.com'}) 'Origin': 'http://example.com'})
def client_delete(client, url): def client_delete(client: TestClient, url):
# Add fake Origin to ensure CORS kicks in # Add fake Origin to ensure CORS kicks in
return client.delete(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), return client.delete(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS),
'Origin': 'http://example.com'}) 'Origin': 'http://example.com'})
@ -561,7 +560,7 @@ def test_api_locks(botclient):
assert rc.json()['lock_count'] == 1 assert rc.json()['lock_count'] == 1
rc = client_post(client, f"{BASE_URI}/locks/delete", rc = client_post(client, f"{BASE_URI}/locks/delete",
data='{"pair": "XRP/BTC"}') data={"pair": "XRP/BTC"})
assert_response(rc) assert_response(rc)
assert rc.json()['lock_count'] == 0 assert rc.json()['lock_count'] == 0
@ -1062,7 +1061,7 @@ def test_api_blacklist(botclient, mocker):
# Add ETH/BTC to blacklist # Add ETH/BTC to blacklist
rc = client_post(client, f"{BASE_URI}/blacklist", rc = client_post(client, f"{BASE_URI}/blacklist",
data='{"blacklist": ["ETH/BTC"]}') data={"blacklist": ["ETH/BTC"]})
assert_response(rc) assert_response(rc)
assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"], assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC"],
"blacklist_expanded": ["ETH/BTC"], "blacklist_expanded": ["ETH/BTC"],
@ -1072,7 +1071,7 @@ def test_api_blacklist(botclient, mocker):
} }
rc = client_post(client, f"{BASE_URI}/blacklist", rc = client_post(client, f"{BASE_URI}/blacklist",
data='{"blacklist": ["XRP/.*"]}') data={"blacklist": ["XRP/.*"]})
assert_response(rc) assert_response(rc)
assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"], assert rc.json() == {"blacklist": ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"],
"blacklist_expanded": ["ETH/BTC", "XRP/BTC", "XRP/USDT"], "blacklist_expanded": ["ETH/BTC", "XRP/BTC", "XRP/USDT"],
@ -1134,7 +1133,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
ftbot, client = botclient ftbot, client = botclient
rc = client_post(client, f"{BASE_URI}/{endpoint}", rc = client_post(client, f"{BASE_URI}/{endpoint}",
data='{"pair": "ETH/BTC"}') data={"pair": "ETH/BTC"})
assert_response(rc, 502) assert_response(rc, 502)
assert rc.json() == {"error": f"Error querying /api/v1/{endpoint}: Force_entry not enabled."} assert rc.json() == {"error": f"Error querying /api/v1/{endpoint}: Force_entry not enabled."}
@ -1144,7 +1143,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
fbuy_mock = MagicMock(return_value=None) fbuy_mock = MagicMock(return_value=None)
mocker.patch("freqtrade.rpc.RPC._rpc_force_entry", fbuy_mock) mocker.patch("freqtrade.rpc.RPC._rpc_force_entry", fbuy_mock)
rc = client_post(client, f"{BASE_URI}/{endpoint}", rc = client_post(client, f"{BASE_URI}/{endpoint}",
data='{"pair": "ETH/BTC"}') data={"pair": "ETH/BTC"})
assert_response(rc) assert_response(rc)
assert rc.json() == {"status": "Error entering long trade for pair ETH/BTC."} assert rc.json() == {"status": "Error entering long trade for pair ETH/BTC."}
@ -1171,7 +1170,7 @@ def test_api_force_entry(botclient, mocker, fee, endpoint):
mocker.patch("freqtrade.rpc.RPC._rpc_force_entry", fbuy_mock) mocker.patch("freqtrade.rpc.RPC._rpc_force_entry", fbuy_mock)
rc = client_post(client, f"{BASE_URI}/{endpoint}", rc = client_post(client, f"{BASE_URI}/{endpoint}",
data='{"pair": "ETH/BTC"}') data={"pair": "ETH/BTC"})
assert_response(rc) assert_response(rc)
assert rc.json() == { assert rc.json() == {
'amount': 1.0, 'amount': 1.0,
@ -1246,7 +1245,7 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets):
patch_get_signal(ftbot) patch_get_signal(ftbot)
rc = client_post(client, f"{BASE_URI}/forceexit", rc = client_post(client, f"{BASE_URI}/forceexit",
data='{"tradeid": "1"}') data={"tradeid": "1"})
assert_response(rc, 502) assert_response(rc, 502)
assert rc.json() == {"error": "Error querying /api/v1/forceexit: invalid argument"} assert rc.json() == {"error": "Error querying /api/v1/forceexit: invalid argument"}
Trade.query.session.rollback() Trade.query.session.rollback()
@ -1255,7 +1254,7 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets):
trade = Trade.get_trades([Trade.id == 5]).first() trade = Trade.get_trades([Trade.id == 5]).first()
assert pytest.approx(trade.amount) == 123 assert pytest.approx(trade.amount) == 123
rc = client_post(client, f"{BASE_URI}/forceexit", rc = client_post(client, f"{BASE_URI}/forceexit",
data='{"tradeid": "5", "ordertype": "market", "amount": 23}') data={"tradeid": "5", "ordertype": "market", "amount": 23})
assert_response(rc) assert_response(rc)
assert rc.json() == {'result': 'Created sell order for trade 5.'} assert rc.json() == {'result': 'Created sell order for trade 5.'}
Trade.query.session.rollback() Trade.query.session.rollback()
@ -1265,7 +1264,7 @@ def test_api_forceexit(botclient, mocker, ticker, fee, markets):
assert trade.is_open is True assert trade.is_open is True
rc = client_post(client, f"{BASE_URI}/forceexit", rc = client_post(client, f"{BASE_URI}/forceexit",
data='{"tradeid": "5"}') data={"tradeid": "5"})
assert_response(rc) assert_response(rc)
assert rc.json() == {'result': 'Created sell order for trade 5.'} assert rc.json() == {'result': 'Created sell order for trade 5.'}
Trade.query.session.rollback() Trade.query.session.rollback()
@ -1616,7 +1615,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
"dry_run_wallet": 1000, "dry_run_wallet": 1000,
"enable_protections": False "enable_protections": False
} }
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) rc = client_post(client, f"{BASE_URI}/backtest", data=data)
assert_response(rc) assert_response(rc)
result = rc.json() result = rc.json()
@ -1667,7 +1666,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
assert result['status'] == 'running' assert result['status'] == 'running'
# Post to backtest that's still running # Post to backtest that's still running
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) rc = client_post(client, f"{BASE_URI}/backtest", data=data)
assert_response(rc, 502) assert_response(rc, 502)
result = rc.json() result = rc.json()
assert 'Bot Background task already running' in result['error'] assert 'Bot Background task already running' in result['error']
@ -1675,7 +1674,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
ApiServer._bgtask_running = False ApiServer._bgtask_running = False
# Rerun backtest (should get previous result) # Rerun backtest (should get previous result)
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) rc = client_post(client, f"{BASE_URI}/backtest", data=data)
assert_response(rc) assert_response(rc)
result = rc.json() result = rc.json()
assert log_has_re('Reusing result of previous backtest.*', caplog) assert log_has_re('Reusing result of previous backtest.*', caplog)
@ -1684,7 +1683,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy',
side_effect=DependencyException()) side_effect=DependencyException())
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) rc = client_post(client, f"{BASE_URI}/backtest", data=data)
assert log_has("Backtesting caused an error: ", caplog) assert log_has("Backtesting caused an error: ", caplog)
# Delete backtesting to avoid leakage since the backtest-object may stick around. # Delete backtesting to avoid leakage since the backtest-object may stick around.
@ -1698,7 +1697,7 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
# Disallow base64 strategies # Disallow base64 strategies
data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik=" data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik="
rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) rc = client_post(client, f"{BASE_URI}/backtest", data=data)
assert_response(rc, 500) assert_response(rc, 500)
@ -1766,7 +1765,7 @@ def test_api_ws_subscribe(botclient, mocker):
assert sub_mock.call_count == 1 assert sub_mock.call_count == 1
def test_api_ws_requests(botclient, mocker, caplog): def test_api_ws_requests(botclient, caplog):
caplog.set_level(logging.DEBUG) caplog.set_level(logging.DEBUG)
ftbot, client = botclient ftbot, client = botclient

2
tests/rpc/test_rpc_telegram.py
View File

@ -253,6 +253,8 @@ def test_telegram_status_multi_entry(default_conf, update, mocker, fee) -> None:
ft_order_side='buy', ft_order_side='buy',
ft_pair=trade.pair, ft_pair=trade.pair,
ft_is_open=False, ft_is_open=False,
ft_amount=trade.amount,
ft_price=trade.open_rate,
status="closed", status="closed",
symbol=trade.pair, symbol=trade.pair,
order_type="market", order_type="market",

62
tests/strategy/strats/freqai_rl_test_strat.py
View File

@ -1,11 +1,10 @@
import logging import logging
from functools import reduce from functools import reduce
import pandas as pd
import talib.abstract as ta import talib.abstract as ta
from pandas import DataFrame from pandas import DataFrame
from freqtrade.strategy import IStrategy, merge_informative_pair from freqtrade.strategy import IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -22,52 +21,39 @@ class freqai_rl_test_strat(IStrategy):
process_only_new_candles = True process_only_new_candles = True
stoploss = -0.05 stoploss = -0.05
use_exit_signal = True use_exit_signal = True
startup_candle_count: int = 30 startup_candle_count: int = 300
can_short = False can_short = False
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
if informative is None: dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods return dataframe
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t) def feature_engineering_expand_basic(self, dataframe: DataFrame, **kwargs):
informative[f"%-{pair}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
# The following columns are necessary for RL models. dataframe["%-pct-change"] = dataframe["close"].pct_change()
informative[f"%-{pair}raw_close"] = informative["close"] dataframe["%-raw_volume"] = dataframe["volume"]
informative[f"%-{pair}raw_open"] = informative["open"]
informative[f"%-{pair}raw_high"] = informative["high"]
informative[f"%-{pair}raw_low"] = informative["low"]
indicators = [col for col in informative if col.startswith("%")] return dataframe
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) def feature_engineering_standard(self, dataframe, **kwargs):
skip_columns = [
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
# function to populate indicators during training). Notice how we ensure not to dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
# add them multiple times
if set_generalized_indicators:
# For RL, there are no direct targets to set. This is filler (neutral)
# until the agent sends an action.
df["&-action"] = 0
return df dataframe["%-raw_close"] = dataframe["close"]
dataframe["%-raw_open"] = dataframe["open"]
dataframe["%-raw_high"] = dataframe["high"]
dataframe["%-raw_low"] = dataframe["low"]
return dataframe
def set_freqai_targets(self, dataframe, **kwargs):
dataframe["&-action"] = 0
return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

61
tests/strategy/strats/freqai_test_classifier.py
View File

@ -2,11 +2,10 @@ import logging
from functools import reduce from functools import reduce
import numpy as np import numpy as np
import pandas as pd
import talib.abstract as ta import talib.abstract as ta
from pandas import DataFrame from pandas import DataFrame
from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -57,55 +56,35 @@ class freqai_test_classifier(IStrategy):
informative_pairs.append((pair, tf)) informative_pairs.append((pair, tf))
return informative_pairs return informative_pairs
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
coin = pair.split('/')[0] dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
if informative is None: return dataframe
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods def feature_engineering_expand_basic(self, dataframe: DataFrame, **kwargs):
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t) dataframe["%-pct-change"] = dataframe["close"].pct_change()
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) dataframe["%-raw_volume"] = dataframe["volume"]
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) dataframe["%-raw_price"] = dataframe["close"]
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"%-{coin}pct-change"] = informative["close"].pct_change() return dataframe
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")] def feature_engineering_standard(self, dataframe, **kwargs):
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
skip_columns = [ dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this return dataframe
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) def set_freqai_targets(self, dataframe, **kwargs):
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py
df['&s-up_or_down'] = np.where(df["close"].shift(-100) > df["close"], 'up', 'down')
return df dataframe['&s-up_or_down'] = np.where(dataframe["close"].shift(-100) >
dataframe["close"], 'up', 'down')
return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

66
tests/strategy/strats/freqai_test_multimodel_classifier_strat.py
View File

@ -2,11 +2,10 @@ import logging
from functools import reduce from functools import reduce
import numpy as np import numpy as np
import pandas as pd
import talib.abstract as ta import talib.abstract as ta
from pandas import DataFrame from pandas import DataFrame
from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -44,59 +43,38 @@ class freqai_test_multimodel_classifier_strat(IStrategy):
) )
max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True)
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
coin = pair.split('/')[0] dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
if informative is None: return dataframe
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods def feature_engineering_expand_basic(self, dataframe: DataFrame, **kwargs):
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t) dataframe["%-pct-change"] = dataframe["close"].pct_change()
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) dataframe["%-raw_volume"] = dataframe["volume"]
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) dataframe["%-raw_price"] = dataframe["close"]
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"%-{coin}pct-change"] = informative["close"].pct_change() return dataframe
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")] def feature_engineering_standard(self, dataframe, **kwargs):
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
skip_columns = [ dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this return dataframe
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) def set_freqai_targets(self, dataframe, **kwargs):
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py
df['&s-up_or_down'] = np.where(df["close"].shift(-50) >
df["close"], 'up', 'down')
df['&s-up_or_down2'] = np.where(df["close"].shift(-50) > dataframe['&s-up_or_down'] = np.where(dataframe["close"].shift(-50) >
df["close"], 'up2', 'down2') dataframe["close"], 'up', 'down')
return df dataframe['&s-up_or_down2'] = np.where(dataframe["close"].shift(-50) >
dataframe["close"], 'up2', 'down2')
return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

70
tests/strategy/strats/freqai_test_multimodel_strat.py
View File

@ -1,11 +1,10 @@
import logging import logging
from functools import reduce from functools import reduce
import pandas as pd
import talib.abstract as ta import talib.abstract as ta
from pandas import DataFrame from pandas import DataFrame
from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -43,74 +42,53 @@ class freqai_test_multimodel_strat(IStrategy):
) )
max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True)
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
coin = pair.split('/')[0] dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
if informative is None: return dataframe
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods def feature_engineering_expand_basic(self, dataframe: DataFrame, **kwargs):
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t) dataframe["%-pct-change"] = dataframe["close"].pct_change()
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) dataframe["%-raw_volume"] = dataframe["volume"]
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) dataframe["%-raw_price"] = dataframe["close"]
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"%-{coin}pct-change"] = informative["close"].pct_change() return dataframe
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")] def feature_engineering_standard(self, dataframe, **kwargs):
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
skip_columns = [ dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this return dataframe
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) def set_freqai_targets(self, dataframe, **kwargs):
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py dataframe["&-s_close"] = (
df["&-s_close"] = ( dataframe["close"]
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean() .mean()
/ df["close"] / dataframe["close"]
- 1 - 1
) )
df["&-s_range"] = ( dataframe["&-s_range"] = (
df["close"] dataframe["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.max() .max()
- -
df["close"] dataframe["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.min() .min()
) )
return df return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

64
tests/strategy/strats/freqai_test_strat.py
View File

@ -1,11 +1,10 @@
import logging import logging
from functools import reduce from functools import reduce
import pandas as pd
import talib.abstract as ta import talib.abstract as ta
from pandas import DataFrame from pandas import DataFrame
from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, merge_informative_pair from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -43,62 +42,41 @@ class freqai_test_strat(IStrategy):
) )
max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True)
def populate_any_indicators( def feature_engineering_expand_all(self, dataframe, period, **kwargs):
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
coin = pair.split('/')[0] dataframe["%-rsi-period"] = ta.RSI(dataframe, timeperiod=period)
dataframe["%-mfi-period"] = ta.MFI(dataframe, timeperiod=period)
dataframe["%-adx-period"] = ta.ADX(dataframe, timeperiod=period)
if informative is None: return dataframe
informative = self.dp.get_pair_dataframe(pair, tf)
# first loop is automatically duplicating indicators for time periods def feature_engineering_expand_basic(self, dataframe: DataFrame, **kwargs):
for t in self.freqai_info["feature_parameters"]["indicator_periods_candles"]:
t = int(t) dataframe["%-pct-change"] = dataframe["close"].pct_change()
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t) dataframe["%-raw_volume"] = dataframe["volume"]
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t) dataframe["%-raw_price"] = dataframe["close"]
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"%-{coin}pct-change"] = informative["close"].pct_change() return dataframe
informative[f"%-{coin}raw_volume"] = informative["volume"]
informative[f"%-{coin}raw_price"] = informative["close"]
indicators = [col for col in informative if col.startswith("%")] def feature_engineering_standard(self, dataframe, **kwargs):
# This loop duplicates and shifts all indicators to add a sense of recency to data
for n in range(self.freqai_info["feature_parameters"]["include_shifted_candles"] + 1):
if n == 0:
continue
informative_shift = informative[indicators].shift(n)
informative_shift = informative_shift.add_suffix("_shift-" + str(n))
informative = pd.concat((informative, informative_shift), axis=1)
df = merge_informative_pair(df, informative, self.config["timeframe"], tf, ffill=True) dataframe["%-day_of_week"] = dataframe["date"].dt.dayofweek
skip_columns = [ dataframe["%-hour_of_day"] = dataframe["date"].dt.hour
(s + "_" + tf) for s in ["date", "open", "high", "low", "close", "volume"]
]
df = df.drop(columns=skip_columns)
# Add generalized indicators here (because in live, it will call this return dataframe
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if set_generalized_indicators:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below) def set_freqai_targets(self, dataframe, **kwargs):
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py dataframe["&-s_close"] = (
df["&-s_close"] = ( dataframe["close"]
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"]) .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
.rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
.mean() .mean()
/ df["close"] / dataframe["close"]
- 1 - 1
) )
return df return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

3
tests/test_freqtradebot.py
View File

@ -1168,6 +1168,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf_usdt, fee, caplog, is_
order_id='100', order_id='100',
ft_pair=trade.pair, ft_pair=trade.pair,
ft_is_open=True, ft_is_open=True,
ft_amount=trade.amount,
ft_price=0.0,
)) ))
assert trade assert trade
@ -4615,6 +4617,7 @@ def test_get_real_amount_open_trade_usdt(default_conf_usdt, fee, mocker):
'amount': amount, 'amount': amount,
'status': 'open', 'status': 'open',
'side': 'buy', 'side': 'buy',
'price': 0.245441,
} }
freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt) freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
order_obj = Order.parse_from_ccxt_object(order, 'LTC/ETH', 'buy') order_obj = Order.parse_from_ccxt_object(order, 'LTC/ETH', 'buy')