From f8331e0326e12cee9ec9a84fc1c5a7ba76731ab9 Mon Sep 17 00:00:00 2001 From: th0rntwig Date: Thu, 13 Oct 2022 10:53:25 +0200 Subject: [PATCH 1/4] Add model libs info --- docs/freqai-configuration.md | 38 +++++++++++++++++++++++++++++----- docs/freqai-parameter-table.md | 2 +- docs/freqai-running.md | 4 ++-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 737f64824..2d40918e5 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -204,17 +204,45 @@ If this value is set, FreqAI will initially use the predictions from the trainin ## Using different prediction models -FreqAI has multiple example prediction model libraries that are ready to be used as is via the flag `--freqaimodel`. These libraries include `Catboost`, `LightGBM`, and `XGBoost` regression, classification, and multi-target models, and can be found in `freqai/prediction_models/`. However, it is possible to customize and create your own prediction models using the `IFreqaiModel` class. You are encouraged to inherit `fit()`, `train()`, and `predict()` to let these customize various aspects of the training procedures. +FreqAI has multiple example prediction model libraries that are ready to be used as is via the flag `--freqaimodel`. These libraries include `CatBoost`, `LightGBM`, and `XGBoost` regression, classification, and multi-target models, and can be found in `freqai/prediction_models/`. -You can place custom FreqAI models in `user_data/freqaimodels` - and freqtrade will pick them up from there based on the provided `--freqaimodel` name - which has to correspond to the class name of your custom model. +Regression and classification models differ in what targets they predict - a regression model will predict a target of continous values, for example what price BTC will be at tomorrow, whilst a classifier will predict a target of descrete values, for example if the price of BTC will go up tomorrow or not. This means that you have to specify your targets differently depending on which model type you are using (see details [below](#setting-model-targets)). + +All of the aforementioned model libraries implement gradient boosted decision tree algorithms. They all work on the principle of emsemble learning, where predictions from multiple simple learners are combined to get a final prediction that is more stable and generalized. The simple learners in this case are decision trees. Gradient boosting refers to the method of learning, where each simple learner is built in sequence - the subsequent learner is used to improve on the error from the previous learner. If you want to learn more about the different model libraries you can find the information in their respective docs: + +* CatBoost: https://catboost.ai/en/docs/ +* LightGBM: https://lightgbm.readthedocs.io/en/v3.3.2/# +* XGBoost: https://xgboost.readthedocs.io/en/stable/# + +There are also numerous online articles describing and comparing the algorithms. Some relatively light-weight examples would be [CatBoost vs. LightGBM vs. XGBoost — Which is the best algorithm?](https://towardsdatascience.com/catboost-vs-lightgbm-vs-xgboost-c80f40662924#:~:text=In%20CatBoost%2C%20symmetric%20trees%2C%20or,the%20same%20depth%20can%20differ.) and [XGBoost, LightGBM or CatBoost — which boosting algorithm should I use?](https://medium.com/riskified-technology/xgboost-lightgbm-or-catboost-which-boosting-algorithm-should-i-use-e7fda7bb36bc). Keep in mind that the performance of each model is highly dependent on the application and so any reported metrics might not be true for your particular use of the model. + +Apart from the models already vailable in FreqAI, it is also possible to customize and create your own prediction models using the `IFreqaiModel` class. You are encouraged to inherit `fit()`, `train()`, and `predict()` to let these customize various aspects of the training procedures. You can place custom FreqAI models in `user_data/freqaimodels` - and freqtrade will pick them up from there based on the provided `--freqaimodel` name - which has to correspond to the class name of your custom model. Make sure to use unique names to avoid overriding built-in models. -### Setting classifier targets +### Setting model targets -FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example: +#### Regressors + +If you are using a regressor, you need to specify a target that has continous values. FreqAI includes a variety of regressors, such as the `CatboostRegressor`via the flag `--freqaimodel CatboostRegressor`. An example of how you could set a regression target for predicting the price 100 candles into the future would be + +```python +df['&s-close_price'] = df['close'].shift(-100) +``` + +If you want to predict multiple targets, you need to define multiple labels using the same syntax as shown above. + +#### Classifiers + +If you are using a classifier, you need to specify a target that has descrete values. FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example, if you want to predict if the price 100 candles into the future goes up or down you would set ```python df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down') ``` -Additionally, the example classifier models do not accommodate multiple labels, but they do allow multi-class classification within a single label column. +If you want to predict multiple targets you must specify all labels in the same label column. You could, for example, add the label `same` to define where the price was unchanged by setting + + +```python +df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down') +df['&s-up_or_down'] = np.where( df["close"].shift(-100) == df["close"], 'same', df['&s-up_or_down']) +``` diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 38d7ece94..7d9864fc0 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -42,7 +42,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `test_size` | The fraction of data that should be used for testing instead of training.
**Datatype:** Positive float < 1. | `shuffle` | Shuffle the training data points during training. Typically, to not remove the chronological order of data in time-series forecasting, this is set to `False`.
**Datatype:** Boolean.
Defaut: `False`. | | **Model training parameters** -| `model_training_parameters` | A flexible dictionary that includes all parameters available by the selected model library. For example, if you use `LightGBMRegressor`, this dictionary can contain any parameter available by the `LightGBMRegressor` [here](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html) (external website). If you select a different model, this dictionary can contain any parameter from that model.
**Datatype:** Dictionary. +| `model_training_parameters` | A flexible dictionary that includes all parameters available by the selected model library. For example, if you use `LightGBMRegressor`, this dictionary can contain any parameter available by the `LightGBMRegressor` [here](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html) (external website). If you select a different model, this dictionary can contain any parameter from that model. A list of the currently available models can be found [here](freqai-configuration.md#using-different-prediction-models).
**Datatype:** Dictionary. | `n_estimators` | The number of boosted trees to fit in the training of the model.
**Datatype:** Integer. | `learning_rate` | Boosting learning rate during training of the model.
**Datatype:** Float. | `n_jobs`, `thread_count`, `task_type` | Set the number of threads for parallel processing and the `task_type` (`gpu` or `cpu`). Different model libraries use different parameter names.
**Datatype:** Float. diff --git a/docs/freqai-running.md b/docs/freqai-running.md index f6aa7b2e1..9e2b13c44 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -144,14 +144,14 @@ This specific hyperopt would help you understand the appropriate `DI_values` for ## Using Tensorboard -Catboost models benefit from tracking training metrics via Tensorboard. You can take advantage of the FreqAI integration to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command: +CatBoost models benefit from tracking training metrics via Tensorboard. You can take advantage of the FreqAI integration to track training and evaluation performance across all coins and across all retrainings. Tensorboard is activated via the following command: ```bash cd freqtrade tensorboard --logdir user_data/models/unique-id ``` -where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell if the user wishes to view the output in their browser at 127.0.0.1:6060 (6060 is the default port used by Tensorboard). +where `unique-id` is the `identifier` set in the `freqai` configuration file. This command must be run in a separate shell if you wish to view the output in your browser at 127.0.0.1:6060 (6060 is the default port used by Tensorboard). ![tensorboard](assets/tensorboard.jpg) From c71c0e8da0e3326c9de12e97ffc36333baf00c3a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Oct 2022 18:16:39 +0200 Subject: [PATCH 2/4] Fix some typos --- docs/freqai-configuration.md | 13 ++++++------- docs/freqai-running.md | 1 - 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/freqai-configuration.md b/docs/freqai-configuration.md index 2d40918e5..d162fe373 100644 --- a/docs/freqai-configuration.md +++ b/docs/freqai-configuration.md @@ -204,11 +204,11 @@ If this value is set, FreqAI will initially use the predictions from the trainin ## Using different prediction models -FreqAI has multiple example prediction model libraries that are ready to be used as is via the flag `--freqaimodel`. These libraries include `CatBoost`, `LightGBM`, and `XGBoost` regression, classification, and multi-target models, and can be found in `freqai/prediction_models/`. +FreqAI has multiple example prediction model libraries that are ready to be used as is via the flag `--freqaimodel`. These libraries include `CatBoost`, `LightGBM`, and `XGBoost` regression, classification, and multi-target models, and can be found in `freqai/prediction_models/`. -Regression and classification models differ in what targets they predict - a regression model will predict a target of continous values, for example what price BTC will be at tomorrow, whilst a classifier will predict a target of descrete values, for example if the price of BTC will go up tomorrow or not. This means that you have to specify your targets differently depending on which model type you are using (see details [below](#setting-model-targets)). +Regression and classification models differ in what targets they predict - a regression model will predict a target of continuous values, for example what price BTC will be at tomorrow, whilst a classifier will predict a target of discrete values, for example if the price of BTC will go up tomorrow or not. This means that you have to specify your targets differently depending on which model type you are using (see details [below](#setting-model-targets)). -All of the aforementioned model libraries implement gradient boosted decision tree algorithms. They all work on the principle of emsemble learning, where predictions from multiple simple learners are combined to get a final prediction that is more stable and generalized. The simple learners in this case are decision trees. Gradient boosting refers to the method of learning, where each simple learner is built in sequence - the subsequent learner is used to improve on the error from the previous learner. If you want to learn more about the different model libraries you can find the information in their respective docs: +All of the aforementioned model libraries implement gradient boosted decision tree algorithms. They all work on the principle of ensemble learning, where predictions from multiple simple learners are combined to get a final prediction that is more stable and generalized. The simple learners in this case are decision trees. Gradient boosting refers to the method of learning, where each simple learner is built in sequence - the subsequent learner is used to improve on the error from the previous learner. If you want to learn more about the different model libraries you can find the information in their respective docs: * CatBoost: https://catboost.ai/en/docs/ * LightGBM: https://lightgbm.readthedocs.io/en/v3.3.2/# @@ -216,14 +216,14 @@ All of the aforementioned model libraries implement gradient boosted decision tr There are also numerous online articles describing and comparing the algorithms. Some relatively light-weight examples would be [CatBoost vs. LightGBM vs. XGBoost — Which is the best algorithm?](https://towardsdatascience.com/catboost-vs-lightgbm-vs-xgboost-c80f40662924#:~:text=In%20CatBoost%2C%20symmetric%20trees%2C%20or,the%20same%20depth%20can%20differ.) and [XGBoost, LightGBM or CatBoost — which boosting algorithm should I use?](https://medium.com/riskified-technology/xgboost-lightgbm-or-catboost-which-boosting-algorithm-should-i-use-e7fda7bb36bc). Keep in mind that the performance of each model is highly dependent on the application and so any reported metrics might not be true for your particular use of the model. -Apart from the models already vailable in FreqAI, it is also possible to customize and create your own prediction models using the `IFreqaiModel` class. You are encouraged to inherit `fit()`, `train()`, and `predict()` to let these customize various aspects of the training procedures. You can place custom FreqAI models in `user_data/freqaimodels` - and freqtrade will pick them up from there based on the provided `--freqaimodel` name - which has to correspond to the class name of your custom model. +Apart from the models already available in FreqAI, it is also possible to customize and create your own prediction models using the `IFreqaiModel` class. You are encouraged to inherit `fit()`, `train()`, and `predict()` to customize various aspects of the training procedures. You can place custom FreqAI models in `user_data/freqaimodels` - and freqtrade will pick them up from there based on the provided `--freqaimodel` name - which has to correspond to the class name of your custom model. Make sure to use unique names to avoid overriding built-in models. ### Setting model targets #### Regressors -If you are using a regressor, you need to specify a target that has continous values. FreqAI includes a variety of regressors, such as the `CatboostRegressor`via the flag `--freqaimodel CatboostRegressor`. An example of how you could set a regression target for predicting the price 100 candles into the future would be +If you are using a regressor, you need to specify a target that has continuous values. FreqAI includes a variety of regressors, such as the `CatboostRegressor`via the flag `--freqaimodel CatboostRegressor`. An example of how you could set a regression target for predicting the price 100 candles into the future would be ```python df['&s-close_price'] = df['close'].shift(-100) @@ -233,7 +233,7 @@ If you want to predict multiple targets, you need to define multiple labels usin #### Classifiers -If you are using a classifier, you need to specify a target that has descrete values. FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example, if you want to predict if the price 100 candles into the future goes up or down you would set +If you are using a classifier, you need to specify a target that has discrete values. FreqAI includes a variety of classifiers, such as the `CatboostClassifier` via the flag `--freqaimodel CatboostClassifier`. If you elects to use a classifier, the classes need to be set using strings. For example, if you want to predict if the price 100 candles into the future goes up or down you would set ```python df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down') @@ -241,7 +241,6 @@ df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'do If you want to predict multiple targets you must specify all labels in the same label column. You could, for example, add the label `same` to define where the price was unchanged by setting - ```python df['&s-up_or_down'] = np.where( df["close"].shift(-100) > df["close"], 'up', 'down') df['&s-up_or_down'] = np.where( df["close"].shift(-100) == df["close"], 'same', df['&s-up_or_down']) diff --git a/docs/freqai-running.md b/docs/freqai-running.md index 9e2b13c44..7972a6c92 100644 --- a/docs/freqai-running.md +++ b/docs/freqai-running.md @@ -155,7 +155,6 @@ where `unique-id` is the `identifier` set in the `freqai` configuration file. Th ![tensorboard](assets/tensorboard.jpg) - ## Setting up a follower You can indicate to the bot that it should not train models, but instead should look for models trained by a leader with a specific `identifier` by defining: From 7672586de9ff92f20993977802374156d533e64a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Oct 2022 19:43:37 +0200 Subject: [PATCH 3/4] Fix unreliable hyperopt test --- tests/optimize/test_hyperopt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 5666ebabc..c52bc9799 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -910,8 +910,9 @@ def test_in_strategy_auto_hyperopt_with_parallel(mocker, hyperopt_conf, tmpdir, }) hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.exchange.get_max_leverage = lambda *x, **xx: 1.0 - hyperopt.backtesting.exchange.get_min_pair_stake_amount = lambda *x, **xx: 1.0 + hyperopt.backtesting.exchange.get_min_pair_stake_amount = lambda *x, **xx: 0.00001 hyperopt.backtesting.exchange.get_max_pair_stake_amount = lambda *x, **xx: 100.0 + hyperopt.backtesting.exchange._markets = get_markets() assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter) From f0194710513de86e59a086be28518180dbf52cd5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 13 Oct 2022 19:30:59 +0200 Subject: [PATCH 4/4] Don't round prices if no custom prices have been used closes #7573 --- freqtrade/optimize/backtesting.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 83159dfe4..720069f84 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -617,13 +617,16 @@ class Backtesting: exit_reason = row[EXIT_TAG_IDX] # Custom exit pricing only for exit-signals if order_type == 'limit': - close_rate = strategy_safe_wrapper(self.strategy.custom_exit_price, - default_retval=close_rate)( + rate = strategy_safe_wrapper(self.strategy.custom_exit_price, + default_retval=close_rate)( pair=trade.pair, trade=trade, # type: ignore[arg-type] current_time=exit_candle_time, proposed_rate=close_rate, current_profit=current_profit, exit_tag=exit_reason) + if rate != close_rate: + close_rate = price_to_precision(rate, trade.price_precision, + self.precision_mode) # We can't place orders lower than current low. # freqtrade does not support this in live, and the order would fill immediately if trade.is_short: @@ -660,7 +663,6 @@ class Backtesting: # amount = amount or trade.amount amount = amount_to_contract_precision(amount or trade.amount, trade.amount_precision, self.precision_mode, trade.contract_size) - rate = price_to_precision(close_rate, trade.price_precision, self.precision_mode) order = Order( id=self.order_id_counter, ft_trade_id=trade.id, @@ -674,12 +676,12 @@ class Backtesting: side=trade.exit_side, order_type=order_type, status="open", - price=rate, - average=rate, + price=close_rate, + average=close_rate, amount=amount, filled=0, remaining=amount, - cost=amount * rate, + cost=amount * close_rate, ) trade.orders.append(order) return trade @@ -726,18 +728,21 @@ class Backtesting: def get_valid_price_and_stake( self, pair: str, row: Tuple, propose_rate: float, stake_amount: float, direction: LongShort, current_time: datetime, entry_tag: Optional[str], - trade: Optional[LocalTrade], order_type: str + trade: Optional[LocalTrade], order_type: str, price_precision: Optional[float] ) -> Tuple[float, float, float, float]: if order_type == 'limit': - propose_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, - default_retval=propose_rate)( + new_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, + default_retval=propose_rate)( pair=pair, current_time=current_time, proposed_rate=propose_rate, entry_tag=entry_tag, side=direction, ) # default value is the open rate # We can't place orders higher than current high (otherwise it'd be a stop limit entry) # which freqtrade does not support in live. + if new_rate != propose_rate: + propose_rate = price_to_precision(new_rate, price_precision, + self.precision_mode) if direction == "short": propose_rate = max(propose_rate, row[LOW_IDX]) else: @@ -799,9 +804,11 @@ class Backtesting: pos_adjust = trade is not None and requested_rate is None stake_amount_ = stake_amount or (trade.stake_amount if trade else 0.0) + precision_price = self.exchange.get_precision_price(pair) + propose_rate, stake_amount, leverage, min_stake_amount = self.get_valid_price_and_stake( pair, row, row[OPEN_IDX], stake_amount_, direction, current_time, entry_tag, trade, - order_type + order_type, precision_price, ) # replace proposed rate if another rate was requested @@ -817,8 +824,6 @@ class Backtesting: if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): self.order_id_counter += 1 base_currency = self.exchange.get_pair_base_currency(pair) - precision_price = self.exchange.get_precision_price(pair) - propose_rate = price_to_precision(propose_rate, precision_price, self.precision_mode) amount_p = (stake_amount / propose_rate) * leverage contract_size = self.exchange.get_contract_size(pair)