From 324e54c0159aa823167d90eb639ba8ba0b9aaa4d Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 26 Jul 2022 17:29:29 +0200 Subject: [PATCH] fix possible memory leak associated with Catboost Pool object --- freqtrade/freqai/freqai_interface.py | 6 +++--- .../freqai/prediction_models/BaseRegressionModel.py | 2 +- .../freqai/prediction_models/BaseTensorFlowModel.py | 2 +- .../freqai/prediction_models/CatboostPredictionModel.py | 9 ++++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 5eecffd9c..4bee3fefd 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -120,7 +120,8 @@ class IFreqaiModel(ABC): dk = self.start_backtesting(dataframe, metadata, self.dk) dataframe = dk.remove_features_from_df(dk.return_dataframe) - return self.return_values(dataframe, dk) + del dk + return self.return_values(dataframe) @threaded def start_scanning(self, strategy: IStrategy) -> None: @@ -560,12 +561,11 @@ class IFreqaiModel(ABC): """ @abstractmethod - def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: + def return_values(self, dataframe: DataFrame) -> DataFrame: """ User defines the dataframe to be returned to strategy here. :param dataframe: DataFrame = the full dataframe for the current prediction (live) or --timerange (backtesting) - :param dk: FreqaiDataKitchen = Data management/analysis tool associated to present pair only :return: dataframe: DataFrame = dataframe filled with user defined data """ diff --git a/freqtrade/freqai/prediction_models/BaseRegressionModel.py b/freqtrade/freqai/prediction_models/BaseRegressionModel.py index 611b5fb04..2baec9fc3 100644 --- a/freqtrade/freqai/prediction_models/BaseRegressionModel.py +++ b/freqtrade/freqai/prediction_models/BaseRegressionModel.py @@ -18,7 +18,7 @@ class BaseRegressionModel(IFreqaiModel): such as prediction_models/CatboostPredictionModel.py for guidance. """ - def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: + def return_values(self, dataframe: DataFrame) -> DataFrame: """ User uses this function to add any additional return values to the dataframe. e.g. diff --git a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py index bfbd5aa46..d94378494 100644 --- a/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py +++ b/freqtrade/freqai/prediction_models/BaseTensorFlowModel.py @@ -16,7 +16,7 @@ class BaseTensorFlowModel(IFreqaiModel): User *must* inherit from this class and set fit() and predict(). """ - def return_values(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> DataFrame: + def return_values(self, dataframe: DataFrame) -> DataFrame: """ User uses this function to add any additional return values to the dataframe. e.g. diff --git a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py index 56b84c08d..884933803 100644 --- a/freqtrade/freqai/prediction_models/CatboostPredictionModel.py +++ b/freqtrade/freqai/prediction_models/CatboostPredictionModel.py @@ -1,6 +1,6 @@ import logging from typing import Any, Dict - +import gc from catboost import CatBoostRegressor, Pool from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel @@ -42,8 +42,11 @@ class CatboostPredictionModel(BaseRegressionModel): **self.model_training_parameters, ) - if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: - test_data = None model.fit(X=train_data, eval_set=test_data) + # some evidence that catboost pools have memory leaks: + # https://github.com/catboost/catboost/issues/1835 + del train_data, test_data + gc.collect() + return model