From 2cc00a1a2c1af419787b81e37fd9c782b127988f Mon Sep 17 00:00:00 2001 From: paranoidandy Date: Sat, 24 Sep 2022 12:21:01 +0100 Subject: [PATCH] Allow use of --strategy-list with freqai, with warning (#7455) * Allow use of --strategy-list with freqai, with warning * ensure populate_any_indicators is identical for resused identifiers * use pair instead of metadata["pair"] Co-authored-by: robcaulk --- freqtrade/freqai/data_drawer.py | 10 ++++++++++ freqtrade/freqai/freqai_interface.py | 14 +++++++++----- freqtrade/optimize/backtesting.py | 4 ++-- tests/freqai/test_freqai_backtesting.py | 20 +++++++++++--------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index b4b92d984..7f4459fa5 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -430,6 +430,16 @@ class FreqaiDataDrawer: return + def load_metadata(self, dk: FreqaiDataKitchen) -> None: + """ + Load only metadata into datakitchen to increase performance during + presaved backtesting (prediction file loading). + """ + with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp: + dk.data = json.load(fp) + dk.training_features_list = dk.data["training_features_list"] + dk.label_list = dk.data["label_list"] + def load_data(self, coin: str, dk: FreqaiDataKitchen) -> Any: """ loads all data required to make a prediction on a sub-train time range diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index bdc418083..e0a45fb38 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -65,7 +65,7 @@ class IFreqaiModel(ABC): self.first = True self.set_full_path() self.follow_mode: bool = self.freqai_info.get("follow_mode", False) - self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", False) + self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True) if self.save_backtest_models: logger.info('Backtesting module configured to save all models.') self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) @@ -260,7 +260,7 @@ class IFreqaiModel(ABC): tr_train.stopts, tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT) logger.info( - f"Training {metadata['pair']}, {self.pair_it}/{self.total_pairs} pairs" + f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs" f" from {tr_train_startts_str} to {tr_train_stopts_str}, {train_it}/{total_trains} " "trains" ) @@ -273,11 +273,13 @@ class IFreqaiModel(ABC): dk.set_new_model_names(pair, trained_timestamp) if dk.check_if_backtest_prediction_exists(): + self.dd.load_metadata(dk) + self.check_if_feature_list_matches_strategy(dataframe_train, dk) append_df = dk.get_backtesting_prediction() dk.append_predictions(append_df) else: if not self.model_exists( - metadata["pair"], dk, trained_timestamp=trained_timestamp_int + pair, dk, trained_timestamp=trained_timestamp_int ): dk.find_features(dataframe_train) self.model = self.train(dataframe_train, pair, dk) @@ -429,14 +431,16 @@ class IFreqaiModel(ABC): if "training_features_list_raw" in dk.data: feature_list = dk.data["training_features_list_raw"] else: - feature_list = dk.training_features_list + feature_list = dk.data['training_features_list'] if dk.training_features_list != feature_list: raise OperationalException( "Trying to access pretrained model with `identifier` " "but found different features furnished by current strategy." "Change `identifier` to train from scratch, or ensure the" "strategy is furnishing the same features as the pretrained" - "model" + "model. In case of --strategy-list, please be aware that FreqAI " + "requires all strategies to maintain identical " + "populate_any_indicator() functions" ) def data_cleaning_train(self, dk: FreqaiDataKitchen) -> None: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0a05d740d..9ba610b69 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -91,8 +91,8 @@ class Backtesting: if self.config.get('strategy_list'): if self.config.get('freqai', {}).get('enabled', False): - raise OperationalException( - "You can't use strategy_list and freqai at the same time.") + logger.warning("Using --strategy-list with FreqAI REQUIRES all strategies " + "to have identical populate_any_indicators.") for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat diff --git a/tests/freqai/test_freqai_backtesting.py b/tests/freqai/test_freqai_backtesting.py index ea127fa99..b1881b2f5 100644 --- a/tests/freqai/test_freqai_backtesting.py +++ b/tests/freqai/test_freqai_backtesting.py @@ -3,21 +3,21 @@ from datetime import datetime, timezone from pathlib import Path from unittest.mock import PropertyMock -import pytest - -from freqtrade.commands.optimize_commands import start_backtesting -from freqtrade.exceptions import OperationalException +from freqtrade.commands.optimize_commands import setup_optimize_configuration +from freqtrade.enums import RunMode from freqtrade.optimize.backtesting import Backtesting from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange, patched_configuration_load_config_file) -def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir): +def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog): patch_exchange(mocker) + now = datetime.now(timezone.utc) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT'])) - # mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) + mocker.patch('freqtrade.optimize.backtesting.history.load_data') + mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now)) patched_configuration_load_config_file(mocker, freqai_conf) @@ -30,9 +30,11 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir): '--strategy-list', CURRENT_TEST_STRATEGY ] args = get_args(args) - with pytest.raises(OperationalException, - match=r"You can't use strategy_list and freqai at the same time\."): - start_backtesting(args) + bt_config = setup_optimize_configuration(args, RunMode.BACKTEST) + Backtesting(bt_config) + assert log_has_re('Using --strategy-list with FreqAI REQUIRES all strategies to have identical ' + 'populate_any_indicators.', caplog) + Backtesting.cleanup() def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):