From 5ee3b8cbbb89c8a57cb42cc3253001e47720991b Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Fri, 21 Oct 2022 19:48:26 -0600 Subject: [PATCH 1/8] update config recording to use all configs, fix tests --- freqtrade/freqai/freqai_interface.py | 21 ++++++++++++++++----- tests/freqai/test_freqai_interface.py | 21 ++++++++++++--------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index db0d4c379..fef5df40f 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -61,15 +61,16 @@ class IFreqaiModel(ABC): "data_split_parameters", {}) self.model_training_parameters: Dict[str, Any] = config.get("freqai", {}).get( "model_training_parameters", {}) + self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.retrain = False self.first = True self.set_full_path() + self.record_configs() self.follow_mode: bool = self.freqai_info.get("follow_mode", False) self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True) if self.save_backtest_models: logger.info('Backtesting module configured to save all models.') self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) - self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] self.keras: bool = self.freqai_info.get("keras", False) @@ -526,14 +527,24 @@ class IFreqaiModel(ABC): return file_exists def set_full_path(self) -> None: + """ + Creates and sets the full path for the identifier + """ self.full_path = Path( self.config["user_data_dir"] / "models" / f"{self.freqai_info['identifier']}" ) self.full_path.mkdir(parents=True, exist_ok=True) - shutil.copy( - self.config["config_files"][0], - Path(self.full_path, Path(self.config["config_files"][0]).name), - ) + + def record_configs(self) -> None: + """ + Records configs in the full path for reproducibility + """ + self.config_record_path = self.full_path / Path("configs") + self.config_record_path.mkdir(parents=True, exist_ok=True) + + for config in self.config["config_files"]: + dest_config_path = self.config_record_path / Path(config).name + shutil.copyfile(config, dest_config_path) def extract_data_and_train_model( self, diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index b619c0611..66eb6d696 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -19,7 +19,7 @@ from tests.freqai.conftest import get_patched_freqai_strategy def is_arm() -> bool: machine = platform.machine() - return "arm" in machine or "aarch64" in machine + return "arm" in machine or "aarch74" in machine def is_mac() -> bool: @@ -160,12 +160,12 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): @pytest.mark.parametrize( "model, num_files, strat", [ - ("LightGBMRegressor", 6, "freqai_test_strat"), - ("XGBoostRegressor", 6, "freqai_test_strat"), - ("CatboostRegressor", 6, "freqai_test_strat"), - ("XGBoostClassifier", 6, "freqai_test_classifier"), - ("LightGBMClassifier", 6, "freqai_test_classifier"), - ("CatboostClassifier", 6, "freqai_test_classifier") + ("LightGBMRegressor", 7, "freqai_test_strat"), + ("XGBoostRegressor", 7, "freqai_test_strat"), + ("CatboostRegressor", 7, "freqai_test_strat"), + ("XGBoostClassifier", 7, "freqai_test_classifier"), + ("LightGBMClassifier", 7, "freqai_test_classifier"), + ("CatboostClassifier", 7, "freqai_test_classifier") ], ) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): @@ -200,6 +200,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] + # Changed from 6 to 7 because of the /configs directory assert len(model_folders) == num_files assert log_has_re( "Removed features ", @@ -234,7 +235,9 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): metadata = {"pair": "LTC/BTC"} freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 9 + + # Changed from 9 to 10 because of the /configs dir + assert len(model_folders) == 10 shutil.rmtree(Path(freqai.dk.full_path)) @@ -260,7 +263,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 6 + assert len(model_folders) == 7 # without deleting the existing folder structure, re-run From 4464e91256d646ea6d216a9658d0f11c2741b556 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Fri, 21 Oct 2022 19:53:33 -0600 Subject: [PATCH 2/8] use self.identifier in full path --- freqtrade/freqai/freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index fef5df40f..4727e61af 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -531,7 +531,7 @@ class IFreqaiModel(ABC): Creates and sets the full path for the identifier """ self.full_path = Path( - self.config["user_data_dir"] / "models" / f"{self.freqai_info['identifier']}" + self.config["user_data_dir"] / "models" / f"{self.identifier}" ) self.full_path.mkdir(parents=True, exist_ok=True) From c4a2ee05e7cfc8eda18fd6a2404d14b212b409c5 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sat, 22 Oct 2022 09:31:55 -0600 Subject: [PATCH 3/8] fix freqai test --- tests/freqai/test_freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 66eb6d696..6bcfb43cc 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -19,7 +19,7 @@ from tests.freqai.conftest import get_patched_freqai_strategy def is_arm() -> bool: machine = platform.machine() - return "arm" in machine or "aarch74" in machine + return "arm" in machine or "aarch64" in machine def is_mac() -> bool: From 07e813dfa0f390081c1bb0f60611dabe9dc1b238 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 23 Oct 2022 12:09:07 -0600 Subject: [PATCH 4/8] change param record to only get certain params --- freqtrade/freqai/freqai_interface.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 4727e61af..bba4cecdb 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1,5 +1,5 @@ +import json import logging -import shutil import threading import time from abc import ABC, abstractmethod @@ -65,7 +65,6 @@ class IFreqaiModel(ABC): self.retrain = False self.first = True self.set_full_path() - self.record_configs() self.follow_mode: bool = self.freqai_info.get("follow_mode", False) self.save_backtest_models: bool = self.freqai_info.get("save_backtest_models", True) if self.save_backtest_models: @@ -98,6 +97,8 @@ class IFreqaiModel(ABC): self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() + self.record_params() + def __getstate__(self): """ Return an empty state to be pickled in hyperopt @@ -535,16 +536,23 @@ class IFreqaiModel(ABC): ) self.full_path.mkdir(parents=True, exist_ok=True) - def record_configs(self) -> None: + def record_params(self) -> None: """ - Records configs in the full path for reproducibility + Records run params in the full path for reproducibility """ - self.config_record_path = self.full_path / Path("configs") - self.config_record_path.mkdir(parents=True, exist_ok=True) + self.params_record_path = self.full_path / "run_params.json" - for config in self.config["config_files"]: - dest_config_path = self.config_record_path / Path(config).name - shutil.copyfile(config, dest_config_path) + run_params = { + "freqai": self.config.get('freqai', {}), + "timeframe": self.config.get('timeframe'), + "stake_amount": self.config.get('stake_amount'), + "stake_currency": self.config.get('stake_currency'), + "max_open_trades": self.config.get('max_open_trades'), + "pairs": self.config.get('exchange', {}).get('pair_whitelist') + } + + with open(self.params_record_path, "w") as handle: + json.dump(run_params, handle, indent=4) def extract_data_and_train_model( self, From bb0674522707df9aa9c6f52c6c2bf51bbec2dd9b Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 23 Oct 2022 12:25:39 -0600 Subject: [PATCH 5/8] fix tests --- tests/freqai/test_freqai_interface.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 6bcfb43cc..c46f9e815 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -160,12 +160,12 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): @pytest.mark.parametrize( "model, num_files, strat", [ - ("LightGBMRegressor", 7, "freqai_test_strat"), - ("XGBoostRegressor", 7, "freqai_test_strat"), - ("CatboostRegressor", 7, "freqai_test_strat"), - ("XGBoostClassifier", 7, "freqai_test_classifier"), - ("LightGBMClassifier", 7, "freqai_test_classifier"), - ("CatboostClassifier", 7, "freqai_test_classifier") + ("LightGBMRegressor", 6, "freqai_test_strat"), + ("XGBoostRegressor", 6, "freqai_test_strat"), + ("CatboostRegressor", 6, "freqai_test_strat"), + ("XGBoostClassifier", 6, "freqai_test_classifier"), + ("LightGBMClassifier", 6, "freqai_test_classifier"), + ("CatboostClassifier", 6, "freqai_test_classifier") ], ) def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog): @@ -200,7 +200,6 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - # Changed from 6 to 7 because of the /configs directory assert len(model_folders) == num_files assert log_has_re( "Removed features ", @@ -236,8 +235,7 @@ def test_start_backtesting_subdaily_backtest_period(mocker, freqai_conf): freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - # Changed from 9 to 10 because of the /configs dir - assert len(model_folders) == 10 + assert len(model_folders) == 9 shutil.rmtree(Path(freqai.dk.full_path)) @@ -263,7 +261,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 7 + assert len(model_folders) == 6 # without deleting the existing folder structure, re-run From 4d2b7a74f100b0aba2c5df9b3a638fd32d96780d Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 23 Oct 2022 20:51:32 +0200 Subject: [PATCH 6/8] move record params to utils, use rapidjson --- freqtrade/freqai/freqai_interface.py | 23 ++-------------------- freqtrade/freqai/utils.py | 29 +++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index bba4cecdb..6cb7f79f0 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1,4 +1,3 @@ -import json import logging import threading import time @@ -21,7 +20,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from freqtrade.freqai.utils import plot_feature_importance +from freqtrade.freqai.utils import plot_feature_importance, record_params from freqtrade.strategy.interface import IStrategy @@ -97,7 +96,7 @@ class IFreqaiModel(ABC): self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() - self.record_params() + record_params(config, self.full_path) def __getstate__(self): """ @@ -536,24 +535,6 @@ class IFreqaiModel(ABC): ) self.full_path.mkdir(parents=True, exist_ok=True) - def record_params(self) -> None: - """ - Records run params in the full path for reproducibility - """ - self.params_record_path = self.full_path / "run_params.json" - - run_params = { - "freqai": self.config.get('freqai', {}), - "timeframe": self.config.get('timeframe'), - "stake_amount": self.config.get('stake_amount'), - "stake_currency": self.config.get('stake_currency'), - "max_open_trades": self.config.get('max_open_trades'), - "pairs": self.config.get('exchange', {}).get('pair_whitelist') - } - - with open(self.params_record_path, "w") as handle: - json.dump(run_params, handle, indent=4) - def extract_data_and_train_model( self, new_trained_timerange: TimeRange, diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 22bc1e06e..b3f25d4d1 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,9 +1,11 @@ import logging from datetime import datetime, timezone -from typing import Any +from pathlib import Path +from typing import Any, Dict import numpy as np import pandas as pd +import rapidjson from freqtrade.configuration import TimeRange from freqtrade.constants import Config @@ -191,3 +193,28 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, fig.update_layout(title_text=f"Best and worst features by importance {pair}") label = label.replace('&', '').replace('%', '') # escape two FreqAI specific characters store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path) + + +def record_params(config: Dict[str, Any], full_path: Path) -> None: + """ + Records run params in the full path for reproducibility + """ + params_record_path = full_path / "run_params.json" + + run_params = { + "freqai": config.get('freqai', {}), + "timeframe": config.get('timeframe'), + "stake_amount": config.get('stake_amount'), + "stake_currency": config.get('stake_currency'), + "max_open_trades": config.get('max_open_trades'), + "pairs": config.get('exchange', {}).get('pair_whitelist') + } + + with open(params_record_path, "w") as handle: + rapidjson.dump(run_params, handle, indent=4, default=np_encoder, + number_mode=rapidjson.NM_NATIVE) + + +def np_encoder(self, object): + if isinstance(object, np.generic): + return object.item() From 51be45547f81dfbcddb6151f65e5ded1163a0b62 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Mon, 24 Oct 2022 12:23:54 -0600 Subject: [PATCH 7/8] remove np object, make default str --- freqtrade/freqai/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index b3f25d4d1..89fd86ea8 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -211,10 +211,10 @@ def record_params(config: Dict[str, Any], full_path: Path) -> None: } with open(params_record_path, "w") as handle: - rapidjson.dump(run_params, handle, indent=4, default=np_encoder, - number_mode=rapidjson.NM_NATIVE) - - -def np_encoder(self, object): - if isinstance(object, np.generic): - return object.item() + rapidjson.dump( + run_params, + handle, + indent=4, + default=str, + number_mode=rapidjson.NM_NATIVE + ) From b9bf9edb02e04ef712922da0bc98d971511314ca Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 25 Oct 2022 14:12:13 -0600 Subject: [PATCH 8/8] update rapidjson opts --- freqtrade/freqai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 89fd86ea8..5a2b9b7d6 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -216,5 +216,5 @@ def record_params(config: Dict[str, Any], full_path: Path) -> None: handle, indent=4, default=str, - number_mode=rapidjson.NM_NATIVE + number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN )