From 0e0bda8f130488b7de38e563e483b92b4649a6f1 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 5 Oct 2022 14:08:03 +0200 Subject: [PATCH 1/3] improve freqai tests --- tests/freqai/conftest.py | 7 +- tests/freqai/test_freqai_datadrawer.py | 2 +- tests/freqai/test_freqai_datakitchen.py | 10 +-- tests/freqai/test_freqai_interface.py | 88 +++++++++++++++++++++---- 4 files changed, 87 insertions(+), 20 deletions(-) diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 2c6210a0e..026b45afc 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -29,15 +29,16 @@ def freqai_conf(default_conf, tmpdir): "enabled": True, "startup_candles": 10000, "purge_old_models": True, - "train_period_days": 5, + "train_period_days": 2, "backtest_period_days": 2, "live_retrain_hours": 0, "expiration_hours": 1, "identifier": "uniqe-id100", "live_trained_timestamp": 0, + "data_kitchen_thread_count": 2, "feature_parameters": { "include_timeframes": ["5m"], - "include_corr_pairlist": ["ADA/BTC", "DASH/BTC"], + "include_corr_pairlist": ["ADA/BTC"], "label_period_candles": 20, "include_shifted_candles": 1, "DI_threshold": 0.9, @@ -47,7 +48,7 @@ def freqai_conf(default_conf, tmpdir): "stratify_training_data": 0, "indicator_periods_candles": [10], }, - "data_split_parameters": {"test_size": 0.33, "random_state": 1}, + "data_split_parameters": {"test_size": 0.33, "shuffle": False}, "model_training_parameters": {"n_estimators": 100}, }, "config_files": [Path('config_examples', 'config_freqai.example.json')] diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index a6df60e61..1d1c44a1e 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -90,5 +90,5 @@ def test_use_strategy_to_populate_indicators(mocker, freqai_conf): df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC') - assert len(df.columns) == 45 + assert len(df.columns) == 33 shutil.rmtree(Path(freqai.dk.full_path)) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index b99ac236d..023193818 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -71,14 +71,14 @@ def test_use_DBSCAN_to_remove_outliers(mocker, freqai_conf, caplog): freqai = make_data_dictionary(mocker, freqai_conf) # freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 1}) freqai.dk.use_DBSCAN_to_remove_outliers(predict=False) - assert log_has_re(r"DBSCAN found eps of 2\.3\d\.", caplog) + assert log_has_re(r"DBSCAN found eps of 1.75", caplog) def test_compute_distances(mocker, freqai_conf): freqai = make_data_dictionary(mocker, freqai_conf) freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 1}) avg_mean_dist = freqai.dk.compute_distances() - assert round(avg_mean_dist, 2) == 2.54 + assert round(avg_mean_dist, 2) == 1.99 def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf, caplog): @@ -86,7 +86,7 @@ def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf, freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 0.1}) freqai.dk.use_SVM_to_remove_outliers(predict=False) assert log_has_re( - "SVM detected 8.66%", + "SVM detected 7.36%", caplog, ) @@ -125,7 +125,7 @@ def test_normalize_data(mocker, freqai_conf): freqai = make_data_dictionary(mocker, freqai_conf) data_dict = freqai.dk.data_dictionary freqai.dk.normalize_data(data_dict) - assert len(freqai.dk.data) == 56 + assert len(freqai.dk.data) == 32 def test_filter_features(mocker, freqai_conf): @@ -139,7 +139,7 @@ def test_filter_features(mocker, freqai_conf): training_filter=True, ) - assert len(filtered_df.columns) == 26 + assert len(filtered_df.columns) == 14 def test_make_train_test_datasets(mocker, freqai_conf): diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 4512a43f0..7921659bc 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -7,10 +7,14 @@ import pytest from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import RunMode from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.utils import download_all_data_for_training, get_required_data_timerange +from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange, log_has_re from tests.freqai.conftest import get_patched_freqai_strategy +from freqtrade.optimize.backtesting import Backtesting def is_arm() -> bool: @@ -18,15 +22,21 @@ def is_arm() -> bool: return "arm" in machine or "aarch64" in machine +def is_mac() -> bool: + machine = platform.system() + return "Darwin" in machine + + @pytest.mark.parametrize('model', [ 'LightGBMRegressor', 'XGBoostRegressor', 'CatboostRegressor', ]) -def test_extract_data_and_train_model_Regressors(mocker, freqai_conf, model): +def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model): if is_arm() and model == 'CatboostRegressor': pytest.skip("CatBoost is not supported on ARM") + model_save_ext = 'joblib' freqai_conf.update({"freqaimodel": model}) freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"strategy": "freqai_test_strat"}) @@ -43,16 +53,16 @@ def test_extract_data_and_train_model_Regressors(mocker, freqai_conf, model): freqai.dd.pair_dict = MagicMock() - data_load_timerange = TimeRange.parse_timerange("20180110-20180130") - new_timerange = TimeRange.parse_timerange("20180120-20180130") + data_load_timerange = TimeRange.parse_timerange("20180125-20180130") + new_timerange = TimeRange.parse_timerange("20180127-20180130") freqai.extract_data_and_train_model( new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) - assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file() + assert Path(freqai.dk.data_path / + f"{freqai.dk.model_filename}_model.{model_save_ext}").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() - assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() shutil.rmtree(Path(freqai.dk.full_path)) @@ -92,7 +102,7 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model): assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file() assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file() - assert len(freqai.dk.data['training_features_list']) == 26 + assert len(freqai.dk.data['training_features_list']) == 14 shutil.rmtree(Path(freqai.dk.full_path)) @@ -136,9 +146,28 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): shutil.rmtree(Path(freqai.dk.full_path)) -def test_start_backtesting(mocker, freqai_conf): - freqai_conf.update({"timerange": "20180120-20180130"}) +@pytest.mark.parametrize( + "model, num_files, strat", + [ + ("LightGBMRegressor", 6, "freqai_test_strat"), + ("XGBoostRegressor", 6, "freqai_test_strat"), + ("CatboostRegressor", 6, "freqai_test_strat"), + ("XGBoostClassifier", 6, "freqai_test_classifier"), + ("LightGBMClassifier", 6, "freqai_test_classifier"), + ("CatboostClassifier", 6, "freqai_test_classifier") + ], + ) +def test_start_backtesting(mocker, freqai_conf, model, num_files, strat): freqai_conf.get("freqai", {}).update({"save_backtest_models": True}) + freqai_conf['runmode'] = RunMode.BACKTEST + Trade.use_db = False + if is_arm() and "Catboost" in model: + pytest.skip("CatBoost is not supported on ARM") + + freqai_conf.update({"freqaimodel": model}) + freqai_conf.update({"timerange": "20180120-20180130"}) + freqai_conf.update({"strategy": strat}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) @@ -157,8 +186,8 @@ def test_start_backtesting(mocker, freqai_conf): freqai.start_backtesting(df, metadata, freqai.dk) model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] - assert len(model_folders) == 6 - + assert len(model_folders) == num_files + Backtesting.cleanup() shutil.rmtree(Path(freqai.dk.full_path)) @@ -211,7 +240,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): assert len(model_folders) == 6 - # without deleting the exiting folder structure, re-run + # without deleting the existing folder structure, re-run freqai_conf.update({"timerange": "20180120-20180130"}) strategy = get_patched_freqai_strategy(mocker, freqai_conf) @@ -375,3 +404,40 @@ def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs): pairs_b = strategy.gather_informative_pairs() # we expect unique pairs * timeframes assert len(pairs_b) == len(set(pairlist + corr_pairs)) * len(timeframes) + + +def test_start_set_train_queue(mocker, freqai_conf, caplog): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + pairlist = PairListManager(exchange, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange, pairlist) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = False + + freqai.train_queue = freqai._set_train_queue() + + assert log_has_re( + "Set fresh train queue from whitelist.", + caplog, + ) + + +def test_get_required_data_timerange(mocker, freqai_conf): + time_range = get_required_data_timerange(freqai_conf) + assert (time_range.stopts - time_range.startts) == 177300 + + +def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmpdir): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + pairlist = PairListManager(exchange, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange, pairlist) + freqai_conf['pairs'] = freqai_conf['exchange']['pair_whitelist'] + freqai_conf['datadir'] = Path(tmpdir) + download_all_data_for_training(strategy.dp, freqai_conf) + + assert log_has_re( + "Downloading", + caplog, + ) From 4edb30bfa82bbfeed89eefedb222afd18aec0819 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 5 Oct 2022 14:11:19 +0200 Subject: [PATCH 2/3] isort --- tests/freqai/test_freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 7921659bc..a61853c47 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -10,11 +10,11 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.utils import download_all_data_for_training, get_required_data_timerange +from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange, log_has_re from tests.freqai.conftest import get_patched_freqai_strategy -from freqtrade.optimize.backtesting import Backtesting def is_arm() -> bool: From 0d67afe15b75fc433ef962bb84c8fa8b1672ba2e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 5 Oct 2022 14:38:50 +0200 Subject: [PATCH 3/3] allow less precision, ensure regex is catching the right chars --- tests/freqai/test_freqai_datakitchen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 023193818..f60b29bf1 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -71,7 +71,7 @@ def test_use_DBSCAN_to_remove_outliers(mocker, freqai_conf, caplog): freqai = make_data_dictionary(mocker, freqai_conf) # freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 1}) freqai.dk.use_DBSCAN_to_remove_outliers(predict=False) - assert log_has_re(r"DBSCAN found eps of 1.75", caplog) + assert log_has_re(r"DBSCAN found eps of 1\.7\d\.", caplog) def test_compute_distances(mocker, freqai_conf):