From 2e1624c7e886cafdfb750cce4de7ba49b591381a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jul 2022 17:46:43 +0200 Subject: [PATCH] Simplify freqAI tests, ensure they use a tempdir for modelstorage --- tests/freqai/conftest.py | 7 +- tests/freqai/test_freqai_datakitchen.py | 79 ++++++++---------- tests/freqai/test_freqai_interface.py | 103 ++++++++++-------------- 3 files changed, 81 insertions(+), 108 deletions(-) diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 1d0ea0e54..ede991240 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -2,6 +2,8 @@ from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock +import pytest + from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.freqai.data_kitchen import FreqaiDataKitchen @@ -10,13 +12,14 @@ from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver from tests.conftest import get_patched_exchange -# @pytest.fixture(scope="function") -def freqai_conf(default_conf): +@pytest.fixture(scope="function") +def freqai_conf(default_conf, tmpdir): freqaiconf = deepcopy(default_conf) freqaiconf.update( { "datadir": Path(default_conf["datadir"]), "strategy": "freqai_test_strat", + "user_data_dir": Path(tmpdir), "strategy-path": "freqtrade/tests/strategy/strats", "freqaimodel": "LightGBMPredictionModel", "freqaimodel_path": "freqai/prediction_models", diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index d0d82b489..ee1d83982 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -1,6 +1,3 @@ -# from unittest.mock import MagicMock -# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge -import copy import datetime import shutil from pathlib import Path @@ -13,7 +10,7 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from tests.conftest import get_patched_exchange -from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_freqai_strategy +from tests.freqai.conftest import get_patched_data_kitchen, get_patched_freqai_strategy @pytest.mark.parametrize( @@ -24,15 +21,15 @@ from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_pat ], ) def test_create_fulltimerange( - timerange, train_period_days, expected_result, default_conf, mocker, caplog + timerange, train_period_days, expected_result, freqai_conf, mocker, caplog ): - dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) + dk = get_patched_data_kitchen(mocker, freqai_conf) assert dk.create_fulltimerange(timerange, train_period_days) == expected_result shutil.rmtree(Path(dk.full_path)) -def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf): - dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) +def test_create_fulltimerange_incorrect_backtest_period(mocker, freqai_conf): + dk = get_patched_data_kitchen(mocker, freqai_conf) with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"): dk.create_fulltimerange("20220101-20220201", 0.5) with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"): @@ -49,11 +46,10 @@ def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf): ], ) def test_split_timerange( - mocker, default_conf, timerange, train_period_days, backtest_period_days, expected_result + mocker, freqai_conf, timerange, train_period_days, backtest_period_days, expected_result ): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - freqaiconf.update({"timerange": "20220101-20220401"}) - dk = get_patched_data_kitchen(mocker, freqaiconf) + freqai_conf.update({"timerange": "20220101-20220401"}) + dk = get_patched_data_kitchen(mocker, freqai_conf) tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days) assert len(tr_list) == len(bt_list) == expected_result @@ -64,14 +60,13 @@ def test_split_timerange( shutil.rmtree(Path(dk.full_path)) -def test_update_historic_data(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) +def test_update_historic_data(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) freqai = strategy.freqai freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dk.load_all_pair_histories(timerange) @@ -93,69 +88,65 @@ def test_update_historic_data(mocker, default_conf): (datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False), ], ) -def test_check_if_model_expired(mocker, default_conf, timestamp, expected): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - dk = get_patched_data_kitchen(mocker, freqaiconf) +def test_check_if_model_expired(mocker, freqai_conf, timestamp, expected): + dk = get_patched_data_kitchen(mocker, freqai_conf) assert dk.check_if_model_expired(timestamp) == expected shutil.rmtree(Path(dk.full_path)) -def test_load_all_pairs_histories(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) +def test_load_all_pairs_histories(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) freqai = strategy.freqai freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dk.load_all_pair_histories(timerange) assert len(freqai.dd.historic_data.keys()) == len( - freqaiconf.get("exchange", {}).get("pair_whitelist") + freqai_conf.get("exchange", {}).get("pair_whitelist") ) assert len(freqai.dd.historic_data["ADA/BTC"]) == len( - freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") + freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") ) shutil.rmtree(Path(freqai.dk.full_path)) -def test_get_base_and_corr_dataframes(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) +def test_get_base_and_corr_dataframes(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) freqai = strategy.freqai freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dk.load_all_pair_histories(timerange) sub_timerange = TimeRange.parse_timerange("20180111-20180114") corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") num_tfs = len( - freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") + freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") ) assert len(base_df.keys()) == num_tfs assert len(corr_df.keys()) == len( - freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist") + freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist") ) assert len(corr_df["ADA/BTC"].keys()) == num_tfs shutil.rmtree(Path(freqai.dk.full_path)) -def test_use_strategy_to_populate_indicators(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) - strategy.freqai_info = freqaiconf.get("freqai", {}) +def test_use_strategy_to_populate_indicators(mocker, freqai_conf): + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dk.load_all_pair_histories(timerange) sub_timerange = TimeRange.parse_timerange("20180111-20180114") diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index d6e67caeb..0bb2dac79 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -1,6 +1,5 @@ # from unittest.mock import MagicMock # from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge -import copy import platform import shutil from pathlib import Path @@ -12,20 +11,19 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from tests.conftest import get_patched_exchange, log_has_re -from tests.freqai.conftest import freqai_conf, get_patched_freqai_strategy +from tests.freqai.conftest import get_patched_freqai_strategy -def test_train_model_in_series_LightGBM(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - freqaiconf.update({"timerange": "20180110-20180130"}) +def test_train_model_in_series_LightGBM(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) - strategy.freqai_info = freqaiconf.get("freqai", {}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dk.load_all_pair_histories(timerange) @@ -61,19 +59,18 @@ def test_train_model_in_series_LightGBM(mocker, default_conf): @pytest.mark.skipif("arm" in platform.uname()[-1], reason="no ARM for Catboost ...") -def test_train_model_in_series_Catboost(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - freqaiconf.update({"timerange": "20180110-20180130"}) - freqaiconf.update({"freqaimodel": "CatboostPredictionModel"}) - del freqaiconf['freqai']['model_training_parameters']['verbosity'] - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) +def test_train_model_in_series_Catboost(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"freqaimodel": "CatboostPredictionModel"}) + del freqai_conf['freqai']['model_training_parameters']['verbosity'] + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) - strategy.freqai_info = freqaiconf.get("freqai", {}) + strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dk.load_all_pair_histories(timerange) @@ -85,40 +82,23 @@ def test_train_model_in_series_Catboost(mocker, default_conf): freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib")) - .resolve() - .exists() - ) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json")) - .resolve() - .exists() - ) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl")) - .resolve() - .exists() - ) - assert ( - Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib")) - .resolve() - .exists() - ) + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists() shutil.rmtree(Path(freqai.dk.full_path)) -def test_start_backtesting(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - freqaiconf.update({"timerange": "20180120-20180130"}) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) - strategy.freqai_info = freqaiconf.get("freqai", {}) +def test_start_backtesting(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dk.load_all_pair_histories(timerange) sub_timerange = TimeRange.parse_timerange("20180110-20180130") @@ -135,16 +115,15 @@ def test_start_backtesting(mocker, default_conf): shutil.rmtree(Path(freqai.dk.full_path)) -def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - freqaiconf.update({"timerange": "20180120-20180130"}) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) - strategy.freqai_info = freqaiconf.get("freqai", {}) +def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog): + freqai_conf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dk.load_all_pair_histories(timerange) sub_timerange = TimeRange.parse_timerange("20180110-20180130") @@ -160,14 +139,14 @@ def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog): # without deleting the exiting folder structure, re-run - freqaiconf.update({"timerange": "20180120-20180130"}) - strategy = get_patched_freqai_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) - strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai_conf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) freqai = strategy.freqai freqai.live = False - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dk.load_all_pair_histories(timerange) sub_timerange = TimeRange.parse_timerange("20180110-20180130")