From d43c146676a8c5842b247d247c7537ad86905405 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 20 Jul 2022 12:56:46 +0200 Subject: [PATCH] add more tests for datakitchen functionalities, add regression tests for freqai_interface train/backtest --- freqtrade/freqai/data_kitchen.py | 15 -- freqtrade/freqai/freqai_interface.py | 3 +- setup.sh | 10 +- tests/freqai/conftest.py | 61 +++++++- tests/freqai/test_freqai.py | 95 ------------ tests/freqai/test_freqai_datakitchen.py | 167 +++++++++++++++++++++ tests/freqai/test_freqai_interface.py | 183 ++++++++++++++++++++++++ 7 files changed, 415 insertions(+), 119 deletions(-) delete mode 100644 tests/freqai/test_freqai.py create mode 100644 tests/freqai/test_freqai_datakitchen.py create mode 100644 tests/freqai/test_freqai_interface.py diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 5c05d94e0..0966f8421 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -690,8 +690,6 @@ class FreqaiDataKitchen: Append backtest prediction from current backtest period to all previous periods """ - # ones = np.ones(len(predictions)) - # target_mean, target_std = ones * self.data["target_mean"], ones * self.data["target_std"] self.append_df = DataFrame() for label in self.label_list: self.append_df[label] = predictions[label] @@ -707,13 +705,6 @@ class FreqaiDataKitchen: else: self.full_df = pd.concat([self.full_df, self.append_df], axis=0) - # self.full_predictions = np.append(self.full_predictions, predictions) - # self.full_do_predict = np.append(self.full_do_predict, do_predict) - # if self.freqai_config.get("feature_parameters", {}).get("DI_threshold", 0) > 0: - # self.full_DI_values = np.append(self.full_DI_values, self.DI_values) - # self.full_target_mean = np.append(self.full_target_mean, target_mean) - # self.full_target_std = np.append(self.full_target_std, target_std) - return def fill_predictions(self, dataframe): @@ -734,12 +725,6 @@ class FreqaiDataKitchen: self.append_df = DataFrame() self.full_df = DataFrame() - # self.full_predictions = np.append(filler, self.full_predictions) - # self.full_do_predict = np.append(filler, self.full_do_predict) - # if self.freqai_config.get("feature_parameters", {}).get("DI_threshold", 0) > 0: - # self.full_DI_values = np.append(filler, self.full_DI_values) - # self.full_target_mean = np.append(filler, self.full_target_mean) - # self.full_target_std = np.append(filler, self.full_target_std) return diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 56a179dc3..5acfcb9cf 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -170,11 +170,10 @@ class IFreqaiModel(ABC): gc.collect() dk.data = {} # clean the pair specific data between training window sliding self.training_timerange = tr_train - # self.training_timerange_timerange = tr_train dataframe_train = dk.slice_dataframe(tr_train, dataframe) dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe) - trained_timestamp = tr_train # TimeRange.parse_timerange(tr_train) + trained_timestamp = tr_train tr_train_startts_str = datetime.datetime.utcfromtimestamp(tr_train.startts).strftime( "%Y-%m-%d %H:%M:%S" ) diff --git a/setup.sh b/setup.sh index 202cb70c7..1a4a285a3 100755 --- a/setup.sh +++ b/setup.sh @@ -77,7 +77,15 @@ function updateenv() { fi fi - ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} + REQUIREMENTS_FREQAI="" + read -p "Do you want to install dependencies for freqai [y/N]? " + dev=$REPLY + if [[ $REPLY =~ ^[Yy]$ ]] + then + REQUIREMENTS_FREQAI="-r requirements-freqai.txt" + fi + + ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI} if [ $? -ne 0 ]; then echo "Failed installing dependencies" exit 1 diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index cfc23939e..d9f4cc635 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -2,9 +2,12 @@ from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.resolvers import StrategyResolver from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver +from tests.conftest import get_patched_exchange # @pytest.fixture(scope="function") @@ -21,16 +24,17 @@ def freqai_conf(default_conf): "freqai": { "startup_candles": 10000, "purge_old_models": True, - "train_period_days": 15, - "backtest_period_days": 7, + "train_period_days": 5, + "backtest_period_days": 2, "live_retrain_hours": 0, - "identifier": "uniqe-id7", + "expiration_hours": 1, + "identifier": "uniqe-id100", "live_trained_timestamp": 0, "feature_parameters": { "include_timeframes": ["5m"], "include_corr_pairlist": ["ADA/BTC", "DASH/BTC"], "label_period_candles": 20, - "include_shifted_candles": 2, + "include_shifted_candles": 1, "DI_threshold": 0.9, "weight_factor": 0.9, "principal_component_analysis": False, @@ -40,7 +44,7 @@ def freqai_conf(default_conf): "indicator_periods_candles": [10], }, "data_split_parameters": {"test_size": 0.33, "random_state": 1}, - "model_training_parameters": {"n_estimators": 1000, "task_type": "CPU"}, + "model_training_parameters": {"n_estimators": 100}, }, "config_files": [Path('config_examples', 'config_freqai_futures.example.json')] } @@ -55,7 +59,7 @@ def get_patched_data_kitchen(mocker, freqaiconf): return dk -def get_patched_strategy(mocker, freqaiconf): +def get_patched_freqai_strategy(mocker, freqaiconf): strategy = StrategyResolver.load_strategy(freqaiconf) strategy.bot_start() @@ -66,3 +70,48 @@ def get_patched_freqaimodel(mocker, freqaiconf): freqaimodel = FreqaiModelResolver.load_freqaimodel(freqaiconf) return freqaimodel + + +def get_freqai_live_analyzed_dataframe(mocker, freqaiconf): + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + + strategy.analyze_pair('ADA/BTC', '5m') + return strategy.dp.get_analyzed_dataframe('ADA/BTC', '5m') + + +def get_freqai_analyzed_dataframe(mocker, freqaiconf): + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + return freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC') + + +def get_ready_to_train(mocker, freqaiconf): + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + return corr_df, base_df, freqai, strategy diff --git a/tests/freqai/test_freqai.py b/tests/freqai/test_freqai.py deleted file mode 100644 index 185e55744..000000000 --- a/tests/freqai/test_freqai.py +++ /dev/null @@ -1,95 +0,0 @@ -# from unittest.mock import MagicMock -# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge -import copy - -import pytest - -from freqtrade.configuration import TimeRange -from freqtrade.data.dataprovider import DataProvider -# from freqtrade.freqai.data_drawer import FreqaiDataDrawer -from freqtrade.exceptions import OperationalException -from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from tests.conftest import get_patched_exchange -from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_strategy - - -@pytest.mark.parametrize( - "timerange, train_period_days, expected_result", - [ - ("20220101-20220201", 30, "20211202-20220201"), - ("20220301-20220401", 15, "20220214-20220401"), - ], -) -def test_create_fulltimerange( - timerange, train_period_days, expected_result, default_conf, mocker, caplog -): - dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) - assert dk.create_fulltimerange(timerange, train_period_days) == expected_result - - -def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf): - dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) - with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"): - dk.create_fulltimerange("20220101-20220201", 0.5) - with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"): - dk.create_fulltimerange("20220101-20220201", -1) - - -def test_split_timerange(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - freqaiconf.update({"timerange": "20220101-20220401"}) - dk = get_patched_data_kitchen(mocker, freqaiconf) - tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 7) - assert len(tr_list) == len(bt_list) == 9 - - tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 0.5) - assert len(tr_list) == len(bt_list) == 120 - - tr_list, bt_list = dk.split_timerange("20220101-20220201", 10, 1) - assert len(tr_list) == len(bt_list) == 80 - - with pytest.raises( - OperationalException, match=r"train_period_days must be an integer greater than 0." - ): - dk.split_timerange("20220101-20220201", -1, 0.5) - - -def test_update_historic_data(mocker, default_conf): - freqaiconf = freqai_conf(copy.deepcopy(default_conf)) - strategy = get_patched_strategy(mocker, freqaiconf) - exchange = get_patched_exchange(mocker, freqaiconf) - strategy.dp = DataProvider(freqaiconf, exchange) - freqai = strategy.model.bridge - freqai.live = True - freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) - timerange = TimeRange.parse_timerange("20180110-20180114") - - freqai.dk.load_all_pair_histories(timerange) - historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) - dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) - candle_difference = dp_candles - historic_candles - freqai.dk.update_historic_data(strategy) - - updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) - - assert updated_historic_candles - historic_candles == candle_difference - - -# def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): -# np.random.seed(42) -# tf_mins = timeframe_to_minutes(timeframe) - -# base = np.random.normal(20, 2, size=size) - -# date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') -# df = pd.DataFrame({ -# 'date': date, -# 'open': base, -# 'high': base + np.random.normal(2, 1, size=size), -# 'low': base - np.random.normal(2, 1, size=size), -# 'close': base + np.random.normal(0, 1, size=size), -# 'volume': np.random.normal(200, size=size) -# } -# ) -# df = df.dropna() -# return df diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py new file mode 100644 index 000000000..49374b257 --- /dev/null +++ b/tests/freqai/test_freqai_datakitchen.py @@ -0,0 +1,167 @@ +# from unittest.mock import MagicMock +# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge +import copy +import datetime +import shutil +from pathlib import Path + +import pytest + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +# from freqtrade.freqai.data_drawer import FreqaiDataDrawer +from freqtrade.exceptions import OperationalException +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from tests.conftest import get_patched_exchange +from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_freqai_strategy + + +@pytest.mark.parametrize( + "timerange, train_period_days, expected_result", + [ + ("20220101-20220201", 30, "20211202-20220201"), + ("20220301-20220401", 15, "20220214-20220401"), + ], +) +def test_create_fulltimerange( + timerange, train_period_days, expected_result, default_conf, mocker, caplog +): + dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) + assert dk.create_fulltimerange(timerange, train_period_days) == expected_result + shutil.rmtree(Path(dk.full_path)) + + +def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf): + dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) + with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"): + dk.create_fulltimerange("20220101-20220201", 0.5) + with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"): + dk.create_fulltimerange("20220101-20220201", -1) + shutil.rmtree(Path(dk.full_path)) + + +@pytest.mark.parametrize( + "timerange, train_period_days, backtest_period_days, expected_result", + [ + ("20220101-20220201", 30, 7, 9), + ("20220101-20220201", 30, 0.5, 120), + ("20220101-20220201", 10, 1, 80), + ], +) +def test_split_timerange( + mocker, default_conf, timerange, train_period_days, backtest_period_days, expected_result +): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + freqaiconf.update({"timerange": "20220101-20220401"}) + dk = get_patched_data_kitchen(mocker, freqaiconf) + tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days) + assert len(tr_list) == len(bt_list) == expected_result + + with pytest.raises( + OperationalException, match=r"train_period_days must be an integer greater than 0." + ): + dk.split_timerange("20220101-20220201", -1, 0.5) + shutil.rmtree(Path(dk.full_path)) + + +def test_update_historic_data(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + + freqai.dk.load_all_pair_histories(timerange) + historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) + dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) + candle_difference = dp_candles - historic_candles + freqai.dk.update_historic_data(strategy) + + updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) + + assert updated_historic_candles - historic_candles == candle_difference + shutil.rmtree(Path(freqai.dk.full_path)) + + +@pytest.mark.parametrize( + "timestamp, expected", + [ + (datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - 7200, True), + (datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False), + ], +) +def test_check_if_model_expired(mocker, default_conf, timestamp, expected): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + dk = get_patched_data_kitchen(mocker, freqaiconf) + assert dk.check_if_model_expired(timestamp) == expected + shutil.rmtree(Path(dk.full_path)) + + +def test_load_all_pairs_histories(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + + assert len(freqai.dd.historic_data.keys()) == len( + freqaiconf.get("exchange", {}).get("pair_whitelist") + ) + assert len(freqai.dd.historic_data["ADA/BTC"]) == len( + freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") + ) + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_get_base_and_corr_dataframes(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + num_tfs = len( + freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") + ) + + assert len(base_df.keys()) == num_tfs + + assert len(corr_df.keys()) == len( + freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist") + ) + + assert len(corr_df["ADA/BTC"].keys()) == num_tfs + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_use_strategy_to_populate_indicators(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180111-20180114") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC') + + assert len(df.columns) == 90 + shutil.rmtree(Path(freqai.dk.full_path)) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py new file mode 100644 index 000000000..46ce19f8c --- /dev/null +++ b/tests/freqai/test_freqai_interface.py @@ -0,0 +1,183 @@ +# from unittest.mock import MagicMock +# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge +import copy +import platform +import shutil +from pathlib import Path +from unittest.mock import MagicMock + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +# from freqtrade.freqai.data_drawer import FreqaiDataDrawer +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from tests.conftest import get_patched_exchange, log_has +from tests.freqai.conftest import freqai_conf, get_patched_freqai_strategy + + +def test_train_model_in_series_LightGBM(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + freqaiconf.update({"timerange": "20180110-20180130"}) + + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib")) + .resolve() + .exists() + ) + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json")) + .resolve() + .exists() + ) + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl")) + .resolve() + .exists() + ) + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib")) + .resolve() + .exists() + ) + + shutil.rmtree(Path(freqai.dk.full_path)) + + +# Catboost not available for ARM architecture. using platform lib to check processor type +if "arm" not in platform.uname()[-1]: + + def test_train_model_in_series_Catboost(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + freqaiconf.update({"timerange": "20180110-20180130"}) + freqaiconf.update({"freqaimodel": "CatboostPredictionModel"}) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.train_model_in_series( + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange + ) + + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib")) + .resolve() + .exists() + ) + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json")) + .resolve() + .exists() + ) + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl")) + .resolve() + .exists() + ) + assert ( + Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib")) + .resolve() + .exists() + ) + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_start_backtesting(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + freqaiconf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + + metadata = {"pair": "ADA/BTC"} + freqai.start_backtesting(df, metadata, freqai.dk) + model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] + + assert len(model_folders) == 5 + + shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + freqaiconf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + + metadata = {"pair": "ADA/BTC"} + freqai.start_backtesting(df, metadata, freqai.dk) + model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()] + + assert len(model_folders) == 5 + + # without deleting the exiting folder structure, re-run + + freqaiconf.update({"timerange": "20180120-20180130"}) + strategy = get_patched_freqai_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + strategy.freqai_info = freqaiconf.get("freqai", {}) + freqai = strategy.model.bridge + freqai.live = False + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dk.load_all_pair_histories(timerange) + sub_timerange = TimeRange.parse_timerange("20180110-20180130") + corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") + + df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + freqai.start_backtesting(df, metadata, freqai.dk) + assert log_has( + "Found model at user_data/models/uniqe-id100/sub-train-ADA1517097600/cb_ada_1517097600", + caplog, + ) + + shutil.rmtree(Path(freqai.dk.full_path))