adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve freqai tests

Browse Source
This commit is contained in:
robcaulk 2022-10-05 14:08:03 +02:00
parent 22043deffa
commit 0e0bda8f13
4 changed files with 87 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/freqai/conftest.py
View File

@ -29,15 +29,16 @@ def freqai_conf(default_conf, tmpdir):
"enabled": True,
"startup_candles": 10000,
"purge_old_models": True,
"train_period_days": 5,
"train_period_days": 2,
"backtest_period_days": 2,
"live_retrain_hours": 0,
"expiration_hours": 1,
"identifier": "uniqe-id100",
"live_trained_timestamp": 0,
"data_kitchen_thread_count": 2,
"feature_parameters": {
"include_timeframes": ["5m"],
"include_corr_pairlist": ["ADA/BTC", "DASH/BTC"],
"include_corr_pairlist": ["ADA/BTC"],
"label_period_candles": 20,
"include_shifted_candles": 1,
"DI_threshold": 0.9,
@ -47,7 +48,7 @@ def freqai_conf(default_conf, tmpdir):
"stratify_training_data": 0,
"indicator_periods_candles": [10],
},
"data_split_parameters": {"test_size": 0.33, "random_state": 1},
"data_split_parameters": {"test_size": 0.33, "shuffle": False},
"model_training_parameters": {"n_estimators": 100},
},
"config_files": [Path('config_examples', 'config_freqai.example.json')]

2
tests/freqai/test_freqai_datadrawer.py
View File

@ -90,5 +90,5 @@ def test_use_strategy_to_populate_indicators(mocker, freqai_conf):
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC')
assert len(df.columns) == 45
assert len(df.columns) == 33
shutil.rmtree(Path(freqai.dk.full_path))

10
tests/freqai/test_freqai_datakitchen.py
View File

@ -71,14 +71,14 @@ def test_use_DBSCAN_to_remove_outliers(mocker, freqai_conf, caplog):
freqai = make_data_dictionary(mocker, freqai_conf)
# freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 1})
freqai.dk.use_DBSCAN_to_remove_outliers(predict=False)
assert log_has_re(r"DBSCAN found eps of 2\.3\d\.", caplog)
assert log_has_re(r"DBSCAN found eps of 1.75", caplog)
def test_compute_distances(mocker, freqai_conf):
freqai = make_data_dictionary(mocker, freqai_conf)
freqai_conf['freqai']['feature_parameters'].update({"DI_threshold": 1})
avg_mean_dist = freqai.dk.compute_distances()
assert round(avg_mean_dist, 2) == 2.54
assert round(avg_mean_dist, 2) == 1.99
def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf, caplog):
@ -86,7 +86,7 @@ def test_use_SVM_to_remove_outliers_and_outlier_protection(mocker, freqai_conf,
freqai_conf['freqai']['feature_parameters'].update({"outlier_protection_percentage": 0.1})
freqai.dk.use_SVM_to_remove_outliers(predict=False)
assert log_has_re(
"SVM detected 8.66%",
"SVM detected 7.36%",
caplog,
)
@ -125,7 +125,7 @@ def test_normalize_data(mocker, freqai_conf):
freqai = make_data_dictionary(mocker, freqai_conf)
data_dict = freqai.dk.data_dictionary
freqai.dk.normalize_data(data_dict)
assert len(freqai.dk.data) == 56
assert len(freqai.dk.data) == 32
def test_filter_features(mocker, freqai_conf):
@ -139,7 +139,7 @@ def test_filter_features(mocker, freqai_conf):
training_filter=True,
)
assert len(filtered_df.columns) == 26
assert len(filtered_df.columns) == 14
def test_make_train_test_datasets(mocker, freqai_conf):

88
tests/freqai/test_freqai_interface.py
View File

@ -7,10 +7,14 @@ import pytest
from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import RunMode
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.freqai.utils import download_all_data_for_training, get_required_data_timerange
from freqtrade.persistence import Trade
from freqtrade.plugins.pairlistmanager import PairListManager
from tests.conftest import get_patched_exchange, log_has_re
from tests.freqai.conftest import get_patched_freqai_strategy
from freqtrade.optimize.backtesting import Backtesting
def is_arm() -> bool:
@ -18,15 +22,21 @@ def is_arm() -> bool:
return "arm" in machine or "aarch64" in machine
def is_mac() -> bool:
machine = platform.system()
return "Darwin" in machine
@pytest.mark.parametrize('model', [
'LightGBMRegressor',
'XGBoostRegressor',
'CatboostRegressor',
])
def test_extract_data_and_train_model_Regressors(mocker, freqai_conf, model):
def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model):
if is_arm() and model == 'CatboostRegressor':
pytest.skip("CatBoost is not supported on ARM")
model_save_ext = 'joblib'
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"strategy": "freqai_test_strat"})
@ -43,16 +53,16 @@ def test_extract_data_and_train_model_Regressors(mocker, freqai_conf, model):
freqai.dd.pair_dict = MagicMock()
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
data_load_timerange = TimeRange.parse_timerange("20180125-20180130")
new_timerange = TimeRange.parse_timerange("20180127-20180130")
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").is_file()
assert Path(freqai.dk.data_path /
f"{freqai.dk.model_filename}_model.{model_save_ext}").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file()
shutil.rmtree(Path(freqai.dk.full_path))
@ -92,7 +102,7 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model):
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").is_file()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").is_file()
assert len(freqai.dk.data['training_features_list']) == 26
assert len(freqai.dk.data['training_features_list']) == 14
shutil.rmtree(Path(freqai.dk.full_path))
@ -136,9 +146,28 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
shutil.rmtree(Path(freqai.dk.full_path))
def test_start_backtesting(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180120-20180130"})
@pytest.mark.parametrize(
"model, num_files, strat",
[
("LightGBMRegressor", 6, "freqai_test_strat"),
("XGBoostRegressor", 6, "freqai_test_strat"),
("CatboostRegressor", 6, "freqai_test_strat"),
("XGBoostClassifier", 6, "freqai_test_classifier"),
("LightGBMClassifier", 6, "freqai_test_classifier"),
("CatboostClassifier", 6, "freqai_test_classifier")
],
)
def test_start_backtesting(mocker, freqai_conf, model, num_files, strat):
freqai_conf.get("freqai", {}).update({"save_backtest_models": True})
freqai_conf['runmode'] = RunMode.BACKTEST
Trade.use_db = False
if is_arm() and "Catboost" in model:
pytest.skip("CatBoost is not supported on ARM")
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180120-20180130"})
freqai_conf.update({"strategy": strat})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
@ -157,8 +186,8 @@ def test_start_backtesting(mocker, freqai_conf):
freqai.start_backtesting(df, metadata, freqai.dk)
model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
assert len(model_folders) == 6
assert len(model_folders) == num_files
Backtesting.cleanup()
shutil.rmtree(Path(freqai.dk.full_path))
@ -211,7 +240,7 @@ def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
assert len(model_folders) == 6
# without deleting the exiting folder structure, re-run
# without deleting the existing folder structure, re-run
freqai_conf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
@ -375,3 +404,40 @@ def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs):
pairs_b = strategy.gather_informative_pairs()
# we expect unique pairs * timeframes
assert len(pairs_b) == len(set(pairlist + corr_pairs)) * len(timeframes)
def test_start_set_train_queue(mocker, freqai_conf, caplog):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
pairlist = PairListManager(exchange, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange, pairlist)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = False
freqai.train_queue = freqai._set_train_queue()
assert log_has_re(
"Set fresh train queue from whitelist.",
caplog,
)
def test_get_required_data_timerange(mocker, freqai_conf):
time_range = get_required_data_timerange(freqai_conf)
assert (time_range.stopts - time_range.startts) == 177300
def test_download_all_data_for_training(mocker, freqai_conf, caplog, tmpdir):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
pairlist = PairListManager(exchange, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange, pairlist)
freqai_conf['pairs'] = freqai_conf['exchange']['pair_whitelist']
freqai_conf['datadir'] = Path(tmpdir)
download_all_data_for_training(strategy.dp, freqai_conf)
assert log_has_re(
"Downloading",
caplog,
)