adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reenable Catboost test (#7118)

Browse Source 
* Reenable Catboost test

* Simplify freqAI tests, ensure they use a tempdir for modelstorage
This commit is contained in:
Matthias 2022-07-24 07:32:13 +02:00 committed by GitHub
parent f3d46613ee
commit 95f5218ceb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 96 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/freqai/conftest.py
View File

@ -2,6 +2,8 @@ from copy import deepcopy
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
@ -10,13 +12,14 @@ from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
from tests.conftest import get_patched_exchange
# @pytest.fixture(scope="function")
def freqai_conf(default_conf):
@pytest.fixture(scope="function")
def freqai_conf(default_conf, tmpdir):
freqaiconf = deepcopy(default_conf)
freqaiconf.update(
{
"datadir": Path(default_conf["datadir"]),
"strategy": "freqai_test_strat",
"user_data_dir": Path(tmpdir),
"strategy-path": "freqtrade/tests/strategy/strats",
"freqaimodel": "LightGBMPredictionModel",
"freqaimodel_path": "freqai/prediction_models",

79
tests/freqai/test_freqai_datakitchen.py
View File

@ -1,6 +1,3 @@
# from unittest.mock import MagicMock
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
import copy
import datetime
import shutil
from pathlib import Path
@ -13,7 +10,7 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from tests.conftest import get_patched_exchange
from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_freqai_strategy
from tests.freqai.conftest import get_patched_data_kitchen, get_patched_freqai_strategy
@pytest.mark.parametrize(
@ -24,15 +21,15 @@ from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_pat
],
)
def test_create_fulltimerange(
timerange, train_period_days, expected_result, default_conf, mocker, caplog
timerange, train_period_days, expected_result, freqai_conf, mocker, caplog
):
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
dk = get_patched_data_kitchen(mocker, freqai_conf)
assert dk.create_fulltimerange(timerange, train_period_days) == expected_result
shutil.rmtree(Path(dk.full_path))
def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
def test_create_fulltimerange_incorrect_backtest_period(mocker, freqai_conf):
dk = get_patched_data_kitchen(mocker, freqai_conf)
with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"):
dk.create_fulltimerange("20220101-20220201", 0.5)
with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"):
@ -49,11 +46,10 @@ def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
],
)
def test_split_timerange(
mocker, default_conf, timerange, train_period_days, backtest_period_days, expected_result
mocker, freqai_conf, timerange, train_period_days, backtest_period_days, expected_result
):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
freqaiconf.update({"timerange": "20220101-20220401"})
dk = get_patched_data_kitchen(mocker, freqaiconf)
freqai_conf.update({"timerange": "20220101-20220401"})
dk = get_patched_data_kitchen(mocker, freqai_conf)
tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days)
assert len(tr_list) == len(bt_list) == expected_result
@ -64,14 +60,13 @@ def test_split_timerange(
shutil.rmtree(Path(dk.full_path))
def test_update_historic_data(mocker, default_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
def test_update_historic_data(mocker, freqai_conf):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange)
@ -93,69 +88,65 @@ def test_update_historic_data(mocker, default_conf):
(datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False),
],
)
def test_check_if_model_expired(mocker, default_conf, timestamp, expected):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
dk = get_patched_data_kitchen(mocker, freqaiconf)
def test_check_if_model_expired(mocker, freqai_conf, timestamp, expected):
dk = get_patched_data_kitchen(mocker, freqai_conf)
assert dk.check_if_model_expired(timestamp) == expected
shutil.rmtree(Path(dk.full_path))
def test_load_all_pairs_histories(mocker, default_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
def test_load_all_pairs_histories(mocker, freqai_conf):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange)
assert len(freqai.dd.historic_data.keys()) == len(
freqaiconf.get("exchange", {}).get("pair_whitelist")
freqai_conf.get("exchange", {}).get("pair_whitelist")
)
assert len(freqai.dd.historic_data["ADA/BTC"]) == len(
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
)
shutil.rmtree(Path(freqai.dk.full_path))
def test_get_base_and_corr_dataframes(mocker, default_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
def test_get_base_and_corr_dataframes(mocker, freqai_conf):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180111-20180114")
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
num_tfs = len(
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
)
assert len(base_df.keys()) == num_tfs
assert len(corr_df.keys()) == len(
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist")
freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist")
)
assert len(corr_df["ADA/BTC"].keys()) == num_tfs
shutil.rmtree(Path(freqai.dk.full_path))
def test_use_strategy_to_populate_indicators(mocker, default_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {})
def test_use_strategy_to_populate_indicators(mocker, freqai_conf):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180111-20180114")

130
tests/freqai/test_freqai_interface.py
View File

@ -1,29 +1,29 @@
# from unittest.mock import MagicMock
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
import copy
# import platform
import platform
import shutil
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from tests.conftest import get_patched_exchange, log_has_re
from tests.freqai.conftest import freqai_conf, get_patched_freqai_strategy
from tests.freqai.conftest import get_patched_freqai_strategy
def test_train_model_in_series_LightGBM(mocker, default_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
freqaiconf.update({"timerange": "20180110-20180130"})
def test_train_model_in_series_LightGBM(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180110-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange)
@ -58,64 +58,47 @@ def test_train_model_in_series_LightGBM(mocker, default_conf):
shutil.rmtree(Path(freqai.dk.full_path))
# FIXME: hits segfault
# @pytest.mark.skipif("arm" in platform.uname()[-1], reason="no ARM..")
# def test_train_model_in_series_Catboost(mocker, default_conf):
# freqaiconf = freqai_conf(copy.deepcopy(default_conf))
# freqaiconf.update({"timerange": "20180110-20180130"})
# freqaiconf.update({"freqaimodel": "CatboostPredictionModel"})
# strategy = get_patched_freqai_strategy(mocker, freqaiconf)
# exchange = get_patched_exchange(mocker, freqaiconf)
# strategy.dp = DataProvider(freqaiconf, exchange)
# strategy.freqai_info = freqaiconf.get("freqai", {})
# freqai = strategy.model.bridge
# freqai.live = True
# freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
# timerange = TimeRange.parse_timerange("20180110-20180130")
# freqai.dk.load_all_pair_histories(timerange)
@pytest.mark.skipif("arm" in platform.uname()[-1], reason="no ARM for Catboost ...")
def test_train_model_in_series_Catboost(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"freqaimodel": "CatboostPredictionModel"})
del freqai_conf['freqai']['model_training_parameters']['verbosity']
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
# freqai.dd.pair_dict = MagicMock()
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange)
# data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
# new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.dd.pair_dict = MagicMock()
# freqai.train_model_in_series(new_timerange, "ADA/BTC",
# strategy, freqai.dk, data_load_timerange)
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib"))
# .resolve()
# .exists()
# )
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json"))
# .resolve()
# .exists()
# )
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl"))
# .resolve()
# .exists()
# )
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib"))
# .resolve()
# .exists()
# )
freqai.train_model_in_series(new_timerange, "ADA/BTC",
strategy, freqai.dk, data_load_timerange)
# shutil.rmtree(Path(freqai.dk.full_path))
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists()
shutil.rmtree(Path(freqai.dk.full_path))
def test_start_backtesting(mocker, default_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
freqaiconf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {})
def test_start_backtesting(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = False
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180110-20180130")
@ -132,16 +115,15 @@ def test_start_backtesting(mocker, default_conf):
shutil.rmtree(Path(freqai.dk.full_path))
def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog):
freqaiconf = freqai_conf(copy.deepcopy(default_conf))
freqaiconf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {})
def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqai_conf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = False
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180110-20180130")
@ -157,14 +139,14 @@ def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog):
# without deleting the exiting folder structure, re-run
freqaiconf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
exchange = get_patched_exchange(mocker, freqaiconf)
strategy.dp = DataProvider(freqaiconf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {})
freqai_conf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = False
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180110-20180130")