adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reenable Catboost test (#7118)

Browse Source 
* Reenable Catboost test

* Simplify freqAI tests, ensure they use a tempdir for modelstorage
This commit is contained in:
Matthias 2022-07-24 07:32:13 +02:00 committed by GitHub
parent f3d46613ee
commit 95f5218ceb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 96 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/freqai/conftest.py
View File

@ -2,6 +2,8 @@ from copy import deepcopy
from pathlib import Path from pathlib import Path
from unittest.mock import MagicMock from unittest.mock import MagicMock
import pytest
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider from freqtrade.data.dataprovider import DataProvider
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
@ -10,13 +12,14 @@ from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
from tests.conftest import get_patched_exchange from tests.conftest import get_patched_exchange
# @pytest.fixture(scope="function") @pytest.fixture(scope="function")
def freqai_conf(default_conf): def freqai_conf(default_conf, tmpdir):
freqaiconf = deepcopy(default_conf) freqaiconf = deepcopy(default_conf)
freqaiconf.update( freqaiconf.update(
{ {
"datadir": Path(default_conf["datadir"]), "datadir": Path(default_conf["datadir"]),
"strategy": "freqai_test_strat", "strategy": "freqai_test_strat",
"user_data_dir": Path(tmpdir),
"strategy-path": "freqtrade/tests/strategy/strats", "strategy-path": "freqtrade/tests/strategy/strats",
"freqaimodel": "LightGBMPredictionModel", "freqaimodel": "LightGBMPredictionModel",
"freqaimodel_path": "freqai/prediction_models", "freqaimodel_path": "freqai/prediction_models",

79
tests/freqai/test_freqai_datakitchen.py
View File

@ -1,6 +1,3 @@
# from unittest.mock import MagicMock
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
import copy
import datetime import datetime
import shutil import shutil
from pathlib import Path from pathlib import Path
@ -13,7 +10,7 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from tests.conftest import get_patched_exchange from tests.conftest import get_patched_exchange
from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_freqai_strategy from tests.freqai.conftest import get_patched_data_kitchen, get_patched_freqai_strategy
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -24,15 +21,15 @@ from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_pat
], ],
) )
def test_create_fulltimerange( def test_create_fulltimerange(
timerange, train_period_days, expected_result, default_conf, mocker, caplog timerange, train_period_days, expected_result, freqai_conf, mocker, caplog
): ):
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) dk = get_patched_data_kitchen(mocker, freqai_conf)
assert dk.create_fulltimerange(timerange, train_period_days) == expected_result assert dk.create_fulltimerange(timerange, train_period_days) == expected_result
shutil.rmtree(Path(dk.full_path)) shutil.rmtree(Path(dk.full_path))
def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf): def test_create_fulltimerange_incorrect_backtest_period(mocker, freqai_conf):
dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) dk = get_patched_data_kitchen(mocker, freqai_conf)
with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"): with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"):
dk.create_fulltimerange("20220101-20220201", 0.5) dk.create_fulltimerange("20220101-20220201", 0.5)
with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"): with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"):
@ -49,11 +46,10 @@ def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
], ],
) )
def test_split_timerange( def test_split_timerange(
mocker, default_conf, timerange, train_period_days, backtest_period_days, expected_result mocker, freqai_conf, timerange, train_period_days, backtest_period_days, expected_result
): ):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) freqai_conf.update({"timerange": "20220101-20220401"})
freqaiconf.update({"timerange": "20220101-20220401"}) dk = get_patched_data_kitchen(mocker, freqai_conf)
dk = get_patched_data_kitchen(mocker, freqaiconf)
tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days) tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days)
assert len(tr_list) == len(bt_list) == expected_result assert len(tr_list) == len(bt_list) == expected_result
@ -64,14 +60,13 @@ def test_split_timerange(
shutil.rmtree(Path(dk.full_path)) shutil.rmtree(Path(dk.full_path))
def test_update_historic_data(mocker, default_conf): def test_update_historic_data(mocker, freqai_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.dp = DataProvider(freqaiconf, exchange)
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = True freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114") timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
@ -93,69 +88,65 @@ def test_update_historic_data(mocker, default_conf):
(datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False), (datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False),
], ],
) )
def test_check_if_model_expired(mocker, default_conf, timestamp, expected): def test_check_if_model_expired(mocker, freqai_conf, timestamp, expected):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) dk = get_patched_data_kitchen(mocker, freqai_conf)
dk = get_patched_data_kitchen(mocker, freqaiconf)
assert dk.check_if_model_expired(timestamp) == expected assert dk.check_if_model_expired(timestamp) == expected
shutil.rmtree(Path(dk.full_path)) shutil.rmtree(Path(dk.full_path))
def test_load_all_pairs_histories(mocker, default_conf): def test_load_all_pairs_histories(mocker, freqai_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.dp = DataProvider(freqaiconf, exchange)
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = True freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114") timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
assert len(freqai.dd.historic_data.keys()) == len( assert len(freqai.dd.historic_data.keys()) == len(
freqaiconf.get("exchange", {}).get("pair_whitelist") freqai_conf.get("exchange", {}).get("pair_whitelist")
) )
assert len(freqai.dd.historic_data["ADA/BTC"]) == len( assert len(freqai.dd.historic_data["ADA/BTC"]) == len(
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
) )
shutil.rmtree(Path(freqai.dk.full_path)) shutil.rmtree(Path(freqai.dk.full_path))
def test_get_base_and_corr_dataframes(mocker, default_conf): def test_get_base_and_corr_dataframes(mocker, freqai_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.dp = DataProvider(freqaiconf, exchange)
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = True freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114") timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180111-20180114") sub_timerange = TimeRange.parse_timerange("20180111-20180114")
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC") corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
num_tfs = len( num_tfs = len(
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes") freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
) )
assert len(base_df.keys()) == num_tfs assert len(base_df.keys()) == num_tfs
assert len(corr_df.keys()) == len( assert len(corr_df.keys()) == len(
freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist") freqai_conf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist")
) )
assert len(corr_df["ADA/BTC"].keys()) == num_tfs assert len(corr_df["ADA/BTC"].keys()) == num_tfs
shutil.rmtree(Path(freqai.dk.full_path)) shutil.rmtree(Path(freqai.dk.full_path))
def test_use_strategy_to_populate_indicators(mocker, default_conf): def test_use_strategy_to_populate_indicators(mocker, freqai_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.dp = DataProvider(freqaiconf, exchange) strategy.freqai_info = freqai_conf.get("freqai", {})
strategy.freqai_info = freqaiconf.get("freqai", {})
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = True freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180114") timerange = TimeRange.parse_timerange("20180110-20180114")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180111-20180114") sub_timerange = TimeRange.parse_timerange("20180111-20180114")

130
tests/freqai/test_freqai_interface.py
View File

@ -1,29 +1,29 @@
# from unittest.mock import MagicMock # from unittest.mock import MagicMock
# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge # from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
import copy import platform
# import platform
import shutil import shutil
from pathlib import Path from pathlib import Path
from unittest.mock import MagicMock from unittest.mock import MagicMock
import pytest
from freqtrade.configuration import TimeRange from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider from freqtrade.data.dataprovider import DataProvider
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from tests.conftest import get_patched_exchange, log_has_re from tests.conftest import get_patched_exchange, log_has_re
from tests.freqai.conftest import freqai_conf, get_patched_freqai_strategy from tests.freqai.conftest import get_patched_freqai_strategy
def test_train_model_in_series_LightGBM(mocker, default_conf): def test_train_model_in_series_LightGBM(mocker, freqai_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) freqai_conf.update({"timerange": "20180110-20180130"})
freqaiconf.update({"timerange": "20180110-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqaiconf) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqaiconf, exchange) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {}) strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = True freqai.live = True
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
@ -58,64 +58,47 @@ def test_train_model_in_series_LightGBM(mocker, default_conf):
shutil.rmtree(Path(freqai.dk.full_path)) shutil.rmtree(Path(freqai.dk.full_path))
# FIXME: hits segfault @pytest.mark.skipif("arm" in platform.uname()[-1], reason="no ARM for Catboost ...")
# @pytest.mark.skipif("arm" in platform.uname()[-1], reason="no ARM..") def test_train_model_in_series_Catboost(mocker, freqai_conf):
# def test_train_model_in_series_Catboost(mocker, default_conf): freqai_conf.update({"timerange": "20180110-20180130"})
# freqaiconf = freqai_conf(copy.deepcopy(default_conf)) freqai_conf.update({"freqaimodel": "CatboostPredictionModel"})
# freqaiconf.update({"timerange": "20180110-20180130"}) del freqai_conf['freqai']['model_training_parameters']['verbosity']
# freqaiconf.update({"freqaimodel": "CatboostPredictionModel"}) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
# strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
# exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
# strategy.dp = DataProvider(freqaiconf, exchange)
# strategy.freqai_info = freqaiconf.get("freqai", {})
# freqai = strategy.model.bridge
# freqai.live = True
# freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
# timerange = TimeRange.parse_timerange("20180110-20180130")
# freqai.dk.load_all_pair_histories(timerange)
# freqai.dd.pair_dict = MagicMock() strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange)
# data_load_timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.pair_dict = MagicMock()
# new_timerange = TimeRange.parse_timerange("20180120-20180130")
# freqai.train_model_in_series(new_timerange, "ADA/BTC", data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
# strategy, freqai.dk, data_load_timerange) new_timerange = TimeRange.parse_timerange("20180120-20180130")
# assert ( freqai.train_model_in_series(new_timerange, "ADA/BTC",
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib")) strategy, freqai.dk, data_load_timerange)
# .resolve()
# .exists()
# )
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json"))
# .resolve()
# .exists()
# )
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl"))
# .resolve()
# .exists()
# )
# assert (
# Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib"))
# .resolve()
# .exists()
# )
# shutil.rmtree(Path(freqai.dk.full_path)) assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists()
assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists()
shutil.rmtree(Path(freqai.dk.full_path))
def test_start_backtesting(mocker, default_conf): def test_start_backtesting(mocker, freqai_conf):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) freqai_conf.update({"timerange": "20180120-20180130"})
freqaiconf.update({"timerange": "20180120-20180130"}) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.dp = DataProvider(freqaiconf, exchange) strategy.freqai_info = freqai_conf.get("freqai", {})
strategy.freqai_info = freqaiconf.get("freqai", {})
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = False freqai.live = False
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180110-20180130")
@ -132,16 +115,15 @@ def test_start_backtesting(mocker, default_conf):
shutil.rmtree(Path(freqai.dk.full_path)) shutil.rmtree(Path(freqai.dk.full_path))
def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog): def test_start_backtesting_from_existing_folder(mocker, freqai_conf, caplog):
freqaiconf = freqai_conf(copy.deepcopy(default_conf)) freqai_conf.update({"timerange": "20180120-20180130"})
freqaiconf.update({"timerange": "20180120-20180130"}) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
strategy = get_patched_freqai_strategy(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.dp = DataProvider(freqaiconf, exchange) strategy.freqai_info = freqai_conf.get("freqai", {})
strategy.freqai_info = freqaiconf.get("freqai", {})
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = False freqai.live = False
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180110-20180130")
@ -157,14 +139,14 @@ def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog):
# without deleting the exiting folder structure, re-run # without deleting the exiting folder structure, re-run
freqaiconf.update({"timerange": "20180120-20180130"}) freqai_conf.update({"timerange": "20180120-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqaiconf) strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqaiconf) exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqaiconf, exchange) strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqaiconf.get("freqai", {}) strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai freqai = strategy.freqai
freqai.live = False freqai.live = False
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) freqai.dk = FreqaiDataKitchen(freqai_conf, freqai.dd)
timerange = TimeRange.parse_timerange("20180110-20180130") timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dk.load_all_pair_histories(timerange) freqai.dk.load_all_pair_histories(timerange)
sub_timerange = TimeRange.parse_timerange("20180110-20180130") sub_timerange = TimeRange.parse_timerange("20180110-20180130")