adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backtest_live_models - added new tests and refactoring

Browse Source
This commit is contained in:
Wagner Costa Santos 2022-09-27 10:26:57 -03:00
parent 72aa47fc51
commit 0be115de9c
4 changed files with 216 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
freqtrade/freqai/data_kitchen.py
View File

@ -473,8 +473,8 @@ class FreqaiDataKitchen:
pair_data = self.backtest_live_models_data["pairs_end_dates"][pair]
model_end_dates = []
backtesting_timerange = self.backtest_live_models_data["backtesting_timerange"]
for data in pair_data:
model_end_dates.append(data["model_end_date"])
for end_date in pair_data:
model_end_dates.append(end_date)
model_end_dates.append(backtesting_timerange.stopts)
model_end_dates.sort()
for index, item in enumerate(model_end_dates):

65
freqtrade/freqai/freqai_util.py
View File

@ -36,34 +36,11 @@ def get_timerange_from_ready_models(models_path: Path) -> Tuple[TimeRange, str,
pairs_end_dates: Dict with pair and model end training dates info)
"""
all_models_end_dates = []
pairs_end_dates: Dict[str, Any] = {}
if not models_path.is_dir():
raise OperationalException(
'Model folders not found. Saved models are required '
'to run backtest with the freqai-backtest-live-models option'
)
for model_dir in models_path.iterdir():
if str(model_dir.name).startswith("sub-train"):
model_end_date = int(model_dir.name.split("_")[1])
pair = model_dir.name.split("_")[0].replace("sub-train-", "")
model_file_name = (
f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}"
"_model.joblib"
)
model_path_file = Path(model_dir / model_file_name)
if model_path_file.is_file():
if pair not in pairs_end_dates:
pairs_end_dates[pair] = []
pairs_end_dates[pair].append({
"model_end_date": model_end_date,
"model_path_file": model_path_file,
"model_dir": model_dir
})
if model_end_date not in all_models_end_dates:
all_models_end_dates.append(model_end_date)
pairs_end_dates: Dict[str, Any] = get_pairs_timestamps_training_from_ready_models(models_path)
for key in pairs_end_dates:
for model_end_date in pairs_end_dates[key]:
if model_end_date not in all_models_end_dates:
all_models_end_dates.append(model_end_date)
if len(all_models_end_dates) == 0:
raise OperationalException(
@ -104,3 +81,35 @@ def get_timerange_from_ready_models(models_path: Path) -> Tuple[TimeRange, str,
'date', 'date', min(all_models_end_dates), max(all_models_end_dates)
)
return backtesting_timerange, backtesting_string_timerange, pairs_end_dates
def get_pairs_timestamps_training_from_ready_models(models_path: Path) -> Dict[str, Any]:
"""
Scan the models path and returns all pairs end training dates (timestamp)
:param models_path: FreqAI model path
:returns:
:pairs_end_dates: Dict with pair and model end training dates info
"""
pairs_end_dates: Dict[str, Any] = {}
if not models_path.is_dir():
raise OperationalException(
'Model folders not found. Saved models are required '
'to run backtest with the freqai-backtest-live-models option'
)
for model_dir in models_path.iterdir():
if str(model_dir.name).startswith("sub-train"):
model_end_date = int(model_dir.name.split("_")[1])
pair = model_dir.name.split("_")[0].replace("sub-train-", "")
model_file_name = (
f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}"
"_model.joblib"
)
model_path_file = Path(model_dir / model_file_name)
if model_path_file.is_file():
if pair not in pairs_end_dates:
pairs_end_dates[pair] = []
pairs_end_dates[pair].append(model_end_date)
return pairs_end_dates

105
tests/freqai/test_freqai_util.py Normal file
View File

@ -0,0 +1,105 @@
import platform
from unittest.mock import MagicMock
import pytest
from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.freqai.freqai_util import (get_full_model_path,
get_pairs_timestamps_training_from_ready_models,
get_timerange_from_ready_models)
from tests.conftest import get_patched_exchange
from tests.freqai.conftest import get_patched_freqai_strategy
def is_arm() -> bool:
machine = platform.machine()
return "arm" in machine or "aarch64" in machine
@pytest.mark.parametrize('model', [
'LightGBMRegressor'
])
def test_get_full_model_path(mocker, freqai_conf, model):
if is_arm() and model == 'CatboostRegressor':
pytest.skip("CatBoost is not supported on ARM")
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"strategy": "freqai_test_strat"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
freqai.dd.pair_dict = MagicMock()
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180130")
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
model_path = get_full_model_path(freqai_conf)
assert model_path.is_dir() is True
def test_get_pairs_timestamp_validation(mocker, freqai_conf):
model_path = get_full_model_path(freqai_conf)
with pytest.raises(
OperationalException,
match=r'.*required to run backtest with the freqai-backtest-live-models.*'
):
get_pairs_timestamps_training_from_ready_models(model_path)
@pytest.mark.parametrize('model', [
'LightGBMRegressor'
])
def test_get_timerange_from_ready_models(mocker, freqai_conf, model):
if is_arm() and model == 'CatboostRegressor':
pytest.skip("CatBoost is not supported on ARM")
freqai_conf.update({"freqaimodel": model})
freqai_conf.update({"timerange": "20180110-20180130"})
freqai_conf.update({"strategy": "freqai_test_strat"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.live = True
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180101-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
freqai.dd.pair_dict = MagicMock()
data_load_timerange = TimeRange.parse_timerange("20180101-20180130")
new_timerange = TimeRange.parse_timerange("20180120-20180122")
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
new_timerange = TimeRange.parse_timerange("20180122-20180124")
freqai.extract_data_and_train_model(
new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
model_path = get_full_model_path(freqai_conf)
(backtesting_timerange,
backtesting_string_timerange,
pairs_end_dates) = get_timerange_from_ready_models(models_path=model_path)
assert len(pairs_end_dates["ADA"]) == 2
assert backtesting_string_timerange == '20180122-20180127'
assert backtesting_timerange.startts == 1516579200
assert backtesting_timerange.stopts == 1516924800

72
tests/test_configuration.py
View File

@ -1574,3 +1574,75 @@ def test_flat_vars_to_nested_dict(caplog):
assert log_has("Loading variable 'FREQTRADE__EXCHANGE__SOME_SETTING'", caplog)
assert not log_has("Loading variable 'NOT_RELEVANT'", caplog)
def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(
'freqtrade.configuration.configuration.create_datadir',
lambda c, x: x
)
mocker.patch(
'freqtrade.configuration.configuration.create_userdata_dir',
lambda x, *args, **kwargs: Path(x)
)
arglist = [
'hyperopt',
'--config', 'config.json',
'--strategy', CURRENT_TEST_STRATEGY,
'--timerange', '20220801-20220805',
"--freqaimodel",
"LightGBMRegressorMultiTarget",
"--analyze-per-epoch"
]
args = Arguments(arglist).get_parsed_arg()
configuration = Configuration(args)
config = configuration.get_config()
config['freqai'] = {
"enabled": True
}
with pytest.raises(
OperationalException, match=r".*analyze-per-epoch parameter is not supported.*"
):
validate_config_consistency(config)
def test_setup_freqai_backtest_live_models(mocker, default_conf, caplog) -> None:
patched_configuration_load_config_file(mocker, default_conf)
mocker.patch(
'freqtrade.configuration.configuration.create_datadir',
lambda c, x: x
)
mocker.patch(
'freqtrade.configuration.configuration.create_userdata_dir',
lambda x, *args, **kwargs: Path(x)
)
arglist = [
'backtesting',
'--config', 'config.json',
'--strategy', CURRENT_TEST_STRATEGY,
'--timerange', '20220801-20220805',
"--freqaimodel",
"LightGBMRegressorMultiTarget",
"--freqai-backtest-live-models"
]
args = Arguments(arglist).get_parsed_arg()
configuration = Configuration(args)
config = configuration.get_config()
with pytest.raises(
OperationalException, match=r".*--freqai-backtest-live-models parameter is only.*"
):
validate_config_consistency(config)
conf = deepcopy(config)
conf['freqai'] = {
"enabled": True
}
with pytest.raises(
OperationalException, match=r".* timerange parameter is not supported with .*"
):
validate_config_consistency(conf)