backtest_live_models - added new tests and refactoring

freqtrade/freqai/data_kitchen.py

@@ -473,8 +473,8 @@ class FreqaiDataKitchen:
         pair_data = self.backtest_live_models_data["pairs_end_dates"][pair]
         model_end_dates = []
         backtesting_timerange = self.backtest_live_models_data["backtesting_timerange"]
-        for data in pair_data:
+        for end_date in pair_data:
-            model_end_dates.append(data["model_end_date"])
+            model_end_dates.append(end_date)
         model_end_dates.append(backtesting_timerange.stopts)
         model_end_dates.sort()
         for index, item in enumerate(model_end_dates):

freqtrade/freqai/freqai_util.py

@@ -36,34 +36,11 @@ def get_timerange_from_ready_models(models_path: Path) -> Tuple[TimeRange, str,
     pairs_end_dates: Dict with pair and model end training dates info)
     """
     all_models_end_dates = []
-    pairs_end_dates: Dict[str, Any] = {}
+    pairs_end_dates: Dict[str, Any] = get_pairs_timestamps_training_from_ready_models(models_path)
-    if not models_path.is_dir():
+    for key in pairs_end_dates:
-        raise OperationalException(
+        for model_end_date in pairs_end_dates[key]:
-            'Model folders not found. Saved models are required '
+            if model_end_date not in all_models_end_dates:
-            'to run backtest with the freqai-backtest-live-models option'
+                all_models_end_dates.append(model_end_date)
-        )
-    for model_dir in models_path.iterdir():
-        if str(model_dir.name).startswith("sub-train"):
-            model_end_date = int(model_dir.name.split("_")[1])
-            pair = model_dir.name.split("_")[0].replace("sub-train-", "")
-            model_file_name = (
-                f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}"
-                "_model.joblib"
-            )
-
-            model_path_file = Path(model_dir / model_file_name)
-            if model_path_file.is_file():
-                if pair not in pairs_end_dates:
-                    pairs_end_dates[pair] = []
-
-                pairs_end_dates[pair].append({
-                    "model_end_date": model_end_date,
-                    "model_path_file": model_path_file,
-                    "model_dir": model_dir
-                })
-
-                if model_end_date not in all_models_end_dates:
-                    all_models_end_dates.append(model_end_date)
 
     if len(all_models_end_dates) == 0:
         raise OperationalException(
@@ -104,3 +81,35 @@ def get_timerange_from_ready_models(models_path: Path) -> Tuple[TimeRange, str,
         'date', 'date', min(all_models_end_dates), max(all_models_end_dates)
     )
     return backtesting_timerange, backtesting_string_timerange, pairs_end_dates
+
+
+def get_pairs_timestamps_training_from_ready_models(models_path: Path) -> Dict[str, Any]:
+    """
+    Scan the models path and returns all pairs end training dates (timestamp)
+    :param models_path: FreqAI model path
+
+    :returns:
+    :pairs_end_dates: Dict with pair and model end training dates info
+    """
+    pairs_end_dates: Dict[str, Any] = {}
+    if not models_path.is_dir():
+        raise OperationalException(
+            'Model folders not found. Saved models are required '
+            'to run backtest with the freqai-backtest-live-models option'
+        )
+    for model_dir in models_path.iterdir():
+        if str(model_dir.name).startswith("sub-train"):
+            model_end_date = int(model_dir.name.split("_")[1])
+            pair = model_dir.name.split("_")[0].replace("sub-train-", "")
+            model_file_name = (
+                f"cb_{str(model_dir.name).replace('sub-train-', '').lower()}"
+                "_model.joblib"
+            )
+
+            model_path_file = Path(model_dir / model_file_name)
+            if model_path_file.is_file():
+                if pair not in pairs_end_dates:
+                    pairs_end_dates[pair] = []
+
+                pairs_end_dates[pair].append(model_end_date)
+    return pairs_end_dates

tests/freqai/test_freqai_util.py

@@ -0,0 +1,105 @@
+import platform
+from unittest.mock import MagicMock
+
+import pytest
+
+from freqtrade.configuration import TimeRange
+from freqtrade.data.dataprovider import DataProvider
+from freqtrade.exceptions import OperationalException
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+from freqtrade.freqai.freqai_util import (get_full_model_path,
+                                          get_pairs_timestamps_training_from_ready_models,
+                                          get_timerange_from_ready_models)
+from tests.conftest import get_patched_exchange
+from tests.freqai.conftest import get_patched_freqai_strategy
+
+
+def is_arm() -> bool:
+    machine = platform.machine()
+    return "arm" in machine or "aarch64" in machine
+
+
+@pytest.mark.parametrize('model', [
+    'LightGBMRegressor'
+    ])
+def test_get_full_model_path(mocker, freqai_conf, model):
+    if is_arm() and model == 'CatboostRegressor':
+        pytest.skip("CatBoost is not supported on ARM")
+
+    freqai_conf.update({"freqaimodel": model})
+    freqai_conf.update({"timerange": "20180110-20180130"})
+    freqai_conf.update({"strategy": "freqai_test_strat"})
+
+    strategy = get_patched_freqai_strategy(mocker, freqai_conf)
+    exchange = get_patched_exchange(mocker, freqai_conf)
+    strategy.dp = DataProvider(freqai_conf, exchange)
+    strategy.freqai_info = freqai_conf.get("freqai", {})
+    freqai = strategy.freqai
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqai_conf)
+    timerange = TimeRange.parse_timerange("20180110-20180130")
+    freqai.dd.load_all_pair_histories(timerange, freqai.dk)
+
+    freqai.dd.pair_dict = MagicMock()
+
+    data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
+    new_timerange = TimeRange.parse_timerange("20180120-20180130")
+
+    freqai.extract_data_and_train_model(
+        new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
+
+    model_path = get_full_model_path(freqai_conf)
+    assert model_path.is_dir() is True
+
+
+def test_get_pairs_timestamp_validation(mocker, freqai_conf):
+    model_path = get_full_model_path(freqai_conf)
+    with pytest.raises(
+            OperationalException,
+            match=r'.*required to run backtest with the freqai-backtest-live-models.*'
+            ):
+        get_pairs_timestamps_training_from_ready_models(model_path)
+
+
+@pytest.mark.parametrize('model', [
+    'LightGBMRegressor'
+    ])
+def test_get_timerange_from_ready_models(mocker, freqai_conf, model):
+    if is_arm() and model == 'CatboostRegressor':
+        pytest.skip("CatBoost is not supported on ARM")
+
+    freqai_conf.update({"freqaimodel": model})
+    freqai_conf.update({"timerange": "20180110-20180130"})
+    freqai_conf.update({"strategy": "freqai_test_strat"})
+
+    strategy = get_patched_freqai_strategy(mocker, freqai_conf)
+    exchange = get_patched_exchange(mocker, freqai_conf)
+    strategy.dp = DataProvider(freqai_conf, exchange)
+    strategy.freqai_info = freqai_conf.get("freqai", {})
+    freqai = strategy.freqai
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqai_conf)
+    timerange = TimeRange.parse_timerange("20180101-20180130")
+    freqai.dd.load_all_pair_histories(timerange, freqai.dk)
+
+    freqai.dd.pair_dict = MagicMock()
+
+    data_load_timerange = TimeRange.parse_timerange("20180101-20180130")
+
+    new_timerange = TimeRange.parse_timerange("20180120-20180122")
+    freqai.extract_data_and_train_model(
+        new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
+
+    new_timerange = TimeRange.parse_timerange("20180122-20180124")
+    freqai.extract_data_and_train_model(
+        new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
+
+    model_path = get_full_model_path(freqai_conf)
+    (backtesting_timerange,
+     backtesting_string_timerange,
+     pairs_end_dates) = get_timerange_from_ready_models(models_path=model_path)
+
+    assert len(pairs_end_dates["ADA"]) == 2
+    assert backtesting_string_timerange == '20180122-20180127'
+    assert backtesting_timerange.startts == 1516579200
+    assert backtesting_timerange.stopts == 1516924800

tests/test_configuration.py

@@ -1574,3 +1574,75 @@ def test_flat_vars_to_nested_dict(caplog):
 
     assert log_has("Loading variable 'FREQTRADE__EXCHANGE__SOME_SETTING'", caplog)
     assert not log_has("Loading variable 'NOT_RELEVANT'", caplog)
+
+
+def test_setup_hyperopt_freqai(mocker, default_conf, caplog) -> None:
+    patched_configuration_load_config_file(mocker, default_conf)
+    mocker.patch(
+        'freqtrade.configuration.configuration.create_datadir',
+        lambda c, x: x
+    )
+    mocker.patch(
+        'freqtrade.configuration.configuration.create_userdata_dir',
+        lambda x, *args, **kwargs: Path(x)
+    )
+    arglist = [
+        'hyperopt',
+        '--config', 'config.json',
+        '--strategy', CURRENT_TEST_STRATEGY,
+        '--timerange', '20220801-20220805',
+        "--freqaimodel",
+        "LightGBMRegressorMultiTarget",
+        "--analyze-per-epoch"
+    ]
+
+    args = Arguments(arglist).get_parsed_arg()
+
+    configuration = Configuration(args)
+    config = configuration.get_config()
+    config['freqai'] = {
+        "enabled": True
+    }
+    with pytest.raises(
+        OperationalException, match=r".*analyze-per-epoch parameter is not supported.*"
+    ):
+        validate_config_consistency(config)
+
+
+def test_setup_freqai_backtest_live_models(mocker, default_conf, caplog) -> None:
+    patched_configuration_load_config_file(mocker, default_conf)
+    mocker.patch(
+        'freqtrade.configuration.configuration.create_datadir',
+        lambda c, x: x
+    )
+    mocker.patch(
+        'freqtrade.configuration.configuration.create_userdata_dir',
+        lambda x, *args, **kwargs: Path(x)
+    )
+    arglist = [
+        'backtesting',
+        '--config', 'config.json',
+        '--strategy', CURRENT_TEST_STRATEGY,
+        '--timerange', '20220801-20220805',
+        "--freqaimodel",
+        "LightGBMRegressorMultiTarget",
+        "--freqai-backtest-live-models"
+    ]
+
+    args = Arguments(arglist).get_parsed_arg()
+
+    configuration = Configuration(args)
+    config = configuration.get_config()
+    with pytest.raises(
+        OperationalException, match=r".*--freqai-backtest-live-models parameter is only.*"
+    ):
+        validate_config_consistency(config)
+
+    conf = deepcopy(config)
+    conf['freqai'] = {
+        "enabled": True
+    }
+    with pytest.raises(
+        OperationalException, match=r".* timerange parameter is not supported with .*"
+    ):
+        validate_config_consistency(conf)