backtest_live_models - add function comments and tests

freqtrade/freqai/freqai_util.py

@@ -1,3 +1,6 @@
+"""
+FreqAI generic functions
+"""
 import logging
 from datetime import datetime, timezone
 from pathlib import Path
@@ -12,15 +15,33 @@ logger = logging.getLogger(__name__)
 
 
 def get_full_model_path(config: Config) -> Path:
+    """
+    Returns default FreqAI model path
+    :param config: Configuration dictionary
+    """
     freqai_config: Dict[str, Any] = config["freqai"]
     return Path(
         config["user_data_dir"] / "models" / str(freqai_config.get("identifier"))
     )
 
 
-def get_timerange_from_ready_models(models_path: Path):
+def get_timerange_from_ready_models(models_path: Path) -> tuple[TimeRange, str, dict[str, Any]]:
+    """
+    Returns timerange information based on a FreqAI model directory
+    :param models_path: FreqAI model path
+
+    :returns: a Tuple with (backtesting_timerange: Timerange calculated from directory,
+    backtesting_string_timerange: str timerange calculated from
+    directory (format example '20020822-20220830'), \
+    pairs_end_dates: Dict with pair and model end training dates info)
+    """
     all_models_end_dates = []
     pairs_end_dates: Dict[str, Any] = {}
+    if not models_path.is_dir():
+        raise OperationalException(
+            'Model folders not found. Saved models are required '
+            'to run backtest with the freqai-backtest-live-models option'
+        )
     for model_dir in models_path.iterdir():
         if str(model_dir.name).startswith("sub-train"):
             model_end_date = int(model_dir.name.split("_")[1])
@@ -47,7 +68,7 @@ def get_timerange_from_ready_models(models_path: Path):
     if len(all_models_end_dates) == 0:
         raise OperationalException(
             'At least 1 saved model is required to '
-            'run backtesting with the backtest_live_models option'
+            'run backtest with the freqai-backtest-live-models option'
         )
 
     if len(all_models_end_dates) == 1:

tests/freqai/test_freqai_backtesting.py

@@ -3,8 +3,11 @@ from datetime import datetime, timezone
 from pathlib import Path
 from unittest.mock import PropertyMock
 
+import pytest
+
 from freqtrade.commands.optimize_commands import setup_optimize_configuration
 from freqtrade.enums import RunMode
+from freqtrade.exceptions import OperationalException
 from freqtrade.optimize.backtesting import Backtesting
 from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
@@ -51,3 +54,60 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):
     assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog)
 
     Backtesting.cleanup()
+
+
+def test_freqai_backtest_live_models_validations(freqai_conf, mocker, testdatadir, caplog):
+    patch_exchange(mocker)
+
+    now = datetime.now(timezone.utc)
+    mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
+    mocker.patch('freqtrade.optimize.backtesting.history.load_data')
+    mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
+
+    patched_configuration_load_config_file(mocker, freqai_conf)
+
+    args = [
+        'backtesting',
+        '--config', 'config.json',
+        '--datadir', str(testdatadir),
+        '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'),
+        '--timeframe', '1h',
+        '--timerange', '20220108-20220115',
+        '--freqai-backtest-live-models'
+    ]
+    args = get_args(args)
+    with pytest.raises(OperationalException,
+                       match=r".* timerange parameter is not supported .*"):
+        setup_optimize_configuration(args, RunMode.BACKTEST)
+
+    Backtesting.cleanup()
+
+
+def test_freqai_backtest_live_models_model_not_found(freqai_conf, mocker, testdatadir, caplog):
+    patch_exchange(mocker)
+
+    now = datetime.now(timezone.utc)
+    mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
+    mocker.patch('freqtrade.optimize.backtesting.history.load_data')
+    mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
+    freqai_conf["timerange"] = ""
+    patched_configuration_load_config_file(mocker, freqai_conf)
+
+    args = [
+        'backtesting',
+        '--config', 'config.json',
+        '--datadir', str(testdatadir),
+        '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'),
+        '--timeframe', '1h',
+        '--freqai-backtest-live-models'
+    ]
+    args = get_args(args)
+    bt_config = setup_optimize_configuration(args, RunMode.BACKTEST)
+
+    with pytest.raises(OperationalException,
+                       match=r".* Saved models are required to run backtest .*"):
+        Backtesting(bt_config)
+
+    Backtesting.cleanup()