add more tests for datakitchen functionalities, add regression tests for freqai_interface train/backtest

freqtrade/freqai/data_kitchen.py

@@ -690,8 +690,6 @@ class FreqaiDataKitchen:
         Append backtest prediction from current backtest period to all previous periods
         """
 
-        # ones = np.ones(len(predictions))
-        # target_mean, target_std = ones * self.data["target_mean"], ones * self.data["target_std"]
         self.append_df = DataFrame()
         for label in self.label_list:
             self.append_df[label] = predictions[label]
@@ -707,13 +705,6 @@ class FreqaiDataKitchen:
         else:
             self.full_df = pd.concat([self.full_df, self.append_df], axis=0)
 
-        # self.full_predictions = np.append(self.full_predictions, predictions)
-        # self.full_do_predict = np.append(self.full_do_predict, do_predict)
-        # if self.freqai_config.get("feature_parameters", {}).get("DI_threshold", 0) > 0:
-        #     self.full_DI_values = np.append(self.full_DI_values, self.DI_values)
-        # self.full_target_mean = np.append(self.full_target_mean, target_mean)
-        # self.full_target_std = np.append(self.full_target_std, target_std)
-
         return
 
     def fill_predictions(self, dataframe):
@@ -734,12 +725,6 @@ class FreqaiDataKitchen:
 
         self.append_df = DataFrame()
         self.full_df = DataFrame()
-        # self.full_predictions = np.append(filler, self.full_predictions)
-        # self.full_do_predict = np.append(filler, self.full_do_predict)
-        # if self.freqai_config.get("feature_parameters", {}).get("DI_threshold", 0) > 0:
-        #     self.full_DI_values = np.append(filler, self.full_DI_values)
-        # self.full_target_mean = np.append(filler, self.full_target_mean)
-        # self.full_target_std = np.append(filler, self.full_target_std)
 
         return
 

freqtrade/freqai/freqai_interface.py

@@ -170,11 +170,10 @@ class IFreqaiModel(ABC):
             gc.collect()
             dk.data = {}  # clean the pair specific data between training window sliding
             self.training_timerange = tr_train
-            # self.training_timerange_timerange = tr_train
             dataframe_train = dk.slice_dataframe(tr_train, dataframe)
             dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe)
 
-            trained_timestamp = tr_train  # TimeRange.parse_timerange(tr_train)
+            trained_timestamp = tr_train
             tr_train_startts_str = datetime.datetime.utcfromtimestamp(tr_train.startts).strftime(
                 "%Y-%m-%d %H:%M:%S"
             )

setup.sh

@@ -77,7 +77,15 @@ function updateenv() {
         fi
     fi
 
-    ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT}
+    REQUIREMENTS_FREQAI=""
+    read -p "Do you want to install dependencies for freqai [y/N]? "
+    dev=$REPLY
+    if [[ $REPLY =~ ^[Yy]$ ]]
+    then
+        REQUIREMENTS_FREQAI="-r requirements-freqai.txt"
+    fi
+
+    ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI}
     if [ $? -ne 0 ]; then
         echo "Failed installing dependencies"
         exit 1

tests/freqai/conftest.py

@@ -2,9 +2,12 @@ from copy import deepcopy
 from pathlib import Path
 from unittest.mock import MagicMock
 
+from freqtrade.configuration import TimeRange
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
 from freqtrade.resolvers import StrategyResolver
 from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
+from tests.conftest import get_patched_exchange
 
 
 # @pytest.fixture(scope="function")
@@ -21,16 +24,17 @@ def freqai_conf(default_conf):
             "freqai": {
                 "startup_candles": 10000,
                 "purge_old_models": True,
-                "train_period_days": 15,
+                "train_period_days": 5,
-                "backtest_period_days": 7,
+                "backtest_period_days": 2,
                 "live_retrain_hours": 0,
-                "identifier": "uniqe-id7",
+                "expiration_hours": 1,
+                "identifier": "uniqe-id100",
                 "live_trained_timestamp": 0,
                 "feature_parameters": {
                     "include_timeframes": ["5m"],
                     "include_corr_pairlist": ["ADA/BTC", "DASH/BTC"],
                     "label_period_candles": 20,
-                    "include_shifted_candles": 2,
+                    "include_shifted_candles": 1,
                     "DI_threshold": 0.9,
                     "weight_factor": 0.9,
                     "principal_component_analysis": False,
@@ -40,7 +44,7 @@ def freqai_conf(default_conf):
                     "indicator_periods_candles": [10],
                 },
                 "data_split_parameters": {"test_size": 0.33, "random_state": 1},
-                "model_training_parameters": {"n_estimators": 1000, "task_type": "CPU"},
+                "model_training_parameters": {"n_estimators": 100},
             },
             "config_files": [Path('config_examples', 'config_freqai_futures.example.json')]
         }
@@ -55,7 +59,7 @@ def get_patched_data_kitchen(mocker, freqaiconf):
     return dk
 
 
-def get_patched_strategy(mocker, freqaiconf):
+def get_patched_freqai_strategy(mocker, freqaiconf):
     strategy = StrategyResolver.load_strategy(freqaiconf)
     strategy.bot_start()
 
@@ -66,3 +70,48 @@ def get_patched_freqaimodel(mocker, freqaiconf):
     freqaimodel = FreqaiModelResolver.load_freqaimodel(freqaiconf)
 
     return freqaimodel
+
+
+def get_freqai_live_analyzed_dataframe(mocker, freqaiconf):
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+    freqai.dk.load_all_pair_histories(timerange)
+
+    strategy.analyze_pair('ADA/BTC', '5m')
+    return strategy.dp.get_analyzed_dataframe('ADA/BTC', '5m')
+
+
+def get_freqai_analyzed_dataframe(mocker, freqaiconf):
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180111-20180114")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+
+    return freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC')
+
+
+def get_ready_to_train(mocker, freqaiconf):
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180111-20180114")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+    return corr_df, base_df, freqai, strategy

tests/freqai/test_freqai.py

@@ -1,95 +0,0 @@
-# from unittest.mock import MagicMock
-# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
-import copy
-
-import pytest
-
-from freqtrade.configuration import TimeRange
-from freqtrade.data.dataprovider import DataProvider
-# from freqtrade.freqai.data_drawer import FreqaiDataDrawer
-from freqtrade.exceptions import OperationalException
-from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
-from tests.conftest import get_patched_exchange
-from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_strategy
-
-
-@pytest.mark.parametrize(
-    "timerange, train_period_days, expected_result",
-    [
-        ("20220101-20220201", 30, "20211202-20220201"),
-        ("20220301-20220401", 15, "20220214-20220401"),
-    ],
-)
-def test_create_fulltimerange(
-    timerange, train_period_days, expected_result, default_conf, mocker, caplog
-):
-    dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
-    assert dk.create_fulltimerange(timerange, train_period_days) == expected_result
-
-
-def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
-    dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
-    with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"):
-        dk.create_fulltimerange("20220101-20220201", 0.5)
-    with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"):
-        dk.create_fulltimerange("20220101-20220201", -1)
-
-
-def test_split_timerange(mocker, default_conf):
-    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
-    freqaiconf.update({"timerange": "20220101-20220401"})
-    dk = get_patched_data_kitchen(mocker, freqaiconf)
-    tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 7)
-    assert len(tr_list) == len(bt_list) == 9
-
-    tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 0.5)
-    assert len(tr_list) == len(bt_list) == 120
-
-    tr_list, bt_list = dk.split_timerange("20220101-20220201", 10, 1)
-    assert len(tr_list) == len(bt_list) == 80
-
-    with pytest.raises(
-        OperationalException, match=r"train_period_days must be an integer greater than 0."
-    ):
-        dk.split_timerange("20220101-20220201", -1, 0.5)
-
-
-def test_update_historic_data(mocker, default_conf):
-    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
-    strategy = get_patched_strategy(mocker, freqaiconf)
-    exchange = get_patched_exchange(mocker, freqaiconf)
-    strategy.dp = DataProvider(freqaiconf, exchange)
-    freqai = strategy.model.bridge
-    freqai.live = True
-    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
-    timerange = TimeRange.parse_timerange("20180110-20180114")
-
-    freqai.dk.load_all_pair_histories(timerange)
-    historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
-    dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m"))
-    candle_difference = dp_candles - historic_candles
-    freqai.dk.update_historic_data(strategy)
-
-    updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
-
-    assert updated_historic_candles - historic_candles == candle_difference
-
-
-# def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'):
-#     np.random.seed(42)
-#     tf_mins = timeframe_to_minutes(timeframe)
-
-#     base = np.random.normal(20, 2, size=size)
-
-#     date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC')
-#     df = pd.DataFrame({
-#         'date': date,
-#         'open': base,
-#         'high': base + np.random.normal(2, 1, size=size),
-#         'low': base - np.random.normal(2, 1, size=size),
-#         'close': base + np.random.normal(0, 1, size=size),
-#         'volume': np.random.normal(200, size=size)
-#     }
-#     )
-#     df = df.dropna()
-#     return df

tests/freqai/test_freqai_datakitchen.py

@@ -0,0 +1,167 @@
+# from unittest.mock import MagicMock
+# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
+import copy
+import datetime
+import shutil
+from pathlib import Path
+
+import pytest
+
+from freqtrade.configuration import TimeRange
+from freqtrade.data.dataprovider import DataProvider
+# from freqtrade.freqai.data_drawer import FreqaiDataDrawer
+from freqtrade.exceptions import OperationalException
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+from tests.conftest import get_patched_exchange
+from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_freqai_strategy
+
+
+@pytest.mark.parametrize(
+    "timerange, train_period_days, expected_result",
+    [
+        ("20220101-20220201", 30, "20211202-20220201"),
+        ("20220301-20220401", 15, "20220214-20220401"),
+    ],
+)
+def test_create_fulltimerange(
+    timerange, train_period_days, expected_result, default_conf, mocker, caplog
+):
+    dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
+    assert dk.create_fulltimerange(timerange, train_period_days) == expected_result
+    shutil.rmtree(Path(dk.full_path))
+
+
+def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf):
+    dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf)))
+    with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"):
+        dk.create_fulltimerange("20220101-20220201", 0.5)
+    with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"):
+        dk.create_fulltimerange("20220101-20220201", -1)
+    shutil.rmtree(Path(dk.full_path))
+
+
+@pytest.mark.parametrize(
+    "timerange, train_period_days, backtest_period_days, expected_result",
+    [
+        ("20220101-20220201", 30, 7, 9),
+        ("20220101-20220201", 30, 0.5, 120),
+        ("20220101-20220201", 10, 1, 80),
+    ],
+)
+def test_split_timerange(
+    mocker, default_conf, timerange, train_period_days, backtest_period_days, expected_result
+):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    freqaiconf.update({"timerange": "20220101-20220401"})
+    dk = get_patched_data_kitchen(mocker, freqaiconf)
+    tr_list, bt_list = dk.split_timerange(timerange, train_period_days, backtest_period_days)
+    assert len(tr_list) == len(bt_list) == expected_result
+
+    with pytest.raises(
+        OperationalException, match=r"train_period_days must be an integer greater than 0."
+    ):
+        dk.split_timerange("20220101-20220201", -1, 0.5)
+    shutil.rmtree(Path(dk.full_path))
+
+
+def test_update_historic_data(mocker, default_conf):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+
+    freqai.dk.load_all_pair_histories(timerange)
+    historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
+    dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m"))
+    candle_difference = dp_candles - historic_candles
+    freqai.dk.update_historic_data(strategy)
+
+    updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
+
+    assert updated_historic_candles - historic_candles == candle_difference
+    shutil.rmtree(Path(freqai.dk.full_path))
+
+
+@pytest.mark.parametrize(
+    "timestamp, expected",
+    [
+        (datetime.datetime.now(tz=datetime.timezone.utc).timestamp() - 7200, True),
+        (datetime.datetime.now(tz=datetime.timezone.utc).timestamp(), False),
+    ],
+)
+def test_check_if_model_expired(mocker, default_conf, timestamp, expected):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    dk = get_patched_data_kitchen(mocker, freqaiconf)
+    assert dk.check_if_model_expired(timestamp) == expected
+    shutil.rmtree(Path(dk.full_path))
+
+
+def test_load_all_pairs_histories(mocker, default_conf):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+    freqai.dk.load_all_pair_histories(timerange)
+
+    assert len(freqai.dd.historic_data.keys()) == len(
+        freqaiconf.get("exchange", {}).get("pair_whitelist")
+    )
+    assert len(freqai.dd.historic_data["ADA/BTC"]) == len(
+        freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
+    )
+    shutil.rmtree(Path(freqai.dk.full_path))
+
+
+def test_get_base_and_corr_dataframes(mocker, default_conf):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180111-20180114")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+
+    num_tfs = len(
+        freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_timeframes")
+    )
+
+    assert len(base_df.keys()) == num_tfs
+
+    assert len(corr_df.keys()) == len(
+        freqaiconf.get("freqai", {}).get("feature_parameters", {}).get("include_corr_pairlist")
+    )
+
+    assert len(corr_df["ADA/BTC"].keys()) == num_tfs
+    shutil.rmtree(Path(freqai.dk.full_path))
+
+
+def test_use_strategy_to_populate_indicators(mocker, default_conf):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180114")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180111-20180114")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+
+    df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC')
+
+    assert len(df.columns) == 90
+    shutil.rmtree(Path(freqai.dk.full_path))

tests/freqai/test_freqai_interface.py

@@ -0,0 +1,183 @@
+# from unittest.mock import MagicMock
+# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge
+import copy
+import platform
+import shutil
+from pathlib import Path
+from unittest.mock import MagicMock
+
+from freqtrade.configuration import TimeRange
+from freqtrade.data.dataprovider import DataProvider
+# from freqtrade.freqai.data_drawer import FreqaiDataDrawer
+from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
+from tests.conftest import get_patched_exchange, log_has
+from tests.freqai.conftest import freqai_conf, get_patched_freqai_strategy
+
+
+def test_train_model_in_series_LightGBM(mocker, default_conf):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    freqaiconf.update({"timerange": "20180110-20180130"})
+
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = True
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180130")
+    freqai.dk.load_all_pair_histories(timerange)
+
+    freqai.dd.pair_dict = MagicMock()
+
+    data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
+    new_timerange = TimeRange.parse_timerange("20180120-20180130")
+
+    freqai.train_model_in_series(new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange)
+
+    assert (
+        Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib"))
+        .resolve()
+        .exists()
+    )
+    assert (
+        Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json"))
+        .resolve()
+        .exists()
+    )
+    assert (
+        Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl"))
+        .resolve()
+        .exists()
+    )
+    assert (
+        Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib"))
+        .resolve()
+        .exists()
+    )
+
+    shutil.rmtree(Path(freqai.dk.full_path))
+
+
+# Catboost not available for ARM architecture. using platform lib to check processor type
+if "arm" not in platform.uname()[-1]:
+
+    def test_train_model_in_series_Catboost(mocker, default_conf):
+        freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+        freqaiconf.update({"timerange": "20180110-20180130"})
+        freqaiconf.update({"freqaimodel": "CatboostPredictionModel"})
+        strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+        exchange = get_patched_exchange(mocker, freqaiconf)
+        strategy.dp = DataProvider(freqaiconf, exchange)
+        strategy.freqai_info = freqaiconf.get("freqai", {})
+        freqai = strategy.model.bridge
+        freqai.live = True
+        freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+        timerange = TimeRange.parse_timerange("20180110-20180130")
+        freqai.dk.load_all_pair_histories(timerange)
+
+        freqai.dd.pair_dict = MagicMock()
+
+        data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
+        new_timerange = TimeRange.parse_timerange("20180120-20180130")
+
+        freqai.train_model_in_series(
+            new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange
+        )
+
+        assert (
+            Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_model.joblib"))
+            .resolve()
+            .exists()
+        )
+        assert (
+            Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_metadata.json"))
+            .resolve()
+            .exists()
+        )
+        assert (
+            Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_trained_df.pkl"))
+            .resolve()
+            .exists()
+        )
+        assert (
+            Path(freqai.dk.data_path / str(freqai.dk.model_filename + "_svm_model.joblib"))
+            .resolve()
+            .exists()
+        )
+
+        shutil.rmtree(Path(freqai.dk.full_path))
+
+
+def test_start_backtesting(mocker, default_conf):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    freqaiconf.update({"timerange": "20180120-20180130"})
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = False
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180130")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180110-20180130")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+
+    df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
+
+    metadata = {"pair": "ADA/BTC"}
+    freqai.start_backtesting(df, metadata, freqai.dk)
+    model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
+
+    assert len(model_folders) == 5
+
+    shutil.rmtree(Path(freqai.dk.full_path))
+
+
+def test_start_backtesting_from_existing_folder(mocker, default_conf, caplog):
+    freqaiconf = freqai_conf(copy.deepcopy(default_conf))
+    freqaiconf.update({"timerange": "20180120-20180130"})
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = False
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180130")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180110-20180130")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+
+    df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
+
+    metadata = {"pair": "ADA/BTC"}
+    freqai.start_backtesting(df, metadata, freqai.dk)
+    model_folders = [x for x in freqai.dd.full_path.iterdir() if x.is_dir()]
+
+    assert len(model_folders) == 5
+
+    # without deleting the exiting folder structure, re-run
+
+    freqaiconf.update({"timerange": "20180120-20180130"})
+    strategy = get_patched_freqai_strategy(mocker, freqaiconf)
+    exchange = get_patched_exchange(mocker, freqaiconf)
+    strategy.dp = DataProvider(freqaiconf, exchange)
+    strategy.freqai_info = freqaiconf.get("freqai", {})
+    freqai = strategy.model.bridge
+    freqai.live = False
+    freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
+    timerange = TimeRange.parse_timerange("20180110-20180130")
+    freqai.dk.load_all_pair_histories(timerange)
+    sub_timerange = TimeRange.parse_timerange("20180110-20180130")
+    corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
+
+    df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
+    freqai.start_backtesting(df, metadata, freqai.dk)
+    assert log_has(
+        "Found model at user_data/models/uniqe-id100/sub-train-ADA1517097600/cb_ada_1517097600",
+        caplog,
+    )
+
+    shutil.rmtree(Path(freqai.dk.full_path))