From 714d9534b65edd4103fef7c2d5a9ba3700608b47 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 19 Jul 2022 16:16:44 +0200 Subject: [PATCH] start adding tests --- freqtrade/freqai/data_kitchen.py | 36 ++++-------- tests/freqai/conftest.py | 68 +++++++++++++++++++++++ tests/freqai/test_freqai.py | 95 ++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 24 deletions(-) create mode 100644 tests/freqai/conftest.py create mode 100644 tests/freqai/test_freqai.py diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f320bdc2f..02b121134 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -410,6 +410,11 @@ class FreqaiDataKitchen: bt_split: the backtesting length (dats). Specified in user configuration file """ + if not isinstance(train_split, int) or train_split < 1: + raise OperationalException( + "train_period_days must be an integer greater than 0. " + f"Got {train_split}." + ) train_period_days = train_split * SECONDS_IN_DAY bt_period = bt_split * SECONDS_IN_DAY @@ -742,6 +747,13 @@ class FreqaiDataKitchen: return def create_fulltimerange(self, backtest_tr: str, backtest_period_days: int) -> str: + + if not isinstance(backtest_period_days, int): + raise OperationalException('backtest_period_days must be an integer') + + if backtest_period_days < 0: + raise OperationalException('backtest_period_days must be positive') + backtest_timerange = TimeRange.parse_timerange(backtest_tr) if backtest_timerange.stopts == 0: @@ -869,30 +881,6 @@ class FreqaiDataKitchen: self.model_filename = "cb_" + coin.lower() + "_" + str(int(trained_timerange.stopts)) - # self.freqai_config['live_trained_timerange'] = str(int(trained_timerange.stopts)) - # enables persistence, but not fully implemented into save/load data yer - # self.data['live_trained_timerange'] = str(int(trained_timerange.stopts)) - - # SUPERCEDED - # def download_new_data_for_retraining(self, timerange: TimeRange, metadata: dict, - # strategy: IStrategy) -> None: - - # exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], - # self.config, validate=False, freqai=True) - # # exchange = strategy.dp._exchange # closes ccxt session - # pairs = copy.deepcopy(self.freqai_config.get('corr_pairlist', [])) - # if str(metadata['pair']) not in pairs: - # pairs.append(str(metadata['pair'])) - - # refresh_backtest_ohlcv_data( - # exchange, pairs=pairs, timeframes=self.freqai_config.get('timeframes'), - # datadir=self.config['datadir'], timerange=timerange, - # new_pairs_days=self.config['new_pairs_days'], - # erase=False, data_format=self.config.get('dataformat_ohlcv', 'json'), - # trading_mode=self.config.get('trading_mode', 'spot'), - # prepend=self.config.get('prepend_data', False) - # ) - def download_all_data_for_training(self, timerange: TimeRange) -> None: """ Called only once upon start of bot to download the necessary data for diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py new file mode 100644 index 000000000..cfc23939e --- /dev/null +++ b/tests/freqai/conftest.py @@ -0,0 +1,68 @@ +from copy import deepcopy +from pathlib import Path +from unittest.mock import MagicMock + +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver + + +# @pytest.fixture(scope="function") +def freqai_conf(default_conf): + freqaiconf = deepcopy(default_conf) + freqaiconf.update( + { + "datadir": Path(default_conf["datadir"]), + "strategy": "FreqaiExampleStrategy", + "strategy-path": "freqtrade/templates", + "freqaimodel": "LightGBMPredictionModel", + "freqaimodel_path": "freqai/prediction_models", + "timerange": "20180110-20180115", + "freqai": { + "startup_candles": 10000, + "purge_old_models": True, + "train_period_days": 15, + "backtest_period_days": 7, + "live_retrain_hours": 0, + "identifier": "uniqe-id7", + "live_trained_timestamp": 0, + "feature_parameters": { + "include_timeframes": ["5m"], + "include_corr_pairlist": ["ADA/BTC", "DASH/BTC"], + "label_period_candles": 20, + "include_shifted_candles": 2, + "DI_threshold": 0.9, + "weight_factor": 0.9, + "principal_component_analysis": False, + "use_SVM_to_remove_outliers": True, + "stratify_training_data": 0, + "indicator_max_period_candles": 10, + "indicator_periods_candles": [10], + }, + "data_split_parameters": {"test_size": 0.33, "random_state": 1}, + "model_training_parameters": {"n_estimators": 1000, "task_type": "CPU"}, + }, + "config_files": [Path('config_examples', 'config_freqai_futures.example.json')] + } + ) + freqaiconf['exchange'].update({'pair_whitelist': ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC']}) + return freqaiconf + + +def get_patched_data_kitchen(mocker, freqaiconf): + dd = mocker.patch('freqtrade.freqai.data_drawer', MagicMock()) + dk = FreqaiDataKitchen(freqaiconf, dd) + return dk + + +def get_patched_strategy(mocker, freqaiconf): + strategy = StrategyResolver.load_strategy(freqaiconf) + strategy.bot_start() + + return strategy + + +def get_patched_freqaimodel(mocker, freqaiconf): + freqaimodel = FreqaiModelResolver.load_freqaimodel(freqaiconf) + + return freqaimodel diff --git a/tests/freqai/test_freqai.py b/tests/freqai/test_freqai.py new file mode 100644 index 000000000..185e55744 --- /dev/null +++ b/tests/freqai/test_freqai.py @@ -0,0 +1,95 @@ +# from unittest.mock import MagicMock +# from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge +import copy + +import pytest + +from freqtrade.configuration import TimeRange +from freqtrade.data.dataprovider import DataProvider +# from freqtrade.freqai.data_drawer import FreqaiDataDrawer +from freqtrade.exceptions import OperationalException +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from tests.conftest import get_patched_exchange +from tests.freqai.conftest import freqai_conf, get_patched_data_kitchen, get_patched_strategy + + +@pytest.mark.parametrize( + "timerange, train_period_days, expected_result", + [ + ("20220101-20220201", 30, "20211202-20220201"), + ("20220301-20220401", 15, "20220214-20220401"), + ], +) +def test_create_fulltimerange( + timerange, train_period_days, expected_result, default_conf, mocker, caplog +): + dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) + assert dk.create_fulltimerange(timerange, train_period_days) == expected_result + + +def test_create_fulltimerange_incorrect_backtest_period(mocker, default_conf): + dk = get_patched_data_kitchen(mocker, freqai_conf(copy.deepcopy(default_conf))) + with pytest.raises(OperationalException, match=r"backtest_period_days must be an integer"): + dk.create_fulltimerange("20220101-20220201", 0.5) + with pytest.raises(OperationalException, match=r"backtest_period_days must be positive"): + dk.create_fulltimerange("20220101-20220201", -1) + + +def test_split_timerange(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + freqaiconf.update({"timerange": "20220101-20220401"}) + dk = get_patched_data_kitchen(mocker, freqaiconf) + tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 7) + assert len(tr_list) == len(bt_list) == 9 + + tr_list, bt_list = dk.split_timerange("20220101-20220201", 30, 0.5) + assert len(tr_list) == len(bt_list) == 120 + + tr_list, bt_list = dk.split_timerange("20220101-20220201", 10, 1) + assert len(tr_list) == len(bt_list) == 80 + + with pytest.raises( + OperationalException, match=r"train_period_days must be an integer greater than 0." + ): + dk.split_timerange("20220101-20220201", -1, 0.5) + + +def test_update_historic_data(mocker, default_conf): + freqaiconf = freqai_conf(copy.deepcopy(default_conf)) + strategy = get_patched_strategy(mocker, freqaiconf) + exchange = get_patched_exchange(mocker, freqaiconf) + strategy.dp = DataProvider(freqaiconf, exchange) + freqai = strategy.model.bridge + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd) + timerange = TimeRange.parse_timerange("20180110-20180114") + + freqai.dk.load_all_pair_histories(timerange) + historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) + dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) + candle_difference = dp_candles - historic_candles + freqai.dk.update_historic_data(strategy) + + updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) + + assert updated_historic_candles - historic_candles == candle_difference + + +# def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): +# np.random.seed(42) +# tf_mins = timeframe_to_minutes(timeframe) + +# base = np.random.normal(20, 2, size=size) + +# date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') +# df = pd.DataFrame({ +# 'date': date, +# 'open': base, +# 'high': base + np.random.normal(2, 1, size=size), +# 'low': base - np.random.normal(2, 1, size=size), +# 'close': base + np.random.normal(0, 1, size=size), +# 'volume': np.random.normal(200, size=size) +# } +# ) +# df = df.dropna() +# return df