2022-07-19 14:16:44 +00:00
|
|
|
from copy import deepcopy
|
|
|
|
|
from pathlib import Path
|
2023-03-21 14:38:05 +00:00
|
|
|
from typing import Any, Dict
|
2022-08-28 11:47:01 +00:00
|
|
|
from unittest.mock import MagicMock
|
2022-07-19 14:16:44 +00:00
|
|
|
|
2022-07-24 05:32:13 +00:00
|
|
|
import pytest
|
2022-08-28 11:47:01 +00:00
|
|
|
|
2022-07-20 10:56:46 +00:00
|
|
|
from freqtrade.configuration import TimeRange
|
|
|
|
|
from freqtrade.data.dataprovider import DataProvider
|
2022-07-26 08:24:14 +00:00
|
|
|
from freqtrade.freqai.data_drawer import FreqaiDataDrawer
|
2022-07-26 08:51:39 +00:00
|
|
|
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
|
2022-07-19 14:16:44 +00:00
|
|
|
from freqtrade.resolvers import StrategyResolver
|
|
|
|
|
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
|
2022-07-20 10:56:46 +00:00
|
|
|
from tests.conftest import get_patched_exchange
|
2022-07-19 14:16:44 +00:00
|
|
|
|
|
|
|
|
|
2022-07-24 05:32:13 +00:00
|
|
|
@pytest.fixture(scope="function")
|
|
|
|
|
def freqai_conf(default_conf, tmpdir):
|
2022-07-19 14:16:44 +00:00
|
|
|
freqaiconf = deepcopy(default_conf)
|
|
|
|
|
freqaiconf.update(
|
|
|
|
|
{
|
|
|
|
|
"datadir": Path(default_conf["datadir"]),
|
2022-07-20 12:39:28 +00:00
|
|
|
"strategy": "freqai_test_strat",
|
2022-07-24 05:32:13 +00:00
|
|
|
"user_data_dir": Path(tmpdir),
|
2022-07-20 12:39:28 +00:00
|
|
|
"strategy-path": "freqtrade/tests/strategy/strats",
|
2022-07-09 08:13:33 +00:00
|
|
|
"freqaimodel": "LightGBMRegressor",
|
2022-07-19 14:16:44 +00:00
|
|
|
"freqaimodel_path": "freqai/prediction_models",
|
|
|
|
|
"timerange": "20180110-20180115",
|
|
|
|
|
"freqai": {
|
2022-08-13 07:56:21 +00:00
|
|
|
"enabled": True,
|
2023-02-22 21:27:56 +00:00
|
|
|
"purge_old_models": 2,
|
2022-09-14 22:46:35 +00:00
|
|
|
"train_period_days": 2,
|
2022-11-24 17:16:33 +00:00
|
|
|
"backtest_period_days": 10,
|
2022-07-19 14:16:44 +00:00
|
|
|
"live_retrain_hours": 0,
|
2022-07-20 10:56:46 +00:00
|
|
|
"expiration_hours": 1,
|
|
|
|
|
"identifier": "uniqe-id100",
|
2022-07-19 14:16:44 +00:00
|
|
|
"live_trained_timestamp": 0,
|
2022-09-14 22:46:35 +00:00
|
|
|
"data_kitchen_thread_count": 2,
|
2022-07-19 14:16:44 +00:00
|
|
|
"feature_parameters": {
|
|
|
|
|
"include_timeframes": ["5m"],
|
2022-09-14 22:46:35 +00:00
|
|
|
"include_corr_pairlist": ["ADA/BTC"],
|
2022-07-19 14:16:44 +00:00
|
|
|
"label_period_candles": 20,
|
2022-07-20 10:56:46 +00:00
|
|
|
"include_shifted_candles": 1,
|
2022-07-19 14:16:44 +00:00
|
|
|
"DI_threshold": 0.9,
|
|
|
|
|
"weight_factor": 0.9,
|
|
|
|
|
"principal_component_analysis": False,
|
|
|
|
|
"use_SVM_to_remove_outliers": True,
|
|
|
|
|
"stratify_training_data": 0,
|
|
|
|
|
"indicator_periods_candles": [10],
|
2023-02-21 20:08:34 +00:00
|
|
|
"shuffle_after_split": False,
|
|
|
|
|
"buffer_train_data_candles": 0
|
2022-07-19 14:16:44 +00:00
|
|
|
},
|
2022-09-14 22:46:35 +00:00
|
|
|
"data_split_parameters": {"test_size": 0.33, "shuffle": False},
|
2022-08-06 11:51:19 +00:00
|
|
|
"model_training_parameters": {"n_estimators": 100},
|
2022-07-19 14:16:44 +00:00
|
|
|
},
|
2022-07-31 11:20:11 +00:00
|
|
|
"config_files": [Path('config_examples', 'config_freqai.example.json')]
|
2022-07-19 14:16:44 +00:00
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
freqaiconf['exchange'].update({'pair_whitelist': ['ADA/BTC', 'DASH/BTC', 'ETH/BTC', 'LTC/BTC']})
|
|
|
|
|
return freqaiconf
|
|
|
|
|
|
|
|
|
|
|
2022-10-08 10:10:38 +00:00
|
|
|
def make_rl_config(conf):
|
|
|
|
|
conf.update({"strategy": "freqai_rl_test_strat"})
|
|
|
|
|
conf["freqai"].update({"model_training_parameters": {
|
|
|
|
|
"learning_rate": 0.00025,
|
|
|
|
|
"gamma": 0.9,
|
|
|
|
|
"verbose": 1
|
|
|
|
|
}})
|
|
|
|
|
conf["freqai"]["rl_config"] = {
|
|
|
|
|
"train_cycles": 1,
|
|
|
|
|
"thread_count": 2,
|
|
|
|
|
"max_trade_duration_candles": 300,
|
|
|
|
|
"model_type": "PPO",
|
|
|
|
|
"policy_type": "MlpPolicy",
|
|
|
|
|
"max_training_drawdown_pct": 0.5,
|
|
|
|
|
"net_arch": [32, 32],
|
|
|
|
|
"model_reward_parameters": {
|
|
|
|
|
"rr": 1,
|
|
|
|
|
"profit_aim": 0.02,
|
|
|
|
|
"win_reward_factor": 2
|
2023-03-07 10:33:54 +00:00
|
|
|
},
|
|
|
|
|
"drop_ohlc_from_features": False
|
|
|
|
|
}
|
2022-10-08 10:10:38 +00:00
|
|
|
|
|
|
|
|
return conf
|
|
|
|
|
|
|
|
|
|
|
2023-03-21 14:38:05 +00:00
|
|
|
def mock_pytorch_mlp_model_training_parameters() -> Dict[str, Any]:
|
2023-03-21 14:20:35 +00:00
|
|
|
return {
|
|
|
|
|
"learning_rate": 3e-4,
|
|
|
|
|
"trainer_kwargs": {
|
|
|
|
|
"max_iters": 1,
|
|
|
|
|
"batch_size": 64,
|
|
|
|
|
"max_n_eval_batches": 1,
|
|
|
|
|
},
|
|
|
|
|
"model_kwargs": {
|
|
|
|
|
"hidden_dim": 32,
|
|
|
|
|
"dropout_percent": 0.2,
|
|
|
|
|
"n_layer": 1,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2022-07-19 14:16:44 +00:00
|
|
|
def get_patched_data_kitchen(mocker, freqaiconf):
|
2022-07-26 08:24:14 +00:00
|
|
|
dk = FreqaiDataKitchen(freqaiconf)
|
2022-07-19 14:16:44 +00:00
|
|
|
return dk
|
|
|
|
|
|
|
|
|
|
|
2022-07-26 08:24:14 +00:00
|
|
|
def get_patched_data_drawer(mocker, freqaiconf):
|
|
|
|
|
# dd = mocker.patch('freqtrade.freqai.data_drawer', MagicMock())
|
|
|
|
|
dd = FreqaiDataDrawer(freqaiconf)
|
|
|
|
|
return dd
|
|
|
|
|
|
|
|
|
|
|
2022-07-20 10:56:46 +00:00
|
|
|
def get_patched_freqai_strategy(mocker, freqaiconf):
|
2022-07-19 14:16:44 +00:00
|
|
|
strategy = StrategyResolver.load_strategy(freqaiconf)
|
2022-07-23 14:05:25 +00:00
|
|
|
strategy.ft_bot_start()
|
2022-07-19 14:16:44 +00:00
|
|
|
|
|
|
|
|
return strategy
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_patched_freqaimodel(mocker, freqaiconf):
|
|
|
|
|
freqaimodel = FreqaiModelResolver.load_freqaimodel(freqaiconf)
|
|
|
|
|
|
|
|
|
|
return freqaimodel
|
2022-07-20 10:56:46 +00:00
|
|
|
|
|
|
|
|
|
2022-09-03 13:52:29 +00:00
|
|
|
def make_unfiltered_dataframe(mocker, freqai_conf):
|
|
|
|
|
freqai_conf.update({"timerange": "20180110-20180130"})
|
|
|
|
|
|
|
|
|
|
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
|
|
|
|
|
exchange = get_patched_exchange(mocker, freqai_conf)
|
|
|
|
|
strategy.dp = DataProvider(freqai_conf, exchange)
|
|
|
|
|
strategy.freqai_info = freqai_conf.get("freqai", {})
|
|
|
|
|
freqai = strategy.freqai
|
|
|
|
|
freqai.live = True
|
|
|
|
|
freqai.dk = FreqaiDataKitchen(freqai_conf)
|
2023-03-26 17:22:52 +00:00
|
|
|
freqai.dk.live = True
|
2022-09-03 13:52:29 +00:00
|
|
|
freqai.dk.pair = "ADA/BTC"
|
2022-09-06 17:46:58 +00:00
|
|
|
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
|
|
|
|
|
freqai.dd.load_all_pair_histories(data_load_timerange, freqai.dk)
|
2022-09-03 13:52:29 +00:00
|
|
|
|
|
|
|
|
freqai.dd.pair_dict = MagicMock()
|
2022-09-06 17:56:52 +00:00
|
|
|
|
2022-09-03 13:52:29 +00:00
|
|
|
new_timerange = TimeRange.parse_timerange("20180120-20180130")
|
|
|
|
|
|
|
|
|
|
corr_dataframes, base_dataframes = freqai.dd.get_base_and_corr_dataframes(
|
|
|
|
|
data_load_timerange, freqai.dk.pair, freqai.dk
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
unfiltered_dataframe = freqai.dk.use_strategy_to_populate_indicators(
|
|
|
|
|
strategy, corr_dataframes, base_dataframes, freqai.dk.pair
|
|
|
|
|
)
|
2022-10-06 17:26:33 +00:00
|
|
|
for i in range(5):
|
|
|
|
|
unfiltered_dataframe[f'constant_{i}'] = i
|
2022-09-03 13:52:29 +00:00
|
|
|
|
|
|
|
|
unfiltered_dataframe = freqai.dk.slice_dataframe(new_timerange, unfiltered_dataframe)
|
|
|
|
|
|
|
|
|
|
return freqai, unfiltered_dataframe
|
|
|
|
|
|
|
|
|
|
|
2022-08-28 10:56:39 +00:00
|
|
|
def make_data_dictionary(mocker, freqai_conf):
|
|
|
|
|
freqai_conf.update({"timerange": "20180110-20180130"})
|
|
|
|
|
|
|
|
|
|
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
|
|
|
|
|
exchange = get_patched_exchange(mocker, freqai_conf)
|
|
|
|
|
strategy.dp = DataProvider(freqai_conf, exchange)
|
|
|
|
|
strategy.freqai_info = freqai_conf.get("freqai", {})
|
|
|
|
|
freqai = strategy.freqai
|
|
|
|
|
freqai.live = True
|
|
|
|
|
freqai.dk = FreqaiDataKitchen(freqai_conf)
|
2023-03-26 17:22:52 +00:00
|
|
|
freqai.dk.live = True
|
2022-08-28 10:56:39 +00:00
|
|
|
freqai.dk.pair = "ADA/BTC"
|
2022-09-06 17:46:58 +00:00
|
|
|
data_load_timerange = TimeRange.parse_timerange("20180110-20180130")
|
|
|
|
|
freqai.dd.load_all_pair_histories(data_load_timerange, freqai.dk)
|
2022-08-28 10:56:39 +00:00
|
|
|
|
|
|
|
|
freqai.dd.pair_dict = MagicMock()
|
|
|
|
|
|
|
|
|
|
new_timerange = TimeRange.parse_timerange("20180120-20180130")
|
|
|
|
|
|
|
|
|
|
corr_dataframes, base_dataframes = freqai.dd.get_base_and_corr_dataframes(
|
|
|
|
|
data_load_timerange, freqai.dk.pair, freqai.dk
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
unfiltered_dataframe = freqai.dk.use_strategy_to_populate_indicators(
|
|
|
|
|
strategy, corr_dataframes, base_dataframes, freqai.dk.pair
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
unfiltered_dataframe = freqai.dk.slice_dataframe(new_timerange, unfiltered_dataframe)
|
|
|
|
|
|
|
|
|
|
freqai.dk.find_features(unfiltered_dataframe)
|
|
|
|
|
|
|
|
|
|
features_filtered, labels_filtered = freqai.dk.filter_features(
|
|
|
|
|
unfiltered_dataframe,
|
|
|
|
|
freqai.dk.training_features_list,
|
|
|
|
|
freqai.dk.label_list,
|
|
|
|
|
training_filter=True,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
data_dictionary = freqai.dk.make_train_test_datasets(features_filtered, labels_filtered)
|
|
|
|
|
|
|
|
|
|
data_dictionary = freqai.dk.normalize_data(data_dictionary)
|
|
|
|
|
|
|
|
|
|
return freqai
|
|
|
|
|
|
|
|
|
|
|
2022-07-20 10:56:46 +00:00
|
|
|
def get_freqai_live_analyzed_dataframe(mocker, freqaiconf):
|
|
|
|
|
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
|
|
|
|
exchange = get_patched_exchange(mocker, freqaiconf)
|
|
|
|
|
strategy.dp = DataProvider(freqaiconf, exchange)
|
2022-07-23 14:06:46 +00:00
|
|
|
freqai = strategy.freqai
|
2022-07-20 10:56:46 +00:00
|
|
|
freqai.live = True
|
|
|
|
|
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
|
|
|
|
timerange = TimeRange.parse_timerange("20180110-20180114")
|
|
|
|
|
freqai.dk.load_all_pair_histories(timerange)
|
|
|
|
|
|
|
|
|
|
strategy.analyze_pair('ADA/BTC', '5m')
|
|
|
|
|
return strategy.dp.get_analyzed_dataframe('ADA/BTC', '5m')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_freqai_analyzed_dataframe(mocker, freqaiconf):
|
|
|
|
|
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
|
|
|
|
exchange = get_patched_exchange(mocker, freqaiconf)
|
|
|
|
|
strategy.dp = DataProvider(freqaiconf, exchange)
|
|
|
|
|
strategy.freqai_info = freqaiconf.get("freqai", {})
|
2022-07-23 14:06:46 +00:00
|
|
|
freqai = strategy.freqai
|
2022-07-20 10:56:46 +00:00
|
|
|
freqai.live = True
|
|
|
|
|
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
|
|
|
|
timerange = TimeRange.parse_timerange("20180110-20180114")
|
|
|
|
|
freqai.dk.load_all_pair_histories(timerange)
|
|
|
|
|
sub_timerange = TimeRange.parse_timerange("20180111-20180114")
|
|
|
|
|
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
|
|
|
|
|
|
|
|
|
return freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, 'LTC/BTC')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_ready_to_train(mocker, freqaiconf):
|
|
|
|
|
strategy = get_patched_freqai_strategy(mocker, freqaiconf)
|
|
|
|
|
exchange = get_patched_exchange(mocker, freqaiconf)
|
|
|
|
|
strategy.dp = DataProvider(freqaiconf, exchange)
|
|
|
|
|
strategy.freqai_info = freqaiconf.get("freqai", {})
|
2022-07-23 14:06:46 +00:00
|
|
|
freqai = strategy.freqai
|
2022-07-20 10:56:46 +00:00
|
|
|
freqai.live = True
|
|
|
|
|
freqai.dk = FreqaiDataKitchen(freqaiconf, freqai.dd)
|
|
|
|
|
timerange = TimeRange.parse_timerange("20180110-20180114")
|
|
|
|
|
freqai.dk.load_all_pair_histories(timerange)
|
|
|
|
|
sub_timerange = TimeRange.parse_timerange("20180111-20180114")
|
|
|
|
|
corr_df, base_df = freqai.dk.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC")
|
|
|
|
|
return corr_df, base_df, freqai, strategy
|
