add model expiration feature, fix bug in DI return values
This commit is contained in:
parent
0b0688a91e
commit
f631ae911b
@ -452,6 +452,24 @@ config:
|
|||||||
|
|
||||||
which will automatically purge all models older than the two most recently trained ones.
|
which will automatically purge all models older than the two most recently trained ones.
|
||||||
|
|
||||||
|
## Defining model expirations
|
||||||
|
|
||||||
|
During dry/live, FreqAI trains each pair sequentially (on separate threads/GPU from the main
|
||||||
|
Freqtrade bot). This means there is always an age discrepancy between models. If a user is training
|
||||||
|
on 50 pairs, and each pair requires 5 minutes to train, the oldest model will be over 4 hours old.
|
||||||
|
This may be undesirable if the characteristic time scale (read trade duration target) for a strategy
|
||||||
|
is much less than 4 hours. The user can decide to only make trade entries if the model is less than
|
||||||
|
a certain number of hours in age by setting the `expiration_hours` in the config file:
|
||||||
|
|
||||||
|
```json
|
||||||
|
"freqai": {
|
||||||
|
"expiration_hours": 0.5,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
In the present example, the user will only allow predictions on models that are less than 1/2 hours
|
||||||
|
old.
|
||||||
|
|
||||||
<!-- ## Dynamic target expectation
|
<!-- ## Dynamic target expectation
|
||||||
|
|
||||||
The labels used for model training have a unique statistical distribution for each separate model training.
|
The labels used for model training have a unique statistical distribution for each separate model training.
|
||||||
|
@ -30,6 +30,7 @@ class FreqaiDataDrawer:
|
|||||||
def __init__(self, full_path: Path, config: dict, follow_mode: bool = False):
|
def __init__(self, full_path: Path, config: dict, follow_mode: bool = False):
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
|
self.freqai_info = config.get('freqai', {})
|
||||||
# dictionary holding all pair metadata necessary to load in from disk
|
# dictionary holding all pair metadata necessary to load in from disk
|
||||||
self.pair_dict: Dict[str, Any] = {}
|
self.pair_dict: Dict[str, Any] = {}
|
||||||
# dictionary holding all actively inferenced models in memory given a model filename
|
# dictionary holding all actively inferenced models in memory given a model filename
|
||||||
@ -168,7 +169,8 @@ class FreqaiDataDrawer:
|
|||||||
self.model_return_values[pair]['do_preds'] = dh.full_do_predict
|
self.model_return_values[pair]['do_preds'] = dh.full_do_predict
|
||||||
self.model_return_values[pair]['target_mean'] = dh.full_target_mean
|
self.model_return_values[pair]['target_mean'] = dh.full_target_mean
|
||||||
self.model_return_values[pair]['target_std'] = dh.full_target_std
|
self.model_return_values[pair]['target_std'] = dh.full_target_std
|
||||||
self.model_return_values[pair]['DI_values'] = dh.full_DI_values
|
if self.freqai_info.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
|
self.model_return_values[pair]['DI_values'] = dh.full_DI_values
|
||||||
|
|
||||||
# if not self.follow_mode:
|
# if not self.follow_mode:
|
||||||
# self.save_model_return_values_to_disk()
|
# self.save_model_return_values_to_disk()
|
||||||
@ -189,8 +191,9 @@ class FreqaiDataDrawer:
|
|||||||
|
|
||||||
self.model_return_values[pair]['predictions'] = np.append(
|
self.model_return_values[pair]['predictions'] = np.append(
|
||||||
self.model_return_values[pair]['predictions'][i:], predictions[-1])
|
self.model_return_values[pair]['predictions'][i:], predictions[-1])
|
||||||
self.model_return_values[pair]['DI_values'] = np.append(
|
if self.freqai_info.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
self.model_return_values[pair]['DI_values'][i:], dh.DI_values[-1])
|
self.model_return_values[pair]['DI_values'] = np.append(
|
||||||
|
self.model_return_values[pair]['DI_values'][i:], dh.DI_values[-1])
|
||||||
self.model_return_values[pair]['do_preds'] = np.append(
|
self.model_return_values[pair]['do_preds'] = np.append(
|
||||||
self.model_return_values[pair]['do_preds'][i:], do_preds[-1])
|
self.model_return_values[pair]['do_preds'][i:], do_preds[-1])
|
||||||
self.model_return_values[pair]['target_mean'] = np.append(
|
self.model_return_values[pair]['target_mean'] = np.append(
|
||||||
@ -202,8 +205,9 @@ class FreqaiDataDrawer:
|
|||||||
prepend = np.zeros(abs(length_difference) - 1)
|
prepend = np.zeros(abs(length_difference) - 1)
|
||||||
self.model_return_values[pair]['predictions'] = np.insert(
|
self.model_return_values[pair]['predictions'] = np.insert(
|
||||||
self.model_return_values[pair]['predictions'], 0, prepend)
|
self.model_return_values[pair]['predictions'], 0, prepend)
|
||||||
self.model_return_values[pair]['DI_values'] = np.insert(
|
if self.freqai_info.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
self.model_return_values[pair]['DI_values'], 0, prepend)
|
self.model_return_values[pair]['DI_values'] = np.insert(
|
||||||
|
self.model_return_values[pair]['DI_values'], 0, prepend)
|
||||||
self.model_return_values[pair]['do_preds'] = np.insert(
|
self.model_return_values[pair]['do_preds'] = np.insert(
|
||||||
self.model_return_values[pair]['do_preds'], 0, prepend)
|
self.model_return_values[pair]['do_preds'], 0, prepend)
|
||||||
self.model_return_values[pair]['target_mean'] = np.insert(
|
self.model_return_values[pair]['target_mean'] = np.insert(
|
||||||
@ -215,7 +219,8 @@ class FreqaiDataDrawer:
|
|||||||
dh.full_do_predict = copy.deepcopy(self.model_return_values[pair]['do_preds'])
|
dh.full_do_predict = copy.deepcopy(self.model_return_values[pair]['do_preds'])
|
||||||
dh.full_target_mean = copy.deepcopy(self.model_return_values[pair]['target_mean'])
|
dh.full_target_mean = copy.deepcopy(self.model_return_values[pair]['target_mean'])
|
||||||
dh.full_target_std = copy.deepcopy(self.model_return_values[pair]['target_std'])
|
dh.full_target_std = copy.deepcopy(self.model_return_values[pair]['target_std'])
|
||||||
dh.full_DI_values = copy.deepcopy(self.model_return_values[pair]['DI_values'])
|
if self.freqai_info.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
|
dh.full_DI_values = copy.deepcopy(self.model_return_values[pair]['DI_values'])
|
||||||
|
|
||||||
# if not self.follow_mode:
|
# if not self.follow_mode:
|
||||||
# self.save_model_return_values_to_disk()
|
# self.save_model_return_values_to_disk()
|
||||||
@ -227,7 +232,8 @@ class FreqaiDataDrawer:
|
|||||||
dh.full_do_predict = np.zeros(len_df)
|
dh.full_do_predict = np.zeros(len_df)
|
||||||
dh.full_target_mean = np.zeros(len_df)
|
dh.full_target_mean = np.zeros(len_df)
|
||||||
dh.full_target_std = np.zeros(len_df)
|
dh.full_target_std = np.zeros(len_df)
|
||||||
dh.full_DI_values = np.zeros(len_df)
|
if self.freqai_info.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
|
dh.full_DI_values = np.zeros(len_df)
|
||||||
|
|
||||||
def purge_old_models(self) -> None:
|
def purge_old_models(self) -> None:
|
||||||
|
|
||||||
|
@ -673,7 +673,7 @@ class FreqaiDataKitchen:
|
|||||||
|
|
||||||
self.full_predictions = np.append(self.full_predictions, predictions)
|
self.full_predictions = np.append(self.full_predictions, predictions)
|
||||||
self.full_do_predict = np.append(self.full_do_predict, do_predict)
|
self.full_do_predict = np.append(self.full_do_predict, do_predict)
|
||||||
if self.freqai_config.get('feature_parameters', {}).get('DI-threshold', 0) > 0:
|
if self.freqai_config.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
self.full_DI_values = np.append(self.full_DI_values, self.DI_values)
|
self.full_DI_values = np.append(self.full_DI_values, self.DI_values)
|
||||||
self.full_target_mean = np.append(self.full_target_mean, target_mean)
|
self.full_target_mean = np.append(self.full_target_mean, target_mean)
|
||||||
self.full_target_std = np.append(self.full_target_std, target_std)
|
self.full_target_std = np.append(self.full_target_std, target_std)
|
||||||
@ -689,7 +689,7 @@ class FreqaiDataKitchen:
|
|||||||
filler = np.zeros(len_dataframe - len(self.full_predictions)) # startup_candle_count
|
filler = np.zeros(len_dataframe - len(self.full_predictions)) # startup_candle_count
|
||||||
self.full_predictions = np.append(filler, self.full_predictions)
|
self.full_predictions = np.append(filler, self.full_predictions)
|
||||||
self.full_do_predict = np.append(filler, self.full_do_predict)
|
self.full_do_predict = np.append(filler, self.full_do_predict)
|
||||||
if self.freqai_config.get('feature_parameters', {}).get('DI-threshold', 0) > 0:
|
if self.freqai_config.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
self.full_DI_values = np.append(filler, self.full_DI_values)
|
self.full_DI_values = np.append(filler, self.full_DI_values)
|
||||||
self.full_target_mean = np.append(filler, self.full_target_mean)
|
self.full_target_mean = np.append(filler, self.full_target_mean)
|
||||||
self.full_target_std = np.append(filler, self.full_target_std)
|
self.full_target_std = np.append(filler, self.full_target_std)
|
||||||
@ -725,6 +725,12 @@ class FreqaiDataKitchen:
|
|||||||
|
|
||||||
return full_timerange
|
return full_timerange
|
||||||
|
|
||||||
|
def check_if_model_expired(self, trained_timestamp: int) -> bool:
|
||||||
|
time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
|
||||||
|
elapsed_time = (time - trained_timestamp) / 3600 # hours
|
||||||
|
max_time = self.freqai_config.get('expiration_hours', 0)
|
||||||
|
return elapsed_time > max_time
|
||||||
|
|
||||||
def check_if_new_training_required(self, trained_timestamp: int) -> Tuple[bool,
|
def check_if_new_training_required(self, trained_timestamp: int) -> Tuple[bool,
|
||||||
TimeRange, TimeRange]:
|
TimeRange, TimeRange]:
|
||||||
|
|
||||||
@ -873,6 +879,8 @@ class FreqaiDataKitchen:
|
|||||||
|
|
||||||
# check if newest candle is already appended
|
# check if newest candle is already appended
|
||||||
df_dp = strategy.dp.get_pair_dataframe(pair, tf)
|
df_dp = strategy.dp.get_pair_dataframe(pair, tf)
|
||||||
|
if len(df_dp.index) == 0:
|
||||||
|
continue
|
||||||
if (
|
if (
|
||||||
str(history_data[pair][tf].iloc[-1]['date']) ==
|
str(history_data[pair][tf].iloc[-1]['date']) ==
|
||||||
str(df_dp.iloc[-1:]['date'].iloc[-1])
|
str(df_dp.iloc[-1:]['date'].iloc[-1])
|
||||||
|
@ -8,6 +8,7 @@ from abc import ABC, abstractmethod
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Tuple
|
from typing import Any, Dict, Tuple
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
import numpy.typing as npt
|
import numpy.typing as npt
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from pandas import DataFrame
|
from pandas import DataFrame
|
||||||
@ -65,6 +66,7 @@ class IFreqaiModel(ABC):
|
|||||||
self.identifier = self.freqai_info.get('identifier', 'no_id_provided')
|
self.identifier = self.freqai_info.get('identifier', 'no_id_provided')
|
||||||
self.scanning = False
|
self.scanning = False
|
||||||
self.ready_to_scan = False
|
self.ready_to_scan = False
|
||||||
|
self.first = True
|
||||||
|
|
||||||
def assert_config(self, config: Dict[str, Any]) -> None:
|
def assert_config(self, config: Dict[str, Any]) -> None:
|
||||||
|
|
||||||
@ -252,7 +254,7 @@ class IFreqaiModel(ABC):
|
|||||||
# # trained_timestamp=trained_timestamp,
|
# # trained_timestamp=trained_timestamp,
|
||||||
# # model_filename=model_filename)
|
# # model_filename=model_filename)
|
||||||
|
|
||||||
(self.retrain,
|
(_,
|
||||||
new_trained_timerange,
|
new_trained_timerange,
|
||||||
data_load_timerange) = dh.check_if_new_training_required(trained_timestamp)
|
data_load_timerange) = dh.check_if_new_training_required(trained_timestamp)
|
||||||
dh.set_paths(metadata['pair'], new_trained_timerange.stopts)
|
dh.set_paths(metadata['pair'], new_trained_timerange.stopts)
|
||||||
@ -288,6 +290,7 @@ class IFreqaiModel(ABC):
|
|||||||
|
|
||||||
# load the model and associated data into the data kitchen
|
# load the model and associated data into the data kitchen
|
||||||
self.model = dh.load_data(coin=metadata['pair'])
|
self.model = dh.load_data(coin=metadata['pair'])
|
||||||
|
|
||||||
if not self.model:
|
if not self.model:
|
||||||
logger.warning('No model ready, returning null values to strategy.')
|
logger.warning('No model ready, returning null values to strategy.')
|
||||||
self.data_drawer.return_null_values_to_strategy(dataframe, dh)
|
self.data_drawer.return_null_values_to_strategy(dataframe, dh)
|
||||||
@ -296,22 +299,38 @@ class IFreqaiModel(ABC):
|
|||||||
# ensure user is feeding the correct indicators to the model
|
# ensure user is feeding the correct indicators to the model
|
||||||
self.check_if_feature_list_matches_strategy(dataframe, dh)
|
self.check_if_feature_list_matches_strategy(dataframe, dh)
|
||||||
|
|
||||||
|
self.build_strategy_return_arrays(dataframe, dh, metadata['pair'], trained_timestamp)
|
||||||
|
|
||||||
|
return dh
|
||||||
|
|
||||||
|
def build_strategy_return_arrays(self, dataframe: DataFrame,
|
||||||
|
dh: FreqaiDataKitchen, pair: str,
|
||||||
|
trained_timestamp: int) -> None:
|
||||||
|
|
||||||
# hold the historical predictions in memory so we are sending back
|
# hold the historical predictions in memory so we are sending back
|
||||||
# correct array to strategy FIXME currently broken, but only affecting
|
# correct array to strategy FIXME currently broken, but only affecting
|
||||||
# Frequi reporting. Signals remain unaffeted.
|
# Frequi reporting. Signals remain unaffeted.
|
||||||
if metadata['pair'] not in self.data_drawer.model_return_values:
|
|
||||||
|
if pair not in self.data_drawer.model_return_values:
|
||||||
preds, do_preds = self.predict(dataframe, dh)
|
preds, do_preds = self.predict(dataframe, dh)
|
||||||
dh.append_predictions(preds, do_preds, len(dataframe))
|
dh.append_predictions(preds, do_preds, len(dataframe))
|
||||||
dh.fill_predictions(len(dataframe))
|
dh.fill_predictions(len(dataframe))
|
||||||
self.data_drawer.set_initial_return_values(metadata['pair'], dh)
|
self.data_drawer.set_initial_return_values(pair, dh)
|
||||||
|
return
|
||||||
|
elif self.dh.check_if_model_expired(trained_timestamp):
|
||||||
|
preds, do_preds, dh.DI_values = np.zeros(2), np.ones(2) * 2, np.zeros(2)
|
||||||
|
logger.warning('Model expired, returning null values to strategy. Strategy '
|
||||||
|
'construction should take care to consider this event with '
|
||||||
|
'prediction == 0 and do_predict == 2')
|
||||||
else:
|
else:
|
||||||
preds, do_preds = self.predict(dataframe.iloc[-2:], dh)
|
preds, do_preds = self.predict(dataframe.iloc[-2:], dh)
|
||||||
self.data_drawer.append_model_predictions(metadata['pair'], preds, do_preds,
|
|
||||||
dh.data["target_mean"],
|
|
||||||
dh.data["target_std"], dh,
|
|
||||||
len(dataframe))
|
|
||||||
|
|
||||||
return dh
|
self.data_drawer.append_model_predictions(pair, preds, do_preds,
|
||||||
|
dh.data["target_mean"],
|
||||||
|
dh.data["target_std"],
|
||||||
|
dh,
|
||||||
|
len(dataframe))
|
||||||
|
return
|
||||||
|
|
||||||
def check_if_feature_list_matches_strategy(self, dataframe: DataFrame,
|
def check_if_feature_list_matches_strategy(self, dataframe: DataFrame,
|
||||||
dh: FreqaiDataKitchen) -> None:
|
dh: FreqaiDataKitchen) -> None:
|
||||||
|
@ -24,7 +24,7 @@ class CatboostPredictionModel(IFreqaiModel):
|
|||||||
dataframe["do_predict"] = dh.full_do_predict
|
dataframe["do_predict"] = dh.full_do_predict
|
||||||
dataframe["target_mean"] = dh.full_target_mean
|
dataframe["target_mean"] = dh.full_target_mean
|
||||||
dataframe["target_std"] = dh.full_target_std
|
dataframe["target_std"] = dh.full_target_std
|
||||||
if self.freqai_info.get('feature_parameters', {}).get('DI-threshold', 0) > 0:
|
if self.freqai_info.get('feature_parameters', {}).get('DI_threshold', 0) > 0:
|
||||||
dataframe["DI"] = dh.full_DI_values
|
dataframe["DI"] = dh.full_DI_values
|
||||||
|
|
||||||
return dataframe
|
return dataframe
|
||||||
|
Loading…
Reference in New Issue
Block a user