adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

detect if upper tf candles are new or not, append if so. Correct the epoch for candle update check

Browse Source
This commit is contained in:
robcaulk 2022-06-07 19:49:20 +02:00
parent cab8f517b4
commit 15d049cffe
3 changed files with 44 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
freqtrade/freqai/data_kitchen.py
View File

@ -297,7 +297,7 @@ class FreqaiDataKitchen:
)
if (1 - len(filtered_dataframe) / len(unfiltered_dataframe)) > 0.1 and self.live:
logger.warning(
f' {(1 - len(filtered_dataframe)/len(unfiltered_dataframe)) * 100} percent'
f' {(1 - len(filtered_dataframe)/len(unfiltered_dataframe)) * 100:.2f} percent'
' of training data dropped due to NaNs, model may perform inconsistent'
'with expectations'
)
@ -538,9 +538,10 @@ class FreqaiDataKitchen:
for prediction confidence in the Dissimilarity Index
"""
logger.info("computing average mean distance for all training points")
pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=-1)
tc = self.freqai_config.get('model_training_parameters', {}).get('thread_count', -1)
pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=tc)
avg_mean_dist = pairwise.mean(axis=1).mean()
logger.info("avg_mean_dist %s", avg_mean_dist)
logger.info(f'avg_mean_dist {avg_mean_dist:.2f}')
return avg_mean_dist
@ -668,6 +669,7 @@ class FreqaiDataKitchen:
self.full_predictions = np.append(self.full_predictions, predictions)
self.full_do_predict = np.append(self.full_do_predict, do_predict)
if self.freqai_config.get('feature_parameters', {}).get('DI-threshold', 0) > 0:
self.full_DI_values = np.append(self.full_DI_values, self.DI_values)
self.full_target_mean = np.append(self.full_target_mean, target_mean)
self.full_target_std = np.append(self.full_target_std, target_std)
@ -683,6 +685,7 @@ class FreqaiDataKitchen:
filler = np.zeros(len_dataframe - len(self.full_predictions)) # startup_candle_count
self.full_predictions = np.append(filler, self.full_predictions)
self.full_do_predict = np.append(filler, self.full_do_predict)
if self.freqai_config.get('feature_parameters', {}).get('DI-threshold', 0) > 0:
self.full_DI_values = np.append(filler, self.full_DI_values)
self.full_target_mean = np.append(filler, self.full_target_mean)
self.full_target_std = np.append(filler, self.full_target_std)
@ -728,7 +731,7 @@ class FreqaiDataKitchen:
# find the max indicator length required
max_timeframe_chars = self.freqai_config.get('timeframes')[-1]
max_period = self.freqai_config.get('feature_parameters', {}).get(
'indicator_max_period', 20)
'indicator_max_period', 50)
additional_seconds = 0
if max_timeframe_chars[-1] == 'd':
additional_seconds = max_period * SECONDS_IN_DAY * int(max_timeframe_chars[-2])
@ -863,9 +866,17 @@ class FreqaiDataKitchen:
for pair in self.all_pairs:
for tf in self.freqai_config.get('timeframes'):
lh = len(history_data[pair][tf].index)
history_data[pair][tf].loc[lh] = strategy.dp.get_pair_dataframe(pair,
tf).iloc[-1]
# check if newest candle is already appended
if (
str(history_data[pair][tf].iloc[-1]['date']) ==
str(strategy.dp.get_pair_dataframe(pair, tf).iloc[-1:]['date'].iloc[-1])
):
continue
history_data[pair][tf] = pd.concat(
[history_data[pair][tf],
strategy.dp.get_pair_dataframe(pair, tf).iloc[-1:]],
ignore_index=True, axis=0
)
logger.info(f'Length of history data {len(history_data[pair][tf])}')
@ -908,6 +919,7 @@ class FreqaiDataKitchen:
for training according to user defined train_period
metadata: dict = strategy furnished pair metadata
"""
with self.data_drawer.history_lock:
corr_dataframes: Dict[Any, Any] = {}
base_dataframes: Dict[Any, Any] = {}
historic_data = self.data_drawer.historic_data
@ -924,7 +936,8 @@ class FreqaiDataKitchen:
continue # dont repeat anything from whitelist
if p not in corr_dataframes:
corr_dataframes[p] = {}
corr_dataframes[p][tf] = self.slice_dataframe(timerange, historic_data[p][tf])
corr_dataframes[p][tf] = self.slice_dataframe(timerange,
historic_data[p][tf])
return corr_dataframes, base_dataframes

11
freqtrade/freqai/freqai_interface.py
View File

@ -216,12 +216,9 @@ class IFreqaiModel(ABC):
# append the historic data once per round
if (self.data_drawer.historic_data and
self.update_historic_data >= len(self.config.get('exchange', '')
.get('pair_whitelist'))):
self.config.get('exchange', '').get('pair_whitelist').index(metadata['pair']) == 1):
dh.update_historic_data(strategy)
self.update_historic_data = 1
else:
self.update_historic_data += 1
logger.info(f'Updating historic data on pair {metadata["pair"]}')
# if trainable, check if model needs training, if so compute new timerange,
# then save model and metadata.
@ -405,7 +402,7 @@ class IFreqaiModel(ABC):
# dh.download_new_data_for_retraining(data_load_timerange, metadata, strategy)
# corr_dataframes, base_dataframes = dh.load_pairs_histories(data_load_timerange,
# metadata)
with self.data_drawer.history_lock:
corr_dataframes, base_dataframes = dh.get_base_and_corr_dataframes(data_load_timerange,
metadata)
@ -419,7 +416,6 @@ class IFreqaiModel(ABC):
except Exception as err:
logger.exception(err)
# self.data_drawer.pair_to_end_of_training_queue(metadata['pair'])
self.training_on_separate_thread = False
self.retrain = False
return
@ -428,7 +424,6 @@ class IFreqaiModel(ABC):
model = self.train(unfiltered_dataframe, metadata, dh)
except ValueError:
logger.warning('Value error encountered during training')
# self.data_drawer.pair_to_end_of_training_queue(metadata['pair'])
self.training_on_separate_thread = False
self.retrain = False
return

2
freqtrade/freqai/prediction_models/CatboostPredictionModel.py
View File

@ -59,7 +59,7 @@ class CatboostPredictionModel(IFreqaiModel):
:model: Trained model which can be used to inference (self.predict)
"""
logger.info('--------------------Starting training'
logger.info('--------------------Starting training '
f'{metadata["pair"]} --------------------')
# create the full feature list based on user config info