adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first functional scanning commit

Browse Source
This commit is contained in:
robcaulk 2022-06-16 00:21:15 +02:00
parent 4d472a0ea1
commit c5de0c49e4
5 changed files with 45 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
freqtrade/freqai/data_drawer.py
View File

@ -107,7 +107,7 @@ class FreqaiDataDrawer:
if isinstance(object, np.generic):
return object.item()
def get_pair_dict_info(self, metadata: dict) -> Tuple[str, int, bool, bool]:
def get_pair_dict_info(self, pair: str) -> Tuple[str, int, bool, bool]:
"""
Locate and load existing model metadata from persistent storage. If not located,
create a new one and append the current pair to it and prepare it for its first
@ -120,22 +120,22 @@ class FreqaiDataDrawer:
coin_first: bool = If the coin is fresh without metadata
return_null_array: bool = Follower could not find pair metadata
"""
pair_in_dict = self.pair_dict.get(metadata['pair'])
data_path_set = self.pair_dict.get(metadata['pair'], {}).get('data_path', None)
pair_in_dict = self.pair_dict.get(pair)
data_path_set = self.pair_dict.get(pair, {}).get('data_path', None)
return_null_array = False
if pair_in_dict:
model_filename = self.pair_dict[metadata['pair']]['model_filename']
trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp']
coin_first = self.pair_dict[metadata['pair']]['first']
model_filename = self.pair_dict[pair]['model_filename']
trained_timestamp = self.pair_dict[pair]['trained_timestamp']
coin_first = self.pair_dict[pair]['first']
elif not self.follow_mode:
self.pair_dict[metadata['pair']] = {}
model_filename = self.pair_dict[metadata['pair']]['model_filename'] = ''
coin_first = self.pair_dict[metadata['pair']]['first'] = True
trained_timestamp = self.pair_dict[metadata['pair']]['trained_timestamp'] = 0
self.pair_dict[pair] = {}
model_filename = self.pair_dict[pair]['model_filename'] = ''
coin_first = self.pair_dict[pair]['first'] = True
trained_timestamp = self.pair_dict[pair]['trained_timestamp'] = 0
if not data_path_set and self.follow_mode:
logger.warning(f'Follower could not find current pair {metadata["pair"]} in '
logger.warning(f'Follower could not find current pair {pair} in '
f'pair_dictionary at path {self.full_path}, sending null values '
'back to strategy.')
return_null_array = True

7
freqtrade/freqai/data_kitchen.py
View File

@ -151,6 +151,9 @@ class FreqaiDataKitchen:
:model: User trained model which can be inferenced for new predictions
"""
if not self.data_drawer.pair_dict[coin]['model_filename']:
return None
if self.live:
self.model_filename = self.data_drawer.pair_dict[coin]['model_filename']
self.data_path = Path(self.data_drawer.pair_dict[coin]['data_path'])
@ -747,7 +750,7 @@ class FreqaiDataKitchen:
logger.warning('FreqAI could not detect max timeframe and therefore may not '
'download the proper amount of data for training')
logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days')
# logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days')
if trained_timestamp != 0:
elapsed_time = (time - trained_timestamp) / SECONDS_IN_DAY
@ -937,7 +940,7 @@ class FreqaiDataKitchen:
for tf in self.freqai_config.get('timeframes'):
base_dataframes[tf] = self.slice_dataframe(
timerange,
historic_data[metadata['pair']][tf]
historic_data[pair][tf]
)
if pairs:
for p in pairs:

41
freqtrade/freqai/freqai_interface.py
View File

@ -124,18 +124,19 @@ class IFreqaiModel(ABC):
file_exists = False
# dh.set_paths(pair, trained_timestamp)
dh.set_paths(pair, trained_timestamp)
file_exists = self.model_exists(pair,
dh,
trained_timestamp=trained_timestamp,
model_filename=model_filename)
model_filename=model_filename,
scanning=True)
(self.retrain,
(retrain,
new_trained_timerange,
data_load_timerange) = dh.check_if_new_training_required(trained_timestamp)
dh.set_paths(pair, new_trained_timerange.stopts)
if self.retrain or not file_exists:
if retrain or not file_exists:
self.train_model_in_series(new_trained_timerange, pair,
strategy, dh, data_load_timerange)
@ -226,7 +227,7 @@ class IFreqaiModel(ABC):
# get the model metadata associated with the current pair
(_,
trained_timestamp,
coin_first,
_,
return_null_array) = self.data_drawer.get_pair_dict_info(metadata['pair'])
# if the metadata doesnt exist, the follower returns null arrays to strategy
@ -264,14 +265,18 @@ class IFreqaiModel(ABC):
dh.download_all_data_for_training(data_load_timerange)
dh.load_all_pair_histories(data_load_timerange)
# train the model on the trained timerange
if coin_first and not self.scanning:
self.train_model_in_series(new_trained_timerange, metadata['pair'],
strategy, dh, data_load_timerange)
elif not coin_first and not self.scanning:
if not self.scanning:
self.scanning = True
self.start_scanning(strategy)
# train the model on the trained timerange
# if coin_first and not self.scanning:
# self.train_model_in_series(new_trained_timerange, metadata['pair'],
# strategy, dh, data_load_timerange)
# elif not coin_first and not self.scanning:
# self.scanning = True
# self.start_scanning(strategy)
# elif not trainable and not self.follow_mode:
# logger.info(f'{metadata["pair"]} holds spot '
# f'{self.data_drawer.pair_dict[metadata["pair"]]["priority"]} '
@ -283,6 +288,10 @@ class IFreqaiModel(ABC):
# load the model and associated data into the data kitchen
self.model = dh.load_data(coin=metadata['pair'])
if not self.model:
logger.warning('No model ready, returning null values to strategy.')
self.data_drawer.return_null_values_to_strategy(dataframe, dh)
return dh
# ensure user is feeding the correct indicators to the model
self.check_if_feature_list_matches_strategy(dataframe, dh)
@ -373,7 +382,7 @@ class IFreqaiModel(ABC):
# dh.remove_outliers(predict=True) # creates dropped index
def model_exists(self, pair: str, dh: FreqaiDataKitchen, trained_timestamp: int = None,
model_filename: str = '') -> bool:
model_filename: str = '', scanning: bool = False) -> bool:
"""
Given a pair and path, check if a model already exists
:param pair: pair e.g. BTC/USD
@ -386,9 +395,9 @@ class IFreqaiModel(ABC):
path_to_modelfile = Path(dh.data_path / str(model_filename + "_model.joblib"))
file_exists = path_to_modelfile.is_file()
if file_exists:
if file_exists and not scanning:
logger.info("Found model at %s", dh.data_path / dh.model_filename)
else:
elif not scanning:
logger.info("Could not find model at %s", dh.data_path / dh.model_filename)
return file_exists
@ -453,8 +462,8 @@ class IFreqaiModel(ABC):
with self.lock:
self.data_drawer.pair_to_end_of_training_queue(pair)
dh.save_data(model, coin=pair)
self.training_on_separate_thread = False
self.retrain = False
# self.training_on_separate_thread = False
# self.retrain = False
# each time we finish a training, we check the directory to purge old models.
if self.freqai_info.get('purge_old_models', False):
@ -499,7 +508,7 @@ class IFreqaiModel(ABC):
with self.lock:
self.data_drawer.pair_to_end_of_training_queue(pair)
dh.save_data(model, coin=pair)
self.retrain = False
# self.retrain = False
# Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModlel.py for an example.

6
freqtrade/freqai/prediction_models/CatboostPredictionModel.py
View File

@ -48,7 +48,7 @@ class CatboostPredictionModel(IFreqaiModel):
return dataframe["s"]
def train(self, unfiltered_dataframe: DataFrame,
metadata: dict, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]:
pair: str, dh: FreqaiDataKitchen) -> Tuple[DataFrame, DataFrame]:
"""
Filter the training data and train a model to it. Train makes heavy use of the datahkitchen
for storing, saving, loading, and analyzing the data.
@ -60,7 +60,7 @@ class CatboostPredictionModel(IFreqaiModel):
"""
logger.info('--------------------Starting training '
f'{metadata["pair"]} --------------------')
f'{pair} --------------------')
# create the full feature list based on user config info
dh.training_features_list = dh.find_features(unfiltered_dataframe)
@ -88,7 +88,7 @@ class CatboostPredictionModel(IFreqaiModel):
model = self.fit(data_dictionary)
logger.info(f'--------------------done training {metadata["pair"]}--------------------')
logger.info(f'--------------------done training {pair}--------------------')
return model

3
freqtrade/templates/FreqaiExampleStrategy.py
View File

@ -116,7 +116,6 @@ class FreqaiExampleStrategy(IStrategy):
informative[f"{coin}bb_upperband-period_{t}"]
- informative[f"{coin}bb_lowerband-period_{t}"]
) / informative[f"{coin}bb_middleband-period_{t}"]
informative[f"%-{coin}close-bb_lower-period_{t}"] = (
informative["close"] / informative[f"{coin}bb_lowerband-period_{t}"]
)
@ -153,7 +152,7 @@ class FreqaiExampleStrategy(IStrategy):
# Add generalized indicators here (because in live, it will call this
# function to populate indicators during training). Notice how we ensure not to
# add them multiple times
if pair == metadata["pair"] and tf == self.timeframe:
if pair == self.freqai_info['corr_pairlist'][0] and tf == self.timeframe:
df["%-day_of_week"] = (df["date"].dt.dayofweek + 1) / 7
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25