adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

improve multithreaded training queue system

Browse Source
This commit is contained in:
robcaulk 2022-05-24 15:28:38 +02:00
parent 31ae2b3060
commit 58b5abbaa6
2 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
freqtrade/freqai/data_drawer.py
View File

@ -18,7 +18,7 @@ class FreqaiDataDrawer:
This object remains persistent throughout live/dry, unlike FreqaiDataKitchen, which is This object remains persistent throughout live/dry, unlike FreqaiDataKitchen, which is
reinstantiated for each coin. reinstantiated for each coin.
""" """
def __init__(self, full_path: Path): def __init__(self, full_path: Path, pair_whitelist):
# dictionary holding all pair metadata necessary to load in from disk # dictionary holding all pair metadata necessary to load in from disk
self.pair_dict: Dict[str, Any] = {} self.pair_dict: Dict[str, Any] = {}
@ -27,6 +27,8 @@ class FreqaiDataDrawer:
self.pair_data_dict: Dict[str, Any] = {} self.pair_data_dict: Dict[str, Any] = {}
self.full_path = full_path self.full_path = full_path
self.load_drawer_from_disk() self.load_drawer_from_disk()
self.training_queue: Dict[str, int] = {}
self.create_training_queue(pair_whitelist)
def load_drawer_from_disk(self): def load_drawer_from_disk(self):
exists = Path(self.full_path / str('pair_dictionary.json')).resolve().exists() exists = Path(self.full_path / str('pair_dictionary.json')).resolve().exists()
@ -71,3 +73,15 @@ class FreqaiDataDrawer:
self.pair_dict[metadata['pair']]['trained_timestamp'] = 0 self.pair_dict[metadata['pair']]['trained_timestamp'] = 0
self.pair_dict[metadata['pair']]['priority'] = 1 self.pair_dict[metadata['pair']]['priority'] = 1
return return
def create_training_queue(self, pairs: list) -> None:
for i, pair in enumerate(pairs):
self.training_queue[pair] = i + 1
def pair_to_end_of_training_queue(self, pair: str) -> None:
# march all pairs up in the queue
for p in self.training_queue:
self.training_queue[p] -= 1
# send pair to end of queue
self.training_queue[pair] = len(self.training_queue)

11
freqtrade/freqai/freqai_interface.py
View File

@ -73,7 +73,8 @@ class IFreqaiModel(ABC):
# self.new_trained_timerange = TimeRange() # self.new_trained_timerange = TimeRange()
self.set_full_path() self.set_full_path()
self.data_drawer = FreqaiDataDrawer(Path(self.full_path)) self.data_drawer = FreqaiDataDrawer(Path(self.full_path),
self.config['exchange']['pair_whitelist'])
def assert_config(self, config: Dict[str, Any]) -> None: def assert_config(self, config: Dict[str, Any]) -> None:
@ -110,8 +111,9 @@ class IFreqaiModel(ABC):
# FreqaiDataKitchen is reinstantiated for each coin # FreqaiDataKitchen is reinstantiated for each coin
if self.live: if self.live:
self.data_drawer.set_pair_dict_info(metadata) self.data_drawer.set_pair_dict_info(metadata)
print('Current train queue:', self.data_drawer.training_queue)
if (not self.training_on_separate_thread and if (not self.training_on_separate_thread and
self.data_drawer.pair_dict[metadata['pair']]['priority'] == 1): self.data_drawer.training_queue == 1):
self.dh = FreqaiDataKitchen(self.config, self.data_drawer, self.dh = FreqaiDataKitchen(self.config, self.data_drawer,
self.live, metadata["pair"]) self.live, metadata["pair"])
@ -323,8 +325,9 @@ class IFreqaiModel(ABC):
dh.set_new_model_names(metadata, new_trained_timerange) dh.set_new_model_names(metadata, new_trained_timerange)
# set this coin to lower priority to allow other coins in white list to get trained # send the pair to the end of the queue so other coins can take on the background thread
self.data_drawer.pair_dict[metadata['pair']]['priority'] = 0 # retraining
self.data_drawer.pair_to_end_of_training_queue(metadata['pair'])
dh.save_data(self.model, coin=metadata['pair']) dh.save_data(self.model, coin=metadata['pair'])