Compare commits
5 Commits
develop
...
use-parque
Author | SHA1 | Date | |
---|---|---|---|
|
cae8c3a7a8 | ||
|
1fe58c3938 | ||
|
630cdcb99f | ||
|
b383654378 | ||
|
0883198f40 |
@ -74,8 +74,8 @@ class FreqaiDataDrawer:
|
||||
self.historic_predictions: Dict[str, DataFrame] = {}
|
||||
self.full_path = full_path
|
||||
self.historic_predictions_path = Path(self.full_path / "historic_predictions.pkl")
|
||||
self.historic_predictions_bkp_path = Path(
|
||||
self.full_path / "historic_predictions.backup.pkl")
|
||||
self.historic_predictions_folder = Path(self.full_path / "historic_predictions")
|
||||
self.historic_predictions_bkp_folder = Path(self.full_path / "historic_predictions_backup")
|
||||
self.pair_dictionary_path = Path(self.full_path / "pair_dictionary.json")
|
||||
self.global_metadata_path = Path(self.full_path / "global_metadata.json")
|
||||
self.metric_tracker_path = Path(self.full_path / "metric_tracker.json")
|
||||
@ -163,11 +163,12 @@ class FreqaiDataDrawer:
|
||||
Locate and load a previously saved historic predictions.
|
||||
:return: bool - whether or not the drawer was located
|
||||
"""
|
||||
exists = self.historic_predictions_path.is_file()
|
||||
exists = self.historic_predictions_folder.exists()
|
||||
convert = self.historic_predictions_path.is_file()
|
||||
|
||||
if exists:
|
||||
try:
|
||||
with self.historic_predictions_path.open("rb") as fp:
|
||||
self.historic_predictions = cloudpickle.load(fp)
|
||||
self.load_historic_predictions_from_folder()
|
||||
logger.info(
|
||||
f"Found existing historic predictions at {self.full_path}, but beware "
|
||||
"that statistics may be inaccurate if the bot has been offline for "
|
||||
@ -175,25 +176,54 @@ class FreqaiDataDrawer:
|
||||
)
|
||||
except EOFError:
|
||||
logger.warning(
|
||||
'Historical prediction file was corrupted. Trying to load backup file.')
|
||||
with self.historic_predictions_bkp_path.open("rb") as fp:
|
||||
'Historical prediction files were corrupted. Trying to load backup files.')
|
||||
self.load_historic_predictions_from_folder()
|
||||
logger.warning('FreqAI successfully loaded the backup '
|
||||
'historical predictions files.')
|
||||
|
||||
elif not exists and convert:
|
||||
logger.info("Converting your historic predictions pkl to parquet"
|
||||
"to improve performance.")
|
||||
with Path.open(self.historic_predictions_path, "rb") as fp:
|
||||
self.historic_predictions = cloudpickle.load(fp)
|
||||
logger.warning('FreqAI successfully loaded the backup historical predictions file.')
|
||||
self.save_historic_predictions_to_disk()
|
||||
exists = True
|
||||
|
||||
else:
|
||||
logger.info("Could not find existing historic_predictions, starting from scratch")
|
||||
logger.warning(
|
||||
f"Follower could not find historic predictions at {self.full_path} "
|
||||
"sending null values back to strategy"
|
||||
)
|
||||
|
||||
return exists
|
||||
|
||||
def load_historic_predictions_from_folder(self):
|
||||
"""
|
||||
Try to build the historic_predictions dictionary from parquet
|
||||
files in the historic_predictions_folder
|
||||
"""
|
||||
for file_path in self.historic_predictions_folder.glob("*.parquet"):
|
||||
key = file_path.stem
|
||||
key.replace("_", "/")
|
||||
self.historic_predictions[key] = pd.read_parquet(file_path)
|
||||
|
||||
return
|
||||
|
||||
def save_historic_predictions_to_disk(self):
|
||||
"""
|
||||
Save historic predictions pickle to disk
|
||||
"""
|
||||
with self.historic_predictions_path.open("wb") as fp:
|
||||
cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL)
|
||||
|
||||
self.historic_predictions_folder.mkdir(parents=True, exist_ok=True)
|
||||
for key, value in self.historic_predictions.items():
|
||||
key = key.replace("/", "_")
|
||||
# pytest.set_trace()
|
||||
filename = Path(self.historic_predictions_folder / f"{key}.parquet")
|
||||
value.to_parquet(filename)
|
||||
|
||||
# create a backup
|
||||
shutil.copy(self.historic_predictions_path, self.historic_predictions_bkp_path)
|
||||
shutil.copytree(self.historic_predictions_folder,
|
||||
self.historic_predictions_bkp_folder, dirs_exist_ok=True)
|
||||
|
||||
def save_metric_tracker_to_disk(self):
|
||||
"""
|
||||
@ -675,7 +705,7 @@ class FreqaiDataDrawer:
|
||||
Returns timerange information based on historic predictions file
|
||||
:return: timerange calculated from saved live data
|
||||
"""
|
||||
if not self.historic_predictions_path.is_file():
|
||||
if not self.historic_predictions_folder.exists():
|
||||
raise OperationalException(
|
||||
'Historic predictions not found. Historic predictions data is required '
|
||||
'to run backtest with the freqai-backtest-live-models option '
|
||||
|
Loading…
Reference in New Issue
Block a user