add PTH ruff selection
This commit is contained in:
parent
d014e4590e
commit
26315b6bc2
@ -126,7 +126,7 @@ class FreqaiDataDrawer:
|
|||||||
"""
|
"""
|
||||||
exists = self.global_metadata_path.is_file()
|
exists = self.global_metadata_path.is_file()
|
||||||
if exists:
|
if exists:
|
||||||
with open(self.global_metadata_path, "r") as fp:
|
with self.global_metadata_path.open("r") as fp:
|
||||||
metatada_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
metatada_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
||||||
return metatada_dict
|
return metatada_dict
|
||||||
return {}
|
return {}
|
||||||
@ -139,7 +139,7 @@ class FreqaiDataDrawer:
|
|||||||
"""
|
"""
|
||||||
exists = self.pair_dictionary_path.is_file()
|
exists = self.pair_dictionary_path.is_file()
|
||||||
if exists:
|
if exists:
|
||||||
with open(self.pair_dictionary_path, "r") as fp:
|
with self.pair_dictionary_path.open("r") as fp:
|
||||||
self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
||||||
else:
|
else:
|
||||||
logger.info("Could not find existing datadrawer, starting from scratch")
|
logger.info("Could not find existing datadrawer, starting from scratch")
|
||||||
@ -152,7 +152,7 @@ class FreqaiDataDrawer:
|
|||||||
if self.freqai_info.get('write_metrics_to_disk', False):
|
if self.freqai_info.get('write_metrics_to_disk', False):
|
||||||
exists = self.metric_tracker_path.is_file()
|
exists = self.metric_tracker_path.is_file()
|
||||||
if exists:
|
if exists:
|
||||||
with open(self.metric_tracker_path, "r") as fp:
|
with self.metric_tracker_path.open("r") as fp:
|
||||||
self.metric_tracker = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
self.metric_tracker = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
||||||
logger.info("Loading existing metric tracker from disk.")
|
logger.info("Loading existing metric tracker from disk.")
|
||||||
else:
|
else:
|
||||||
@ -166,7 +166,7 @@ class FreqaiDataDrawer:
|
|||||||
exists = self.historic_predictions_path.is_file()
|
exists = self.historic_predictions_path.is_file()
|
||||||
if exists:
|
if exists:
|
||||||
try:
|
try:
|
||||||
with open(self.historic_predictions_path, "rb") as fp:
|
with self.historic_predictions_path.open("rb") as fp:
|
||||||
self.historic_predictions = cloudpickle.load(fp)
|
self.historic_predictions = cloudpickle.load(fp)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Found existing historic predictions at {self.full_path}, but beware "
|
f"Found existing historic predictions at {self.full_path}, but beware "
|
||||||
@ -176,7 +176,7 @@ class FreqaiDataDrawer:
|
|||||||
except EOFError:
|
except EOFError:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
'Historical prediction file was corrupted. Trying to load backup file.')
|
'Historical prediction file was corrupted. Trying to load backup file.')
|
||||||
with open(self.historic_predictions_bkp_path, "rb") as fp:
|
with self.historic_predictions_bkp_path.open("rb") as fp:
|
||||||
self.historic_predictions = cloudpickle.load(fp)
|
self.historic_predictions = cloudpickle.load(fp)
|
||||||
logger.warning('FreqAI successfully loaded the backup historical predictions file.')
|
logger.warning('FreqAI successfully loaded the backup historical predictions file.')
|
||||||
|
|
||||||
@ -189,7 +189,7 @@ class FreqaiDataDrawer:
|
|||||||
"""
|
"""
|
||||||
Save historic predictions pickle to disk
|
Save historic predictions pickle to disk
|
||||||
"""
|
"""
|
||||||
with open(self.historic_predictions_path, "wb") as fp:
|
with self.historic_predictions_path.open("wb") as fp:
|
||||||
cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL)
|
cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL)
|
||||||
|
|
||||||
# create a backup
|
# create a backup
|
||||||
@ -200,16 +200,16 @@ class FreqaiDataDrawer:
|
|||||||
Save metric tracker of all pair metrics collected.
|
Save metric tracker of all pair metrics collected.
|
||||||
"""
|
"""
|
||||||
with self.save_lock:
|
with self.save_lock:
|
||||||
with open(self.metric_tracker_path, 'w') as fp:
|
with self.metric_tracker_path.open('w') as fp:
|
||||||
rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder,
|
rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder,
|
||||||
number_mode=rapidjson.NM_NATIVE)
|
number_mode=rapidjson.NM_NATIVE)
|
||||||
|
|
||||||
def save_drawer_to_disk(self):
|
def save_drawer_to_disk(self) -> None:
|
||||||
"""
|
"""
|
||||||
Save data drawer full of all pair model metadata in present model folder.
|
Save data drawer full of all pair model metadata in present model folder.
|
||||||
"""
|
"""
|
||||||
with self.save_lock:
|
with self.save_lock:
|
||||||
with open(self.pair_dictionary_path, 'w') as fp:
|
with self.pair_dictionary_path.open('w') as fp:
|
||||||
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
|
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
|
||||||
number_mode=rapidjson.NM_NATIVE)
|
number_mode=rapidjson.NM_NATIVE)
|
||||||
|
|
||||||
@ -218,7 +218,7 @@ class FreqaiDataDrawer:
|
|||||||
Save global metadata json to disk
|
Save global metadata json to disk
|
||||||
"""
|
"""
|
||||||
with self.save_lock:
|
with self.save_lock:
|
||||||
with open(self.global_metadata_path, 'w') as fp:
|
with self.global_metadata_path.open('w') as fp:
|
||||||
rapidjson.dump(metadata, fp, default=self.np_encoder,
|
rapidjson.dump(metadata, fp, default=self.np_encoder,
|
||||||
number_mode=rapidjson.NM_NATIVE)
|
number_mode=rapidjson.NM_NATIVE)
|
||||||
|
|
||||||
@ -424,7 +424,7 @@ class FreqaiDataDrawer:
|
|||||||
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
|
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
|
||||||
dk.data["label_list"] = dk.label_list
|
dk.data["label_list"] = dk.label_list
|
||||||
|
|
||||||
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
|
with (save_path / f"{dk.model_filename}_metadata.json").open("w") as fp:
|
||||||
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
|
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
|
||||||
|
|
||||||
return
|
return
|
||||||
@ -457,7 +457,7 @@ class FreqaiDataDrawer:
|
|||||||
dk.data["training_features_list"] = dk.training_features_list
|
dk.data["training_features_list"] = dk.training_features_list
|
||||||
dk.data["label_list"] = dk.label_list
|
dk.data["label_list"] = dk.label_list
|
||||||
# store the metadata
|
# store the metadata
|
||||||
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
|
with (save_path / f"{dk.model_filename}_metadata.json").open("w") as fp:
|
||||||
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
|
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
|
||||||
|
|
||||||
# save the train data to file so we can check preds for area of applicability later
|
# save the train data to file so we can check preds for area of applicability later
|
||||||
@ -471,7 +471,7 @@ class FreqaiDataDrawer:
|
|||||||
|
|
||||||
if self.freqai_info["feature_parameters"].get("principal_component_analysis"):
|
if self.freqai_info["feature_parameters"].get("principal_component_analysis"):
|
||||||
cloudpickle.dump(
|
cloudpickle.dump(
|
||||||
dk.pca, open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "wb")
|
dk.pca, (dk.data_path / f"{dk.model_filename}_pca_object.pkl").open("wb")
|
||||||
)
|
)
|
||||||
|
|
||||||
self.model_dictionary[coin] = model
|
self.model_dictionary[coin] = model
|
||||||
@ -491,7 +491,7 @@ class FreqaiDataDrawer:
|
|||||||
Load only metadata into datakitchen to increase performance during
|
Load only metadata into datakitchen to increase performance during
|
||||||
presaved backtesting (prediction file loading).
|
presaved backtesting (prediction file loading).
|
||||||
"""
|
"""
|
||||||
with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
|
with (dk.data_path / f"{dk.model_filename}_metadata.json").open("r") as fp:
|
||||||
dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
||||||
dk.training_features_list = dk.data["training_features_list"]
|
dk.training_features_list = dk.data["training_features_list"]
|
||||||
dk.label_list = dk.data["label_list"]
|
dk.label_list = dk.data["label_list"]
|
||||||
@ -514,7 +514,7 @@ class FreqaiDataDrawer:
|
|||||||
dk.data = self.meta_data_dictionary[coin]["meta_data"]
|
dk.data = self.meta_data_dictionary[coin]["meta_data"]
|
||||||
dk.data_dictionary["train_features"] = self.meta_data_dictionary[coin]["train_df"]
|
dk.data_dictionary["train_features"] = self.meta_data_dictionary[coin]["train_df"]
|
||||||
else:
|
else:
|
||||||
with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
|
with (dk.data_path / f"{dk.model_filename}_metadata.json").open("r") as fp:
|
||||||
dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
|
||||||
|
|
||||||
dk.data_dictionary["train_features"] = pd.read_pickle(
|
dk.data_dictionary["train_features"] = pd.read_pickle(
|
||||||
@ -552,7 +552,7 @@ class FreqaiDataDrawer:
|
|||||||
|
|
||||||
if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
|
if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
|
||||||
dk.pca = cloudpickle.load(
|
dk.pca = cloudpickle.load(
|
||||||
open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
|
(dk.data_path / f"{dk.model_filename}_pca_object.pkl").open("rb")
|
||||||
)
|
)
|
||||||
|
|
||||||
return model
|
return model
|
||||||
|
@ -211,7 +211,7 @@ def record_params(config: Dict[str, Any], full_path: Path) -> None:
|
|||||||
"pairs": config.get('exchange', {}).get('pair_whitelist')
|
"pairs": config.get('exchange', {}).get('pair_whitelist')
|
||||||
}
|
}
|
||||||
|
|
||||||
with open(params_record_path, "w") as handle:
|
with params_record_path.open("w") as handle:
|
||||||
rapidjson.dump(
|
rapidjson.dump(
|
||||||
run_params,
|
run_params,
|
||||||
handle,
|
handle,
|
||||||
|
@ -68,5 +68,5 @@ extend-select = [
|
|||||||
# "DTZ", # flake8-datetimez
|
# "DTZ", # flake8-datetimez
|
||||||
# "RSE", # flake8-raise
|
# "RSE", # flake8-raise
|
||||||
# "TCH", # flake8-type-checking
|
# "TCH", # flake8-type-checking
|
||||||
# "PTH", # flake8-use-pathlib
|
"PTH", # flake8-use-pathlib
|
||||||
]
|
]
|
||||||
|
Loading…
Reference in New Issue
Block a user