add PTH ruff selection

This commit is contained in:
Matthias 2023-02-25 17:17:05 +01:00
parent d014e4590e
commit 26315b6bc2
3 changed files with 18 additions and 18 deletions

View File

@ -126,7 +126,7 @@ class FreqaiDataDrawer:
""" """
exists = self.global_metadata_path.is_file() exists = self.global_metadata_path.is_file()
if exists: if exists:
with open(self.global_metadata_path, "r") as fp: with self.global_metadata_path.open("r") as fp:
metatada_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) metatada_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
return metatada_dict return metatada_dict
return {} return {}
@ -139,7 +139,7 @@ class FreqaiDataDrawer:
""" """
exists = self.pair_dictionary_path.is_file() exists = self.pair_dictionary_path.is_file()
if exists: if exists:
with open(self.pair_dictionary_path, "r") as fp: with self.pair_dictionary_path.open("r") as fp:
self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) self.pair_dict = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
else: else:
logger.info("Could not find existing datadrawer, starting from scratch") logger.info("Could not find existing datadrawer, starting from scratch")
@ -152,7 +152,7 @@ class FreqaiDataDrawer:
if self.freqai_info.get('write_metrics_to_disk', False): if self.freqai_info.get('write_metrics_to_disk', False):
exists = self.metric_tracker_path.is_file() exists = self.metric_tracker_path.is_file()
if exists: if exists:
with open(self.metric_tracker_path, "r") as fp: with self.metric_tracker_path.open("r") as fp:
self.metric_tracker = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) self.metric_tracker = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
logger.info("Loading existing metric tracker from disk.") logger.info("Loading existing metric tracker from disk.")
else: else:
@ -166,7 +166,7 @@ class FreqaiDataDrawer:
exists = self.historic_predictions_path.is_file() exists = self.historic_predictions_path.is_file()
if exists: if exists:
try: try:
with open(self.historic_predictions_path, "rb") as fp: with self.historic_predictions_path.open("rb") as fp:
self.historic_predictions = cloudpickle.load(fp) self.historic_predictions = cloudpickle.load(fp)
logger.info( logger.info(
f"Found existing historic predictions at {self.full_path}, but beware " f"Found existing historic predictions at {self.full_path}, but beware "
@ -176,7 +176,7 @@ class FreqaiDataDrawer:
except EOFError: except EOFError:
logger.warning( logger.warning(
'Historical prediction file was corrupted. Trying to load backup file.') 'Historical prediction file was corrupted. Trying to load backup file.')
with open(self.historic_predictions_bkp_path, "rb") as fp: with self.historic_predictions_bkp_path.open("rb") as fp:
self.historic_predictions = cloudpickle.load(fp) self.historic_predictions = cloudpickle.load(fp)
logger.warning('FreqAI successfully loaded the backup historical predictions file.') logger.warning('FreqAI successfully loaded the backup historical predictions file.')
@ -189,7 +189,7 @@ class FreqaiDataDrawer:
""" """
Save historic predictions pickle to disk Save historic predictions pickle to disk
""" """
with open(self.historic_predictions_path, "wb") as fp: with self.historic_predictions_path.open("wb") as fp:
cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL) cloudpickle.dump(self.historic_predictions, fp, protocol=cloudpickle.DEFAULT_PROTOCOL)
# create a backup # create a backup
@ -200,16 +200,16 @@ class FreqaiDataDrawer:
Save metric tracker of all pair metrics collected. Save metric tracker of all pair metrics collected.
""" """
with self.save_lock: with self.save_lock:
with open(self.metric_tracker_path, 'w') as fp: with self.metric_tracker_path.open('w') as fp:
rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder, rapidjson.dump(self.metric_tracker, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE) number_mode=rapidjson.NM_NATIVE)
def save_drawer_to_disk(self): def save_drawer_to_disk(self) -> None:
""" """
Save data drawer full of all pair model metadata in present model folder. Save data drawer full of all pair model metadata in present model folder.
""" """
with self.save_lock: with self.save_lock:
with open(self.pair_dictionary_path, 'w') as fp: with self.pair_dictionary_path.open('w') as fp:
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder, rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE) number_mode=rapidjson.NM_NATIVE)
@ -218,7 +218,7 @@ class FreqaiDataDrawer:
Save global metadata json to disk Save global metadata json to disk
""" """
with self.save_lock: with self.save_lock:
with open(self.global_metadata_path, 'w') as fp: with self.global_metadata_path.open('w') as fp:
rapidjson.dump(metadata, fp, default=self.np_encoder, rapidjson.dump(metadata, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE) number_mode=rapidjson.NM_NATIVE)
@ -424,7 +424,7 @@ class FreqaiDataDrawer:
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns) dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
dk.data["label_list"] = dk.label_list dk.data["label_list"] = dk.label_list
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp: with (save_path / f"{dk.model_filename}_metadata.json").open("w") as fp:
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE) rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
return return
@ -457,7 +457,7 @@ class FreqaiDataDrawer:
dk.data["training_features_list"] = dk.training_features_list dk.data["training_features_list"] = dk.training_features_list
dk.data["label_list"] = dk.label_list dk.data["label_list"] = dk.label_list
# store the metadata # store the metadata
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp: with (save_path / f"{dk.model_filename}_metadata.json").open("w") as fp:
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE) rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
# save the train data to file so we can check preds for area of applicability later # save the train data to file so we can check preds for area of applicability later
@ -471,7 +471,7 @@ class FreqaiDataDrawer:
if self.freqai_info["feature_parameters"].get("principal_component_analysis"): if self.freqai_info["feature_parameters"].get("principal_component_analysis"):
cloudpickle.dump( cloudpickle.dump(
dk.pca, open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "wb") dk.pca, (dk.data_path / f"{dk.model_filename}_pca_object.pkl").open("wb")
) )
self.model_dictionary[coin] = model self.model_dictionary[coin] = model
@ -491,7 +491,7 @@ class FreqaiDataDrawer:
Load only metadata into datakitchen to increase performance during Load only metadata into datakitchen to increase performance during
presaved backtesting (prediction file loading). presaved backtesting (prediction file loading).
""" """
with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp: with (dk.data_path / f"{dk.model_filename}_metadata.json").open("r") as fp:
dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
dk.training_features_list = dk.data["training_features_list"] dk.training_features_list = dk.data["training_features_list"]
dk.label_list = dk.data["label_list"] dk.label_list = dk.data["label_list"]
@ -514,7 +514,7 @@ class FreqaiDataDrawer:
dk.data = self.meta_data_dictionary[coin]["meta_data"] dk.data = self.meta_data_dictionary[coin]["meta_data"]
dk.data_dictionary["train_features"] = self.meta_data_dictionary[coin]["train_df"] dk.data_dictionary["train_features"] = self.meta_data_dictionary[coin]["train_df"]
else: else:
with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp: with (dk.data_path / f"{dk.model_filename}_metadata.json").open("r") as fp:
dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE) dk.data = rapidjson.load(fp, number_mode=rapidjson.NM_NATIVE)
dk.data_dictionary["train_features"] = pd.read_pickle( dk.data_dictionary["train_features"] = pd.read_pickle(
@ -552,7 +552,7 @@ class FreqaiDataDrawer:
if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]: if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
dk.pca = cloudpickle.load( dk.pca = cloudpickle.load(
open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb") (dk.data_path / f"{dk.model_filename}_pca_object.pkl").open("rb")
) )
return model return model

View File

@ -211,7 +211,7 @@ def record_params(config: Dict[str, Any], full_path: Path) -> None:
"pairs": config.get('exchange', {}).get('pair_whitelist') "pairs": config.get('exchange', {}).get('pair_whitelist')
} }
with open(params_record_path, "w") as handle: with params_record_path.open("w") as handle:
rapidjson.dump( rapidjson.dump(
run_params, run_params,
handle, handle,

View File

@ -68,5 +68,5 @@ extend-select = [
# "DTZ", # flake8-datetimez # "DTZ", # flake8-datetimez
# "RSE", # flake8-raise # "RSE", # flake8-raise
# "TCH", # flake8-type-checking # "TCH", # flake8-type-checking
# "PTH", # flake8-use-pathlib "PTH", # flake8-use-pathlib
] ]