adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update code to use historic_predictions for freqai_backtest_live_models

Browse Source
This commit is contained in:
Wagner Costa Santos
2022-11-19 14:15:58 -03:00
parent 3d3195847c
commit 80d070e9ee
8 changed files with 86 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
freqtrade/freqai/data_kitchen.py
View File

@@ -9,7 +9,7 @@ from typing import Any, Dict, List, Tuple
import numpy as np
import numpy.typing as npt
import pandas as pd
from pandas import DataFrame, read_feather
from pandas import DataFrame
from scipy import stats
from sklearn import linear_model
from sklearn.cluster import DBSCAN
@@ -74,9 +74,6 @@ class FreqaiDataKitchen:
self.training_features_list: List = []
self.model_filename: str = ""
self.backtesting_results_path = Path()
self.backtesting_live_model_folder_path = Path()
self.backtesting_live_model_path = Path()
self.backtesting_live_model_bkp_path = Path()
self.backtest_predictions_folder: str = "backtesting_predictions"
self.live = live
self.pair = pair
@@ -90,7 +87,9 @@ class FreqaiDataKitchen:
self.full_path = self.get_full_models_path(self.config)
if self.backtest_live_models:
if self.pair:
if self.pair and not (
self.freqai_config.get("backtest_using_historic_predictions", True)
):
self.set_timerange_from_ready_models()
(self.training_timeranges,
self.backtesting_timeranges) = self.split_timerange_live_models()
@@ -1488,101 +1487,30 @@ class FreqaiDataKitchen:
return dataframe
def set_backtesting_live_dataframe_folder_path(
self
) -> None:
"""
Set live backtesting dataframe path
:param pair: current pair
"""
self.backtesting_live_model_folder_path = Path(
self.full_path / self.backtest_predictions_folder / "live_data")
def set_backtesting_live_dataframe_path(
self, pair: str
) -> None:
"""
Set live backtesting dataframe path
:param pair: current pair
"""
self.set_backtesting_live_dataframe_folder_path()
if not self.backtesting_live_model_folder_path.is_dir():
self.backtesting_live_model_folder_path.mkdir(parents=True, exist_ok=True)
pair_path = pair.split(":")[0].replace("/", "_").lower()
file_name = f"live_backtesting_{pair_path}.feather"
self.backtesting_live_model_path = Path(
self.full_path /
self.backtesting_live_model_folder_path /
file_name)
self.backtesting_live_model_bkp_path = Path(
self.full_path /
self.backtesting_live_model_folder_path /
file_name.replace(".feather", ".backup.feather"))
def save_backtesting_live_dataframe(
self, dataframe: DataFrame, pair: str
) -> None:
"""
Save live backtesting dataframe to feather file format
:param dataframe: current live dataframe
:param pair: current pair
"""
self.set_backtesting_live_dataframe_path(pair)
last_row_df = dataframe.tail(1)
if self.backtesting_live_model_path.is_file():
saved_dataframe = self.get_backtesting_live_dataframe()
concat_dataframe = pd.concat([saved_dataframe, last_row_df])
self.save_backtesting_live_dataframe_to_feather(concat_dataframe)
else:
self.save_backtesting_live_dataframe_to_feather(last_row_df)
shutil.copy(self.backtesting_live_model_path, self.backtesting_live_model_bkp_path)
def save_backtesting_live_dataframe_to_feather(self, dataframe: DataFrame):
dataframe.reset_index(drop=True).to_feather(
self.backtesting_live_model_path, compression_level=9, compression='lz4')
def get_backtesting_live_dataframe(
self
) -> DataFrame:
"""
Get live backtesting dataframe from feather file format
return: saved dataframe from previous dry/run or live
"""
if self.backtesting_live_model_path.is_file():
saved_dataframe = DataFrame()
try:
saved_dataframe = read_feather(self.backtesting_live_model_path)
except Exception:
saved_dataframe = read_feather(self.backtesting_live_model_bkp_path)
return saved_dataframe
else:
raise OperationalException(
"Saved live backtesting dataframe file not found."
)
def get_timerange_from_backtesting_live_dataframe(self) -> TimeRange:
"""
Returns timerange information based on live backtesting dataframe file
Returns timerange information based on historic predictions file
:return: timerange calculated from saved live data
"""
all_assets_start_dates = []
all_assets_end_dates = []
self.set_backtesting_live_dataframe_folder_path()
if not self.backtesting_live_model_folder_path.is_dir():
from freqtrade.freqai.data_drawer import FreqaiDataDrawer
dd = FreqaiDataDrawer(Path(self.full_path), self.config)
if not dd.historic_predictions_path.is_file():
raise OperationalException(
'Saved live data not found. Saved lived data is required '
'Historic predictions not found. Historic predictions data is required '
'to run backtest with the freqai-backtest-live-models option '
'and save_live_data_backtest config option as true'
'and backtest_using_historic_predictions config option as true'
)
for file_in_dir in self.backtesting_live_model_folder_path.iterdir():
if file_in_dir.is_file() and "backup" not in file_in_dir.name:
saved_dataframe = read_feather(file_in_dir)
all_assets_start_dates.append(saved_dataframe.date.min())
all_assets_end_dates.append(saved_dataframe.date.max())
start_date = min(all_assets_start_dates)
end_date = max(all_assets_end_dates)
dd.load_historic_predictions_from_disk()
all_pairs_end_dates = []
for pair in dd.historic_predictions:
pair_historic_data = dd.historic_predictions[pair]
all_pairs_end_dates.append(pair_historic_data.date_pred.max())
global_metadata = dd.load_global_metadata_from_disk()
start_date = datetime.fromtimestamp(int(global_metadata["start_dry_live_date"]))
end_date = max(all_pairs_end_dates)
# add 1 day to string timerange to ensure BT module will load all dataframe data
end_date = end_date + timedelta(days=1)
backtesting_timerange = TimeRange(