From def71a0afedf18378bebed478decc3bb785efad5 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 5 May 2022 15:35:51 +0200 Subject: [PATCH] auto build full_timerange and self manage training_timerange --- config_examples/config_freqai.example.json | 3 -- freqtrade/constants.py | 2 -- freqtrade/freqai/data_handler.py | 41 ++++++++++++++++++---- freqtrade/freqai/freqai_interface.py | 19 ++-------- 4 files changed, 38 insertions(+), 27 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 47109ff31..5bd4de6c4 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -49,12 +49,10 @@ } ], "freqai": { - "btc_pair": "BTC/USDT", "timeframes": [ "5m", "15m" ], - "full_timerange": "20210601-20210901", "train_period": 30, "backtest_period": 7, "identifier": "example", @@ -74,7 +72,6 @@ "LINK/USDT", "DOT/USDT" ], - "training_timerange": "20211220-20220117", "feature_parameters": { "period": 12, "shift": 1, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d9664cff8..d988a164e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -478,13 +478,11 @@ CONF_SCHEMA = { "type": "object", "properties": { "timeframes": {"type": "list"}, - "full_timerange": {"type": "str"}, "train_period": {"type": "integer", "default": 0}, "backtest_period": {"type": "integer", "default": 7}, "identifier": {"type": "str", "default": "example"}, "base_features": {"type": "list"}, "corr_pairlist": {"type": "list"}, - "training_timerange": {"type": "string", "default": None}, "feature_parameters": { "type": "object", "properties": { diff --git a/freqtrade/freqai/data_handler.py b/freqtrade/freqai/data_handler.py index 94df869a1..e58575970 100644 --- a/freqtrade/freqai/data_handler.py +++ b/freqtrade/freqai/data_handler.py @@ -3,6 +3,7 @@ import datetime import json import logging import pickle as pk +import shutil from pathlib import Path from typing import Any, Dict, List, Tuple @@ -30,15 +31,10 @@ class DataHandler: def __init__(self, config: Dict[str, Any], dataframe: DataFrame): self.full_dataframe = dataframe - (self.training_timeranges, self.backtesting_timeranges) = self.split_timerange( - config["freqai"]["full_timerange"], - config["freqai"]["train_period"], - config["freqai"]["backtest_period"], - ) self.data: Dict[Any, Any] = {} self.data_dictionary: Dict[Any, Any] = {} self.config = config - self.freq_config = config["freqai"] + self.freqai_config = config["freqai"] self.predictions = np.array([]) self.do_predict = np.array([]) self.target_mean = np.array([]) @@ -46,6 +42,16 @@ class DataHandler: self.model_path = Path() self.model_filename = "" + self.full_timerange = self.create_fulltimerange( + self.config["timerange"], self.freqai_config["train_period"] + ) + + (self.training_timeranges, self.backtesting_timeranges) = self.split_timerange( + self.full_timerange, + config["freqai"]["train_period"], + config["freqai"]["backtest_period"], + ) + def save_data(self, model: Any) -> None: """ Saves all data associated with a model for a single sub-train time range @@ -539,6 +545,29 @@ class DataHandler: return + def create_fulltimerange(self, backtest_tr: str, backtest_period: int) -> str: + backtest_timerange = TimeRange.parse_timerange(backtest_tr) + + backtest_timerange.startts = backtest_timerange.startts - backtest_period * SECONDS_IN_DAY + start = datetime.datetime.utcfromtimestamp(backtest_timerange.startts) + stop = datetime.datetime.utcfromtimestamp(backtest_timerange.stopts) + full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d") + + self.full_path = Path( + self.config["user_data_dir"] + / "models" + / str(full_timerange + self.freqai_config["identifier"]) + ) + + if not self.full_path.is_dir(): + self.full_path.mkdir(parents=True, exist_ok=True) + shutil.copy( + Path(self.config["config_files"][0]).name, + Path(self.full_path / self.config["config_files"][0]), + ) + + return full_timerange + def np_encoder(self, object): if isinstance(object, np.generic): return object.item() diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 368ed1635..62779a4e1 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -1,6 +1,5 @@ import gc import logging -import shutil from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Dict, Tuple @@ -32,24 +31,13 @@ class IFreqaiModel(ABC): self.data_split_parameters = config["freqai"]["data_split_parameters"] self.model_training_parameters = config["freqai"]["model_training_parameters"] self.feature_parameters = config["freqai"]["feature_parameters"] - self.full_path = Path( - config["user_data_dir"] - / "models" - / str(self.freqai_info["full_timerange"] + self.freqai_info["identifier"]) - ) + self.backtest_timerange = config["timerange"] self.time_last_trained = None self.current_time = None self.model = None self.predictions = None - if not self.full_path.is_dir(): - self.full_path.mkdir(parents=True, exist_ok=True) - shutil.copy( - self.config["config_files"][0], - Path(self.full_path / self.config["config_files"][0]), - ) - def start(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Entry point to the FreqaiModel, it will train a new model if @@ -82,12 +70,11 @@ class IFreqaiModel(ABC): gc.collect() # self.config['timerange'] = tr_train self.dh.data = {} # clean the pair specific data between models - self.freqai_info["training_timerange"] = tr_train + self.training_timerange = tr_train dataframe_train = self.dh.slice_dataframe(tr_train, dataframe) dataframe_backtest = self.dh.slice_dataframe(tr_backtest, dataframe) logger.info("training %s for %s", self.pair, tr_train) - # self.dh.model_path = self.full_path + "/" + "sub-train" + "-" + str(tr_train) + "/" - self.dh.model_path = Path(self.full_path / str("sub-train" + "-" + str(tr_train))) + self.dh.model_path = Path(self.dh.full_path / str("sub-train" + "-" + str(tr_train))) if not self.model_exists(self.pair, training_timerange=tr_train): self.model = self.train(dataframe_train, metadata) self.dh.save_data(self.model)