Merge pull request #7690 from freqtrade/track-current-candle
Track current candle in FreqAI
This commit is contained in:
Robert Caulk
GitHub
commit
820aad670c
@ -542,7 +542,7 @@ CONF_SCHEMA = {
|
||||
"keras": {"type": "boolean", "default": False},
|
||||
"write_metrics_to_disk": {"type": "boolean", "default": False},
|
||||
"purge_old_models": {"type": "boolean", "default": True},
|
||||
"conv_width": {"type": "integer", "default": 2},
|
||||
"conv_width": {"type": "integer", "default": 1},
|
||||
"train_period_days": {"type": "integer", "default": 0},
|
||||
"backtest_period_days": {"type": "number", "default": 7},
|
||||
"identifier": {"type": "string", "default": "example"},
|
||||
|
||||
@ -636,6 +636,8 @@ class FreqaiDataDrawer:
|
||||
axis=0,
|
||||
)
|
||||
|
||||
self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date']
|
||||
|
||||
def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None:
|
||||
"""
|
||||
Load pair histories for all whitelist and corr_pairlist pairs.
|
||||
|
||||
@ -1153,11 +1153,13 @@ class FreqaiDataKitchen:
|
||||
pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
|
||||
|
||||
for pair in pairs:
|
||||
pair = pair.replace(':', '') # lightgbm doesnt like colons
|
||||
valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"]
|
||||
pair_cols = [col for col in dataframe.columns if
|
||||
any(substr in col for substr in valid_strs)]
|
||||
pair_cols.insert(0, 'date')
|
||||
corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
|
||||
if pair_cols:
|
||||
pair_cols.insert(0, 'date')
|
||||
corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1)
|
||||
|
||||
return corr_dataframes
|
||||
|
||||
@ -1175,8 +1177,9 @@ class FreqaiDataKitchen:
|
||||
ready for training
|
||||
"""
|
||||
pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
|
||||
|
||||
current_pair = current_pair.replace(':', '')
|
||||
for pair in pairs:
|
||||
pair = pair.replace(':', '') # lightgbm doesnt work with colons
|
||||
if current_pair != pair:
|
||||
dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date')
|
||||
|
||||
@ -1246,6 +1249,8 @@ class FreqaiDataKitchen:
|
||||
|
||||
self.get_unique_classes_from_labels(dataframe)
|
||||
|
||||
dataframe = self.remove_special_chars_from_feature_names(dataframe)
|
||||
|
||||
return dataframe
|
||||
|
||||
def fit_labels(self) -> None:
|
||||
@ -1344,3 +1349,16 @@ class FreqaiDataKitchen:
|
||||
f"Could not find backtesting prediction file at {path_to_predictionfile}"
|
||||
)
|
||||
return False
|
||||
|
||||
def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Remove all special characters from feature strings (:)
|
||||
:param dataframe: the dataframe that just finished indicator population. (unfiltered)
|
||||
:return: dataframe with cleaned featrue names
|
||||
"""
|
||||
|
||||
spec_chars = [':']
|
||||
for c in spec_chars:
|
||||
dataframe.columns = dataframe.columns.str.replace(c, "")
|
||||
|
||||
return dataframe
|
||||
|
||||
@ -68,6 +68,9 @@ class IFreqaiModel(ABC):
|
||||
if self.save_backtest_models:
|
||||
logger.info('Backtesting module configured to save all models.')
|
||||
self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
|
||||
# set current candle to arbitrary historical date
|
||||
self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc)
|
||||
self.dd.current_candle = self.current_candle
|
||||
self.scanning = False
|
||||
self.ft_params = self.freqai_info["feature_parameters"]
|
||||
self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", [])
|
||||
@ -75,7 +78,7 @@ class IFreqaiModel(ABC):
|
||||
if self.keras and self.ft_params.get("DI_threshold", 0):
|
||||
self.ft_params["DI_threshold"] = 0
|
||||
logger.warning("DI threshold is not configured for Keras models yet. Deactivating.")
|
||||
self.CONV_WIDTH = self.freqai_info.get("conv_width", 2)
|
||||
self.CONV_WIDTH = self.freqai_info.get('conv_width', 1)
|
||||
if self.ft_params.get("inlier_metric_window", 0):
|
||||
self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
|
||||
self.pair_it = 0
|
||||
@ -93,7 +96,6 @@ class IFreqaiModel(ABC):
|
||||
# get_corr_dataframes is controlling the caching of corr_dataframes
|
||||
# for improved performance. Careful with this boolean.
|
||||
self.get_corr_dataframes: bool = True
|
||||
|
||||
self._threads: List[threading.Thread] = []
|
||||
self._stop_event = threading.Event()
|
||||
|
||||
@ -339,6 +341,7 @@ class IFreqaiModel(ABC):
|
||||
if self.dd.historic_data:
|
||||
self.dd.update_historic_data(strategy, dk)
|
||||
logger.debug(f'Updating historic data on pair {metadata["pair"]}')
|
||||
self.track_current_candle()
|
||||
|
||||
if not self.follow_mode:
|
||||
|
||||
@ -683,8 +686,6 @@ class IFreqaiModel(ABC):
|
||||
" avoid blinding open trades and degrading performance.")
|
||||
self.pair_it = 0
|
||||
self.inference_time = 0
|
||||
if self.corr_pairlist:
|
||||
self.get_corr_dataframes = True
|
||||
return
|
||||
|
||||
def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''):
|
||||
@ -760,12 +761,24 @@ class IFreqaiModel(ABC):
|
||||
"is included in the column names when you are creating features "
|
||||
"in `populate_any_indicators()`.")
|
||||
self.get_corr_dataframes = not bool(self.corr_dataframes)
|
||||
else:
|
||||
elif self.corr_dataframes:
|
||||
dataframe = dk.attach_corr_pair_columns(
|
||||
dataframe, self.corr_dataframes, dk.pair)
|
||||
|
||||
return dataframe
|
||||
|
||||
def track_current_candle(self):
|
||||
"""
|
||||
Checks if the latest candle appended by the datadrawer is
|
||||
equivalent to the latest candle seen by FreqAI. If not, it
|
||||
asks to refresh the cached corr_dfs, and resets the pair
|
||||
counter.
|
||||
"""
|
||||
if self.dd.current_candle > self.current_candle:
|
||||
self.get_corr_dataframes = True
|
||||
self.pair_it = 1
|
||||
self.current_candle = self.dd.current_candle
|
||||
|
||||
# Following methods which are overridden by user made prediction models.
|
||||
# See freqai/prediction_models/CatboostPredictionModel.py for an example.
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ def test_update_historic_data(mocker, freqai_conf):
|
||||
historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
|
||||
dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m"))
|
||||
candle_difference = dp_candles - historic_candles
|
||||
freqai.dk.pair = "ADA/BTC"
|
||||
freqai.dd.update_historic_data(strategy, freqai.dk)
|
||||
|
||||
updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"])
|
||||
|
||||
@ -194,6 +194,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog)
|
||||
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk)
|
||||
|
||||
df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC")
|
||||
df = freqai.cache_corr_pairlist_dfs(df, freqai.dk)
|
||||
for i in range(5):
|
||||
df[f'%-constant_{i}'] = i
|
||||
# df.loc[:, f'%-constant_{i}'] = i
|
||||
@ -339,6 +340,7 @@ def test_follow_mode(mocker, freqai_conf):
|
||||
|
||||
df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m')
|
||||
|
||||
freqai.dk.pair = "ADA/BTC"
|
||||
freqai.start_live(df, metadata, strategy, freqai.dk)
|
||||
|
||||
assert len(freqai.dk.return_dataframe.index) == 5702
|
||||
|
||||
Loading…
Reference in New Issue
Block a user