From 255eb71270991fe480cd642ee5ea2ce69964f8a9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 19:32:22 +0100 Subject: [PATCH 01/11] start tracking the current candle in FreqAI, add robustness to corr_df caching and inference timer, add test for cache corr_df --- freqtrade/freqai/data_drawer.py | 3 +++ freqtrade/freqai/freqai_interface.py | 21 +++++++++++++++++---- tests/freqai/test_freqai_interface.py | 1 + 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 0e9d2e605..db535862d 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -98,6 +98,7 @@ class FreqaiDataDrawer: "model_filename": "", "trained_timestamp": 0, "data_path": "", "extras": {}} self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} + self.current_candle: datetime = datetime.fromtimestamp(637887600) def update_metric_tracker(self, metric: str, value: float, pair: str) -> None: """ @@ -636,6 +637,8 @@ class FreqaiDataDrawer: axis=0, ) + self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date'] + def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None: """ Load pair histories for all whitelist and corr_pairlist pairs. diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index dcf902954..219cee893 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -75,7 +75,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info.get("conv_width", 2) + self.CONV_WIDTH = self.freqai_info.get("conv_width", 1) if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 @@ -93,7 +93,7 @@ class IFreqaiModel(ABC): # get_corr_dataframes is controlling the caching of corr_dataframes # for improved performance. Careful with this boolean. self.get_corr_dataframes: bool = True - + self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() @@ -339,6 +339,7 @@ class IFreqaiModel(ABC): if self.dd.historic_data: self.dd.update_historic_data(strategy, dk) logger.debug(f'Updating historic data on pair {metadata["pair"]}') + self.track_current_candle() if not self.follow_mode: @@ -683,8 +684,8 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 - if self.corr_pairlist: - self.get_corr_dataframes = True + # if self.corr_pairlist: + # self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): @@ -766,6 +767,18 @@ class IFreqaiModel(ABC): return dataframe + def track_current_candle(self): + """ + Checks if the latest candle appended by the datadrawer is + equivalent to the latest candle seen by FreqAI. If not, it + asks to refresh the cached corr_dfs, and resets the pair + counter. + """ + if self.dd.current_candle > self.current_candle: + self.get_corr_dataframes = True + self.pair_it = 0 + self.current_candle = self.dd.current_candle + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index c46f9e815..99eefe31b 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -192,6 +192,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + df = freqai.cache_corr_pairlist_dfs(df, freqai.dk) for i in range(5): df[f'%-constant_{i}'] = i # df.loc[:, f'%-constant_{i}'] = i From 1a38c10fc6f89b7b3d9dcf8be63ddc8105991246 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 19:37:47 +0100 Subject: [PATCH 02/11] remove old code --- freqtrade/freqai/freqai_interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 219cee893..1d191b9de 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -684,8 +684,6 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 - # if self.corr_pairlist: - # self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): From ce927311328afdfe95d2c701279f803695c27e19 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 20:20:35 +0100 Subject: [PATCH 03/11] ensure backwards compatitibility --- freqtrade/freqai/data_kitchen.py | 3 ++- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f0e24dd80..cd457d65d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1156,7 +1156,8 @@ class FreqaiDataKitchen: valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] pair_cols = [col for col in dataframe.columns if any(substr in col for substr in valid_strs)] - pair_cols.insert(0, 'date') + if pair_cols: + pair_cols.insert(0, 'date') corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) return corr_dataframes diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 1d191b9de..c3ca174c6 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -759,7 +759,7 @@ class IFreqaiModel(ABC): "is included in the column names when you are creating features " "in `populate_any_indicators()`.") self.get_corr_dataframes = not bool(self.corr_dataframes) - else: + elif self.corr_dataframes: dataframe = dk.attach_corr_pair_columns( dataframe, self.corr_dataframes, dk.pair) From b3b756ec14453ef28302a3d2a42f602124740ca2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 20:30:04 +0100 Subject: [PATCH 04/11] ensure test pass --- tests/freqai/test_freqai_datadrawer.py | 1 + tests/freqai/test_freqai_interface.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 1d1c44a1e..7ab963507 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -22,6 +22,7 @@ def test_update_historic_data(mocker, freqai_conf): historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) candle_difference = dp_candles - historic_candles + freqai.dk.pair = "ADA/BTC" freqai.dd.update_historic_data(strategy, freqai.dk) updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 99eefe31b..c66c9c3b3 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -338,6 +338,7 @@ def test_follow_mode(mocker, freqai_conf): df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m') + freqai.dk.pair = "ADA/BTC" freqai.start_live(df, metadata, strategy, freqai.dk) assert len(freqai.dk.return_dataframe.index) == 5702 From 6c4bdb8f67cd096f668c3d206e88b0338bfde90e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 18:49:39 +0100 Subject: [PATCH 05/11] remove special characters from feature names --- freqtrade/freqai/data_drawer.py | 1 - freqtrade/freqai/data_kitchen.py | 15 +++++++++++++++ freqtrade/freqai/freqai_interface.py | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index db535862d..dda8ebdbf 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -98,7 +98,6 @@ class FreqaiDataDrawer: "model_filename": "", "trained_timestamp": 0, "data_path": "", "extras": {}} self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} - self.current_candle: datetime = datetime.fromtimestamp(637887600) def update_metric_tracker(self, metric: str, value: float, pair: str) -> None: """ diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index cd457d65d..1979a5d36 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1247,6 +1247,8 @@ class FreqaiDataKitchen: self.get_unique_classes_from_labels(dataframe) + dataframe = self.remove_special_chars_from_feature_names(dataframe) + return dataframe def fit_labels(self) -> None: @@ -1335,3 +1337,16 @@ class FreqaiDataKitchen: f"Could not find backtesting prediction file at {path_to_predictionfile}" ) return file_exists + + def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame: + """ + Remove all special characters from feature strings (:) + :param dataframe: the dataframe that just finished indicator population. (unfiltered) + :return: dataframe with cleaned featrue names + """ + + spec_chars = [':'] + for c in spec_chars: + dataframe.columns = dataframe.columns.str.replace(c, "") + + return dataframe \ No newline at end of file diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index c3ca174c6..1420ce5c7 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -68,6 +68,9 @@ class IFreqaiModel(ABC): if self.save_backtest_models: logger.info('Backtesting module configured to save all models.') self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) + # set current candle to arbitrary historical date + self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) + self.dd.current_candle = self.current_candle self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", []) @@ -93,7 +96,6 @@ class IFreqaiModel(ABC): # get_corr_dataframes is controlling the caching of corr_dataframes # for improved performance. Careful with this boolean. self.get_corr_dataframes: bool = True - self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() From d721b50230a8d49235bd7e6531dc2dc54fa4f18e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 19:13:24 +0100 Subject: [PATCH 06/11] flake8 --- freqtrade/freqai/data_kitchen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1979a5d36..77e825d27 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1349,4 +1349,4 @@ class FreqaiDataKitchen: for c in spec_chars: dataframe.columns = dataframe.columns.str.replace(c, "") - return dataframe \ No newline at end of file + return dataframe From db942321ad7ce714211ce98eb19af2dcfe015aef Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:03:48 +0100 Subject: [PATCH 07/11] fix bug with lightgbm and colons --- freqtrade/freqai/data_kitchen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 77e825d27..6acd916d5 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1153,12 +1153,13 @@ class FreqaiDataKitchen: pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: + pair = pair.replace(':', '') # lightgbm doesnt like colons valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] pair_cols = [col for col in dataframe.columns if any(substr in col for substr in valid_strs)] if pair_cols: pair_cols.insert(0, 'date') - corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) + corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) return corr_dataframes From 6938ed6584337a49dd6882eeb52ac4f37872567a Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:11:26 +0100 Subject: [PATCH 08/11] change default conv_width to 1 --- freqtrade/constants.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 70f60867b..022cbd400 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -542,7 +542,7 @@ CONF_SCHEMA = { "keras": {"type": "boolean", "default": False}, "write_metrics_to_disk": {"type": "boolean", "default": False}, "purge_old_models": {"type": "boolean", "default": True}, - "conv_width": {"type": "integer", "default": 2}, + "conv_width": {"type": "integer", "default": 1}, "train_period_days": {"type": "integer", "default": 0}, "backtest_period_days": {"type": "number", "default": 7}, "identifier": {"type": "string", "default": "example"}, diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 3b03c988b..707fefb7f 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -78,7 +78,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info.get("conv_width", 1) + self.CONV_WIDTH = self.freqai_info['conv_width'] if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 From 05b309caf292336d4ed1e3631fdab6bbed99db5e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:17:48 +0100 Subject: [PATCH 09/11] ensure colon replaced for cached attach --- freqtrade/freqai/data_kitchen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index eff40dbf3..9772a308b 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1179,6 +1179,7 @@ class FreqaiDataKitchen: pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: + pair = pair.replace(':', '') # lightgbm doesnt work with colons if current_pair != pair: dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date') From 90c5bfb4b5add4b60b8dd063554de197500f20f6 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:35:12 +0100 Subject: [PATCH 10/11] add default conv_width --- freqtrade/freqai/freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 707fefb7f..bdb8a2860 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -78,7 +78,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info['conv_width'] + self.CONV_WIDTH = self.freqai_info.get('conv_width', 1) if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 From 53df607067a0fdff38c1df7e00cdd0e81a166530 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 5 Nov 2022 15:42:19 +0100 Subject: [PATCH 11/11] avoid duplicating features with okx/gateio, ensure inference timer gets logged --- freqtrade/freqai/data_kitchen.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9772a308b..fab55ceee 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1177,7 +1177,7 @@ class FreqaiDataKitchen: ready for training """ pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) - + current_pair = current_pair.replace(':', '') for pair in pairs: pair = pair.replace(':', '') # lightgbm doesnt work with colons if current_pair != pair: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index bdb8a2860..dad016cbd 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -776,7 +776,7 @@ class IFreqaiModel(ABC): """ if self.dd.current_candle > self.current_candle: self.get_corr_dataframes = True - self.pair_it = 0 + self.pair_it = 1 self.current_candle = self.dd.current_candle # Following methods which are overridden by user made prediction models.