From 72c34291e322bc9470eda6112c0dc104667cb0e8 Mon Sep 17 00:00:00 2001 From: longyu Date: Wed, 17 Aug 2022 15:09:10 +0200 Subject: [PATCH 1/7] newline --- freqtrade/freqai/data_kitchen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 35f51baed..b49df5a4d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -508,6 +508,7 @@ class FreqaiDataKitchen: self.data_dictionary["train_features"], n_jobs=self.thread_count) avg_mean_dist = pairwise.mean(axis=1).mean() + return avg_mean_dist def use_SVM_to_remove_outliers(self, predict: bool) -> None: From 9c38c27eede56022a71f08a31bd8c880793db225 Mon Sep 17 00:00:00 2001 From: longyu Date: Wed, 17 Aug 2022 15:09:57 +0200 Subject: [PATCH 2/7] ignore sample itself distance for avg_mean_dist computation --- freqtrade/freqai/data_kitchen.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index b49df5a4d..85041515a 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -454,7 +454,6 @@ class FreqaiDataKitchen: logger.info("reduced feature dimension by %s", n_components - n_keep_components) logger.info("explained variance %f", np.sum(pca2.explained_variance_ratio_)) train_components = pca2.transform(self.data_dictionary["train_features"]) - test_components = pca2.transform(self.data_dictionary["test_features"]) self.data_dictionary["train_features"] = pd.DataFrame( data=train_components, @@ -468,6 +467,7 @@ class FreqaiDataKitchen: self.training_features_list = self.data_dictionary["train_features"].columns if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0: + test_components = pca2.transform(self.data_dictionary["test_features"]) self.data_dictionary["test_features"] = pd.DataFrame( data=test_components, columns=["PC" + str(i) for i in range(0, n_keep_components)], @@ -506,7 +506,10 @@ class FreqaiDataKitchen: # logger.info("computing average mean distance for all training points") pairwise = pairwise_distances( self.data_dictionary["train_features"], n_jobs=self.thread_count) - avg_mean_dist = pairwise.mean(axis=1).mean() + # remove the diagonal distances which are itself distances ~0 + np.fill_diagonal(pairwise, np.NaN) + pairwise = pairwise.reshape(-1, 1) + avg_mean_dist = pairwise[~np.isnan(pairwise)].mean() return avg_mean_dist From f70b0bab809c57219a9bd0b7f6b42e86f7773064 Mon Sep 17 00:00:00 2001 From: longyu Date: Wed, 17 Aug 2022 23:49:20 +0200 Subject: [PATCH 3/7] remove line --- freqtrade/freqai/data_kitchen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 85041515a..421b30bf5 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -511,7 +511,6 @@ class FreqaiDataKitchen: pairwise = pairwise.reshape(-1, 1) avg_mean_dist = pairwise[~np.isnan(pairwise)].mean() - return avg_mean_dist def use_SVM_to_remove_outliers(self, predict: bool) -> None: From 1fada53dddc2418d89d88392d0e557974fd8690c Mon Sep 17 00:00:00 2001 From: longyu Date: Thu, 18 Aug 2022 19:40:00 +0200 Subject: [PATCH 4/7] remove newline --- freqtrade/templates/FreqaiExampleStrategy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 5810e7881..ae33a3697 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -174,7 +174,6 @@ class FreqaiExampleStrategy(IStrategy): # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) # .min() # ) - return df def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: From 277245c69dca73b9714e28f2aedf392adf2c874c Mon Sep 17 00:00:00 2001 From: longyu Date: Fri, 19 Aug 2022 12:39:00 +0200 Subject: [PATCH 5/7] remove line --- freqtrade/freqai/data_kitchen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 421b30bf5..af2f65696 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -510,7 +510,6 @@ class FreqaiDataKitchen: np.fill_diagonal(pairwise, np.NaN) pairwise = pairwise.reshape(-1, 1) avg_mean_dist = pairwise[~np.isnan(pairwise)].mean() - return avg_mean_dist def use_SVM_to_remove_outliers(self, predict: bool) -> None: From cfa5b3f12ca9f882b8f70b4fa0951fc46baa30d4 Mon Sep 17 00:00:00 2001 From: longyu Date: Fri, 19 Aug 2022 12:39:08 +0200 Subject: [PATCH 6/7] add new line --- freqtrade/freqai/data_kitchen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index af2f65696..421b30bf5 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -510,6 +510,7 @@ class FreqaiDataKitchen: np.fill_diagonal(pairwise, np.NaN) pairwise = pairwise.reshape(-1, 1) avg_mean_dist = pairwise[~np.isnan(pairwise)].mean() + return avg_mean_dist def use_SVM_to_remove_outliers(self, predict: bool) -> None: From 521381ebf026032ff9d5e2cf7bbeb0a9b430f486 Mon Sep 17 00:00:00 2001 From: longyu Date: Fri, 19 Aug 2022 12:40:03 +0200 Subject: [PATCH 7/7] undo example strategy newline --- freqtrade/templates/FreqaiExampleStrategy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index ae33a3697..5810e7881 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -174,6 +174,7 @@ class FreqaiExampleStrategy(IStrategy): # .rolling(self.freqai_info["feature_parameters"]["label_period_candles"]) # .min() # ) + return df def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: