From 255eb71270991fe480cd642ee5ea2ce69964f8a9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 19:32:22 +0100 Subject: [PATCH 01/26] start tracking the current candle in FreqAI, add robustness to corr_df caching and inference timer, add test for cache corr_df --- freqtrade/freqai/data_drawer.py | 3 +++ freqtrade/freqai/freqai_interface.py | 21 +++++++++++++++++---- tests/freqai/test_freqai_interface.py | 1 + 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 0e9d2e605..db535862d 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -98,6 +98,7 @@ class FreqaiDataDrawer: "model_filename": "", "trained_timestamp": 0, "data_path": "", "extras": {}} self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} + self.current_candle: datetime = datetime.fromtimestamp(637887600) def update_metric_tracker(self, metric: str, value: float, pair: str) -> None: """ @@ -636,6 +637,8 @@ class FreqaiDataDrawer: axis=0, ) + self.current_candle = history_data[dk.pair][self.config['timeframe']].iloc[-1]['date'] + def load_all_pair_histories(self, timerange: TimeRange, dk: FreqaiDataKitchen) -> None: """ Load pair histories for all whitelist and corr_pairlist pairs. diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index dcf902954..219cee893 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -75,7 +75,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info.get("conv_width", 2) + self.CONV_WIDTH = self.freqai_info.get("conv_width", 1) if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 @@ -93,7 +93,7 @@ class IFreqaiModel(ABC): # get_corr_dataframes is controlling the caching of corr_dataframes # for improved performance. Careful with this boolean. self.get_corr_dataframes: bool = True - + self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() @@ -339,6 +339,7 @@ class IFreqaiModel(ABC): if self.dd.historic_data: self.dd.update_historic_data(strategy, dk) logger.debug(f'Updating historic data on pair {metadata["pair"]}') + self.track_current_candle() if not self.follow_mode: @@ -683,8 +684,8 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 - if self.corr_pairlist: - self.get_corr_dataframes = True + # if self.corr_pairlist: + # self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): @@ -766,6 +767,18 @@ class IFreqaiModel(ABC): return dataframe + def track_current_candle(self): + """ + Checks if the latest candle appended by the datadrawer is + equivalent to the latest candle seen by FreqAI. If not, it + asks to refresh the cached corr_dfs, and resets the pair + counter. + """ + if self.dd.current_candle > self.current_candle: + self.get_corr_dataframes = True + self.pair_it = 0 + self.current_candle = self.dd.current_candle + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index c46f9e815..99eefe31b 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -192,6 +192,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + df = freqai.cache_corr_pairlist_dfs(df, freqai.dk) for i in range(5): df[f'%-constant_{i}'] = i # df.loc[:, f'%-constant_{i}'] = i From 1a38c10fc6f89b7b3d9dcf8be63ddc8105991246 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 19:37:47 +0100 Subject: [PATCH 02/26] remove old code --- freqtrade/freqai/freqai_interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 219cee893..1d191b9de 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -684,8 +684,6 @@ class IFreqaiModel(ABC): " avoid blinding open trades and degrading performance.") self.pair_it = 0 self.inference_time = 0 - # if self.corr_pairlist: - # self.get_corr_dataframes = True return def train_timer(self, do: Literal['start', 'stop'] = 'start', pair: str = ''): From ce927311328afdfe95d2c701279f803695c27e19 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 20:20:35 +0100 Subject: [PATCH 03/26] ensure backwards compatitibility --- freqtrade/freqai/data_kitchen.py | 3 ++- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index f0e24dd80..cd457d65d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1156,7 +1156,8 @@ class FreqaiDataKitchen: valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] pair_cols = [col for col in dataframe.columns if any(substr in col for substr in valid_strs)] - pair_cols.insert(0, 'date') + if pair_cols: + pair_cols.insert(0, 'date') corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) return corr_dataframes diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 1d191b9de..c3ca174c6 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -759,7 +759,7 @@ class IFreqaiModel(ABC): "is included in the column names when you are creating features " "in `populate_any_indicators()`.") self.get_corr_dataframes = not bool(self.corr_dataframes) - else: + elif self.corr_dataframes: dataframe = dk.attach_corr_pair_columns( dataframe, self.corr_dataframes, dk.pair) From b3b756ec14453ef28302a3d2a42f602124740ca2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 2 Nov 2022 20:30:04 +0100 Subject: [PATCH 04/26] ensure test pass --- tests/freqai/test_freqai_datadrawer.py | 1 + tests/freqai/test_freqai_interface.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index 1d1c44a1e..7ab963507 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -22,6 +22,7 @@ def test_update_historic_data(mocker, freqai_conf): historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) dp_candles = len(strategy.dp.get_pair_dataframe("ADA/BTC", "5m")) candle_difference = dp_candles - historic_candles + freqai.dk.pair = "ADA/BTC" freqai.dd.update_historic_data(strategy, freqai.dk) updated_historic_candles = len(freqai.dd.historic_data["ADA/BTC"]["5m"]) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 99eefe31b..c66c9c3b3 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -338,6 +338,7 @@ def test_follow_mode(mocker, freqai_conf): df = strategy.dp.get_pair_dataframe('ADA/BTC', '5m') + freqai.dk.pair = "ADA/BTC" freqai.start_live(df, metadata, strategy, freqai.dk) assert len(freqai.dk.return_dataframe.index) == 5702 From 6c4bdb8f67cd096f668c3d206e88b0338bfde90e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 18:49:39 +0100 Subject: [PATCH 05/26] remove special characters from feature names --- freqtrade/freqai/data_drawer.py | 1 - freqtrade/freqai/data_kitchen.py | 15 +++++++++++++++ freqtrade/freqai/freqai_interface.py | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index db535862d..dda8ebdbf 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -98,7 +98,6 @@ class FreqaiDataDrawer: "model_filename": "", "trained_timestamp": 0, "data_path": "", "extras": {}} self.metric_tracker: Dict[str, Dict[str, Dict[str, list]]] = {} - self.current_candle: datetime = datetime.fromtimestamp(637887600) def update_metric_tracker(self, metric: str, value: float, pair: str) -> None: """ diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index cd457d65d..1979a5d36 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1247,6 +1247,8 @@ class FreqaiDataKitchen: self.get_unique_classes_from_labels(dataframe) + dataframe = self.remove_special_chars_from_feature_names(dataframe) + return dataframe def fit_labels(self) -> None: @@ -1335,3 +1337,16 @@ class FreqaiDataKitchen: f"Could not find backtesting prediction file at {path_to_predictionfile}" ) return file_exists + + def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame: + """ + Remove all special characters from feature strings (:) + :param dataframe: the dataframe that just finished indicator population. (unfiltered) + :return: dataframe with cleaned featrue names + """ + + spec_chars = [':'] + for c in spec_chars: + dataframe.columns = dataframe.columns.str.replace(c, "") + + return dataframe \ No newline at end of file diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index c3ca174c6..1420ce5c7 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -68,6 +68,9 @@ class IFreqaiModel(ABC): if self.save_backtest_models: logger.info('Backtesting module configured to save all models.') self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) + # set current candle to arbitrary historical date + self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) + self.dd.current_candle = self.current_candle self.scanning = False self.ft_params = self.freqai_info["feature_parameters"] self.corr_pairlist: List[str] = self.ft_params.get("include_corr_pairlist", []) @@ -93,7 +96,6 @@ class IFreqaiModel(ABC): # get_corr_dataframes is controlling the caching of corr_dataframes # for improved performance. Careful with this boolean. self.get_corr_dataframes: bool = True - self.current_candle: datetime = datetime.fromtimestamp(637887600, tz=timezone.utc) self._threads: List[threading.Thread] = [] self._stop_event = threading.Event() From d721b50230a8d49235bd7e6531dc2dc54fa4f18e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 19:13:24 +0100 Subject: [PATCH 06/26] flake8 --- freqtrade/freqai/data_kitchen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 1979a5d36..77e825d27 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1349,4 +1349,4 @@ class FreqaiDataKitchen: for c in spec_chars: dataframe.columns = dataframe.columns.str.replace(c, "") - return dataframe \ No newline at end of file + return dataframe From db942321ad7ce714211ce98eb19af2dcfe015aef Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:03:48 +0100 Subject: [PATCH 07/26] fix bug with lightgbm and colons --- freqtrade/freqai/data_kitchen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 77e825d27..6acd916d5 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1153,12 +1153,13 @@ class FreqaiDataKitchen: pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: + pair = pair.replace(':', '') # lightgbm doesnt like colons valid_strs = [f"%-{pair}", f"%{pair}", f"%_{pair}"] pair_cols = [col for col in dataframe.columns if any(substr in col for substr in valid_strs)] if pair_cols: pair_cols.insert(0, 'date') - corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) + corr_dataframes[pair] = dataframe.filter(pair_cols, axis=1) return corr_dataframes From 6938ed6584337a49dd6882eeb52ac4f37872567a Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:11:26 +0100 Subject: [PATCH 08/26] change default conv_width to 1 --- freqtrade/constants.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 70f60867b..022cbd400 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -542,7 +542,7 @@ CONF_SCHEMA = { "keras": {"type": "boolean", "default": False}, "write_metrics_to_disk": {"type": "boolean", "default": False}, "purge_old_models": {"type": "boolean", "default": True}, - "conv_width": {"type": "integer", "default": 2}, + "conv_width": {"type": "integer", "default": 1}, "train_period_days": {"type": "integer", "default": 0}, "backtest_period_days": {"type": "number", "default": 7}, "identifier": {"type": "string", "default": "example"}, diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 3b03c988b..707fefb7f 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -78,7 +78,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info.get("conv_width", 1) + self.CONV_WIDTH = self.freqai_info['conv_width'] if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 From 05b309caf292336d4ed1e3631fdab6bbed99db5e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:17:48 +0100 Subject: [PATCH 09/26] ensure colon replaced for cached attach --- freqtrade/freqai/data_kitchen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index eff40dbf3..9772a308b 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1179,6 +1179,7 @@ class FreqaiDataKitchen: pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) for pair in pairs: + pair = pair.replace(':', '') # lightgbm doesnt work with colons if current_pair != pair: dataframe = dataframe.merge(corr_dataframes[pair], how='left', on='date') From 90c5bfb4b5add4b60b8dd063554de197500f20f6 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Thu, 3 Nov 2022 21:35:12 +0100 Subject: [PATCH 10/26] add default conv_width --- freqtrade/freqai/freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 707fefb7f..bdb8a2860 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -78,7 +78,7 @@ class IFreqaiModel(ABC): if self.keras and self.ft_params.get("DI_threshold", 0): self.ft_params["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") - self.CONV_WIDTH = self.freqai_info['conv_width'] + self.CONV_WIDTH = self.freqai_info.get('conv_width', 1) if self.ft_params.get("inlier_metric_window", 0): self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2 self.pair_it = 0 From d4cfcbda24bf90ebd771f016f9fcb1fe46e2ac22 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Fri, 4 Nov 2022 17:53:15 +0100 Subject: [PATCH 11/26] move write_metrics_to_disk to proper place in param table --- docs/freqai-parameter-table.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 28a15913b..8a240c372 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -18,6 +18,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `fit_live_predictions_candles` | Number of historical candles to use for computing target (label) statistics from prediction data, instead of from the training dataset (more information can be found [here](freqai-configuration.md#creating-a-dynamic-target-threshold)).
**Datatype:** Positive integer. | `follow_mode` | Use a `follower` that will look for models associated with a specific `identifier` and load those for inferencing. A `follower` will **not** train new models.
**Datatype:** Boolean.
Default: `False`. | `continual_learning` | Use the final state of the most recently trained model as starting point for the new model, allowing for incremental learning (more information can be found [here](freqai-running.md#continual-learning)).
**Datatype:** Boolean.
Default: `False`. +| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | | **Feature parameters** | `feature_parameters` | A dictionary containing the parameters used to engineer the feature set. Details and examples are shown [here](freqai-feature-engineering.md).
**Datatype:** Dictionary. | `include_timeframes` | A list of timeframes that all indicators in `populate_any_indicators` will be created for. The list is added as features to the base indicators dataset.
**Datatype:** List of timeframes (strings). @@ -37,7 +38,6 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `noise_standard_deviation` | If set, FreqAI adds noise to the training features with the aim of preventing overfitting. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. `noise_standard_deviation` should be kept relative to the normalized space, i.e., between -1 and 1. In other words, since data in FreqAI is always normalized to be between -1 and 1, `noise_standard_deviation: 0.05` would result in 32% of the data being randomly increased/decreased by more than 2.5% (i.e., the percent of data falling within the first standard deviation).
**Datatype:** Integer.
Default: `0`. | `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset.
**Datatype:** Float.
Default: `30`. | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it.
**Datatype:** Boolean.
Default: `False` (no reversal). -| `write_metrics_to_disk` | Collect train timings, inference timings and cpu usage in json file.
**Datatype:** Boolean.
Default: `False` | | **Data split parameters** | `data_split_parameters` | Include any additional parameters available from Scikit-learn `test_train_split()`, which are shown [here](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html) (external website).
**Datatype:** Dictionary. | `test_size` | The fraction of data that should be used for testing instead of training.
**Datatype:** Positive float < 1. From 6e09d552ac19377acbabe17846f2d60ce3a59b2c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Nov 2022 13:14:35 +0100 Subject: [PATCH 12/26] Properly handle and test ohlcv min_max with empty files --- freqtrade/data/history/idatahandler.py | 5 +++++ tests/data/test_datahandler.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index cbc3f1a34..b82d2055b 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -102,6 +102,11 @@ class IDataHandler(ABC): :return: (min, max) """ data = self._ohlcv_load(pair, timeframe, None, candle_type) + if data.empty: + return ( + datetime.fromtimestamp(0, tz=timezone.utc), + datetime.fromtimestamp(0, tz=timezone.utc) + ) return data.iloc[0]['date'].to_pydatetime(), data.iloc[-1]['date'].to_pydatetime() @abstractmethod diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index 67eeda7d0..c067d0339 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import re +from datetime import datetime, timezone from pathlib import Path from unittest.mock import MagicMock @@ -154,6 +155,23 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog): assert df.columns.equals(df1.columns) +def test_datahandler_ohlcv_data_min_max(testdatadir): + dh = JsonDataHandler(testdatadir) + min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '5m', 'spot') + assert len(min_max) == 2 + + # Empty pair + min_max = dh.ohlcv_data_min_max('UNITTEST/BTC', '8m', 'spot') + assert len(min_max) == 2 + assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) + assert min_max[0] == min_max[1] + # Empty pair2 + min_max = dh.ohlcv_data_min_max('NOPAIR/XXX', '4m', 'spot') + assert len(min_max) == 2 + assert min_max[0] == datetime.fromtimestamp(0, tz=timezone.utc) + assert min_max[0] == min_max[1] + + def test_datahandler__check_empty_df(testdatadir, caplog): dh = JsonDataHandler(testdatadir) expected_text = r"Price jump in UNITTEST/USDT, 1h, spot between" From 53df607067a0fdff38c1df7e00cdd0e81a166530 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 5 Nov 2022 15:42:19 +0100 Subject: [PATCH 13/26] avoid duplicating features with okx/gateio, ensure inference timer gets logged --- freqtrade/freqai/data_kitchen.py | 2 +- freqtrade/freqai/freqai_interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 9772a308b..fab55ceee 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1177,7 +1177,7 @@ class FreqaiDataKitchen: ready for training """ pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", []) - + current_pair = current_pair.replace(':', '') for pair in pairs: pair = pair.replace(':', '') # lightgbm doesnt work with colons if current_pair != pair: diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index bdb8a2860..dad016cbd 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -776,7 +776,7 @@ class IFreqaiModel(ABC): """ if self.dd.current_candle > self.current_candle: self.get_corr_dataframes = True - self.pair_it = 0 + self.pair_it = 1 self.current_candle = self.dd.current_candle # Following methods which are overridden by user made prediction models. From 25b8d34fe2ac1d95dadc1a25e2f5dcf0bdc34aa8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Nov 2022 17:02:18 +0100 Subject: [PATCH 14/26] Update backtesting test Had bad behavior before, and didn't properly test what it was supposed to --- tests/optimize/test_backtesting.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 290e08455..140cc3394 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -788,13 +788,14 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: assert len(t['orders']) == 2 ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate - assert ln is not None + assert not ln.empty assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) # check close trade rate alignes to close rate or is between high and low - ln = data_pair.loc[data_pair["date"] == t["close_date"]] - assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or - round(ln.iloc[0]["low"], 6) < round( - t["close_rate"], 6) < round(ln.iloc[0]["high"], 6)) + ln1 = data_pair.loc[data_pair["date"] == t["close_date"]] + assert not ln1.empty + assert (round(ln1.iloc[0]["open"], 6) == round(t["close_rate"], 6) or + round(ln1.iloc[0]["low"], 6) < round( + t["close_rate"], 6) < round(ln1.iloc[0]["high"], 6)) def test_backtest_timedout_entry_orders(default_conf, fee, mocker, testdatadir) -> None: From 3af177d8af26f9994c9ac586e6401fe601828947 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Nov 2022 08:43:29 +0100 Subject: [PATCH 15/26] bump ccxt to 2.1.33 closes #7700 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cae9cf3b7..9e3e57ec7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.4 pandas==1.5.1 pandas-ta==0.3.14b -ccxt==2.0.96 +ccxt==2.1.33 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1 aiohttp==3.8.3 From 1d2b89bc138ce6d187b054845a8e765c79f86133 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Nov 2022 09:02:15 +0100 Subject: [PATCH 16/26] Improve handling of get_fee to not allow impossible combinations --- freqtrade/exchange/exchange.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7a2b8ce7d..32d4f3435 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1689,6 +1689,17 @@ class Exchange: @retrier def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, price: float = 1, taker_or_maker: MakerTaker = 'maker') -> float: + """ + Retrieve fee from exchange + :param symbol: Pair + :param type: Type of order (market, limit, ...) + :param side: Side of order (buy, sell) + :param amount: Amount of order + :param price: Price of order + :param taker_or_maker: 'maker' or 'taker' (ignored if "type" is provided) + """ + if type and type == 'market': + taker_or_maker = 'taker' try: if self._config['dry_run'] and self._config.get('fee', None) is not None: return self._config['fee'] From d48a9ae96d8d6de32838077a59d805979c0427fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Nov 2022 09:40:44 +0100 Subject: [PATCH 17/26] Add leverage to backtest results closes #7574 --- freqtrade/data/btanalysis.py | 4 +++- tests/optimize/test_backtesting.py | 1 + tests/optimize/test_backtesting_adjust_position.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index c32db9165..9bc543a9d 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'profit_ratio', 'profit_abs', 'exit_reason', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag', - 'is_short', 'open_timestamp', 'close_timestamp', 'orders' + 'leverage', 'is_short', 'open_timestamp', 'close_timestamp', 'orders' ] @@ -280,6 +280,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non # Compatibility support for pre short Columns if 'is_short' not in df.columns: df['is_short'] = 0 + if 'leverage' not in df.columns: + df['leverage'] = 1.0 if 'enter_tag' not in df.columns: df['enter_tag'] = df['buy_tag'] df = df.drop(['buy_tag'], axis=1) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 140cc3394..053ed7e5e 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -764,6 +764,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'max_rate': [0.10501, 0.1038888], 'is_open': [False, False], 'enter_tag': [None, None], + "leverage": [1.0, 1.0], "is_short": [False, False], 'open_timestamp': [1517251200000, 1517283000000], 'close_timestamp': [1517265300000, 1517285400000], diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 135ec6b15..b97b45e26 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -72,6 +72,7 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'max_rate': [0.10481985, 0.1038888], 'is_open': [False, False], 'enter_tag': [None, None], + 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], 'close_timestamp': [1517265300000, 1517285400000], From 3cbbfde6bc6ecf91fc241cbabc96d231418fa5aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:00:40 +0000 Subject: [PATCH 18/26] Bump jsonschema from 4.16.0 to 4.17.0 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.16.0 to 4.17.0. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.16.0...v4.17.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e3e57ec7..ac771d61a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.3 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.12 -jsonschema==4.16.0 +jsonschema==4.17.0 TA-Lib==0.4.25 technical==1.3.0 tabulate==0.9.0 From 01a31a6e0156c4446542a23640d4b0ed672b4faa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:01:08 +0000 Subject: [PATCH 19/26] Bump mkdocs from 1.4.1 to 1.4.2 Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.4.1 to 1.4.2. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.4.1...1.4.2) --- updated-dependencies: - dependency-name: mkdocs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 0e1e80e09..eca05c70d 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ markdown==3.3.7 -mkdocs==1.4.1 +mkdocs==1.4.2 mkdocs-material==8.5.7 mdx_truly_sane_lists==1.3 pymdown-extensions==9.7 From f174b41fd783972e6be2cfb7e5b1474c7d9efe2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:01:30 +0000 Subject: [PATCH 20/26] Bump sqlalchemy from 1.4.42 to 1.4.43 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.42 to 1.4.43. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e3e57ec7..48e545865 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==2.1.33 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1 aiohttp==3.8.3 -SQLAlchemy==1.4.42 +SQLAlchemy==1.4.43 python-telegram-bot==13.14 arrow==1.2.3 cachetools==4.2.2 From 224507dfa089e2beba920637807d7047c100a310 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:01:36 +0000 Subject: [PATCH 21/26] Bump progressbar2 from 4.1.1 to 4.2.0 Bumps [progressbar2](https://github.com/WoLpH/python-progressbar) from 4.1.1 to 4.2.0. - [Release notes](https://github.com/WoLpH/python-progressbar/releases) - [Changelog](https://github.com/wolph/python-progressbar/blob/develop/CHANGES.rst) - [Commits](https://github.com/WoLpH/python-progressbar/compare/v4.1.1...v4.2.0) --- updated-dependencies: - dependency-name: progressbar2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index fd2731256..4f59ad1fa 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,4 +6,4 @@ scipy==1.9.3 scikit-learn==1.1.3 scikit-optimize==0.9.0 filelock==3.8.0 -progressbar2==4.1.1 +progressbar2==4.2.0 From 37e066bd76fbe3b202fc69422d4468ffaa1f88b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:01:47 +0000 Subject: [PATCH 22/26] Bump prompt-toolkit from 3.0.31 to 3.0.32 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.31 to 3.0.32. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.31...3.0.32) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e3e57ec7..2cd4e8e95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ psutil==5.9.3 colorama==0.4.6 # Building config files interactively questionary==1.10.0 -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.32 # Extensions to datetime library python-dateutil==2.8.2 From 0bb57f738d3e98ce4df8aaf0d80481ece53b1fe1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:01:54 +0000 Subject: [PATCH 23/26] Bump pytest from 7.1.3 to 7.2.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.3 to 7.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.3...7.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 66da72969..a46ec7e48 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8==5.0.4 flake8-tidy-imports==4.8.0 mypy==0.982 pre-commit==2.20.0 -pytest==7.1.3 +pytest==7.2.0 pytest-asyncio==0.20.1 pytest-cov==4.0.0 pytest-mock==3.10.0 From d978ff6bfb8bc810e2a4c9779d249c26ea509c26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 03:02:15 +0000 Subject: [PATCH 24/26] Bump orjson from 3.8.0 to 3.8.1 Bumps [orjson](https://github.com/ijl/orjson) from 3.8.0 to 3.8.1. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.0...3.8.1) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e3e57ec7..d10e3ce32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.0 +orjson==3.8.1 # Notify systemd sdnotify==0.3.2 From 850b04357e29baca0fc305268487f5296481013b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 06:05:47 +0000 Subject: [PATCH 25/26] Bump mkdocs-material from 8.5.7 to 8.5.8 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.5.7 to 8.5.8. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.5.7...8.5.8) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index eca05c70d..d7d2e27b7 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.4.2 -mkdocs-material==8.5.7 +mkdocs-material==8.5.8 mdx_truly_sane_lists==1.3 pymdown-extensions==9.7 jinja2==3.1.2 From 5ba012c592e83c31e5cab10e925460befb4269ba Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Nov 2022 07:18:09 +0000 Subject: [PATCH 26/26] Disable "tick" in test_update_funding_fees_schedule we only want to test run frequency, not time progression. --- tests/test_freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index a0d38563e..10e6228f8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -5305,7 +5305,7 @@ def test_get_valid_price(mocker, default_conf_usdt) -> None: ]) def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, time_machine, t1, t2): - time_machine.move_to(f"{t1} +00:00") + time_machine.move_to(f"{t1} +00:00", tick=False) patch_RPCManager(mocker) patch_exchange(mocker) @@ -5314,7 +5314,7 @@ def test_update_funding_fees_schedule(mocker, default_conf, trading_mode, calls, default_conf['margin_mode'] = 'isolated' freqtrade = get_patched_freqtradebot(mocker, default_conf) - time_machine.move_to(f"{t2} +00:00") + time_machine.move_to(f"{t2} +00:00", tick=False) # Check schedule jobs in debugging with freqtrade._schedule.jobs freqtrade._schedule.run_pending()