From 60eb02bb62f5952695a69d9eef531d33d91727bf Mon Sep 17 00:00:00 2001 From: Emre Date: Sat, 10 Sep 2022 20:13:16 +0300 Subject: [PATCH 01/37] Add XGBoostClassifier --- .../prediction_models/XGBoostClassifier.py | 85 +++++++++++++++++++ tests/freqai/test_freqai_interface.py | 30 +++++++ 2 files changed, 115 insertions(+) create mode 100644 freqtrade/freqai/prediction_models/XGBoostClassifier.py diff --git a/freqtrade/freqai/prediction_models/XGBoostClassifier.py b/freqtrade/freqai/prediction_models/XGBoostClassifier.py new file mode 100644 index 000000000..8bf5d6281 --- /dev/null +++ b/freqtrade/freqai/prediction_models/XGBoostClassifier.py @@ -0,0 +1,85 @@ +import logging +from typing import Any, Dict, Tuple + +import numpy as np +import numpy.typing as npt +import pandas as pd +from pandas import DataFrame +from pandas.api.types import is_integer_dtype +from sklearn.preprocessing import LabelEncoder +from xgboost import XGBClassifier + +from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen + + +logger = logging.getLogger(__name__) + + +class XGBoostClassifier(BaseClassifierModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any: + """ + User sets up the training and test data to fit their desired model here + :params: + :data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + X = data_dictionary["train_features"].to_numpy() + y = data_dictionary["train_labels"].to_numpy()[:, 0] + + le = LabelEncoder() + if not is_integer_dtype(y): + y = pd.Series(le.fit_transform(y), dtype="int64") + + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: + eval_set = None + else: + test_features = data_dictionary["test_features"].to_numpy() + test_labels = data_dictionary["test_labels"].to_numpy()[:, 0] + + if not is_integer_dtype(test_labels): + test_labels = pd.Series(le.transform(test_labels), dtype="int64") + + eval_set = [(test_features, test_labels)] + + train_weights = data_dictionary["train_weights"] + + init_model = self.get_init_model(dk.pair) + + model = XGBClassifier(**self.model_training_parameters) + + model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights, + xgb_model=init_model) + + return model + + def predict( + self, unfiltered_df: DataFrame, dk: FreqaiDataKitchen, **kwargs + ) -> Tuple[DataFrame, npt.NDArray[np.int_]]: + """ + Filter the prediction features data and predict with it. + :param: unfiltered_df: Full dataframe for the current backtest period. + :return: + :pred_df: dataframe containing the predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (PCA and DI index) + """ + + (pred_df, dk.do_predict) = super().predict(unfiltered_df, dk, **kwargs) + + le = LabelEncoder() + label = dk.label_list[0] + labels_before = list(dk.data['labels_std'].keys()) + labels_after = le.fit_transform(labels_before).tolist() + pred_df[label] = le.inverse_transform(pred_df[label]) + pred_df = pred_df.rename( + columns={labels_after[i]: labels_before[i] for i in range(len(labels_before))}) + + return (pred_df, dk.do_predict) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 5f8eeb086..afcc4fd37 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -174,6 +174,36 @@ def test_extract_data_and_train_model_LightGBMClassifier(mocker, freqai_conf): shutil.rmtree(Path(freqai.dk.full_path)) +def test_extract_data_and_train_model_XGBoostClassifier(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"freqaimodel": "XGBoostClassifier"}) + freqai_conf.update({"strategy": "freqai_test_classifier"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.extract_data_and_train_model(new_timerange, "ADA/BTC", + strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists() + + def test_extract_data_and_train_model_XGBoostRegressor(mocker, freqai_conf): freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"freqaimodel": "XGBoostRegressor"}) From d598f4334efb965605f7af0918d358b9d89ec008 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 11 Sep 2022 13:28:14 +0200 Subject: [PATCH 02/37] add search share button to website --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 2e5e0c8c9..257db7867 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -49,6 +49,8 @@ theme: logo: "images/logo.png" favicon: "images/logo.png" custom_dir: "docs/overrides" + features: + - search.share palette: - scheme: default primary: "blue grey" From 4476b5a7f4ab0183197a8f0479b9e3ef0d0e7937 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 08:58:40 +0200 Subject: [PATCH 03/37] add user_data arg to test-pairlist --- docs/utils.md | 4 +++- freqtrade/commands/arguments.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 5646365e4..174fa0527 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -525,12 +525,14 @@ Requires a configuration with specified `pairlists` attribute. Can be used to generate static pairlists to be used during backtesting / hyperopt. ``` -usage: freqtrade test-pairlist [-h] [-v] [-c PATH] +usage: freqtrade test-pairlist [-h] [--userdir PATH] [-v] [-c PATH] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-1] [--print-json] [--exchange EXCHANGE] optional arguments: -h, --help show this help message and exit + --userdir PATH, --user-data-dir PATH + Path to userdata directory. -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). -c PATH, --config PATH Specify configuration file (default: diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 37ce17f21..2835f8582 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -53,8 +53,8 @@ ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one "print_csv", "base_currencies", "quote_currencies", "list_pairs_all", "trading_mode"] -ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column", - "list_pairs_print_json", "exchange"] +ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies", + "print_one_column", "list_pairs_print_json", "exchange"] ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] From ccc70a21f274a55b1f68b15a6e9ffbfb39844a5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 09:02:06 +0200 Subject: [PATCH 04/37] Update pairs_file cli argument description --- docs/data-download.md | 10 +++++----- freqtrade/commands/cli_options.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index b72e7f337..2b76d4f74 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -25,8 +25,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--include-inactive-pairs] [--timerange TIMERANGE] [--dl-trades] [--exchange EXCHANGE] - [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] - [--erase] + [-t TIMEFRAMES [TIMEFRAMES ...]] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] [--trading-mode {spot,margin,futures}] @@ -37,7 +36,8 @@ optional arguments: -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Limit command to these pairs. Pairs are space- separated. - --pairs-file FILE File containing a list of pairs to download. + --pairs-file FILE File containing a list of pairs. Takes precedence over + --pairs or pairs configured in the configuration. --days INT Download data for given number of days. --new-pairs-days INT Download data of new pairs for given number of days. Default: `None`. @@ -50,7 +50,7 @@ optional arguments: as --timeframes/-t. --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. - -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] + -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...] Specify which tickers to download. Space-separated list. Default: `1m 5m`. --erase Clean all existing data for the selected @@ -61,7 +61,7 @@ optional arguments: --data-format-trades {json,jsongz,hdf5} Storage format for downloaded trades data. (default: `jsongz`). - --trading-mode {spot,margin,futures} + --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures} Select Trading mode --prepend Allow data prepending. (Data-appending is disabled) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 4240a8014..9aacbcc97 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -393,7 +393,8 @@ AVAILABLE_CLI_OPTIONS = { # Download data "pairs_file": Arg( '--pairs-file', - help='File containing a list of pairs to download.', + help='File containing a list of pairs. ' + 'Takes precedence over --pairs or pairs configured in the configuration.', metavar='FILE', ), "days": Arg( From 9c8c7a03a160cdaad21b85d472879e5d5649d269 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:24:01 +0200 Subject: [PATCH 05/37] Improve typehint --- freqtrade/data/dataprovider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 4151b7419..e639b3ae7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -86,7 +86,7 @@ class DataProvider: """ _candle_type = CandleType.from_string( candle_type) if candle_type != '' else self._config['candle_type_def'] - saved_pair = (pair, str(timeframe), _candle_type) + saved_pair: PairWithTimeframe = (pair, str(timeframe), _candle_type) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) From a48923c0e43eda833aced9afc8a13e98caa29c8f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:38:36 +0200 Subject: [PATCH 06/37] Extract widget colorization to separate function --- freqtrade/optimize/hyperopt.py | 46 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3becf857f..749e608d3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -410,7 +410,7 @@ class Hyperopt: model_queue_size=SKOPT_MODEL_QUEUE_SIZE, ) - def run_optimizer_parallel(self, parallel, asked, i) -> List: + def run_optimizer_parallel(self, parallel: Parallel, asked: List[List], i: int) -> List: return parallel(delayed( wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @@ -491,6 +491,29 @@ class Hyperopt: else: return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] + def get_progressbar_widgets(self): + if self.print_colorized: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='\N{FULL BLOCK}', + fill_wrap=Fore.GREEN + '{}' + Fore.RESET, + marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + else: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='\N{FULL BLOCK}', + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + return widgets + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state')) logger.info(f"Using optimizer random state: {self.random_state}") @@ -526,26 +549,7 @@ class Hyperopt: logger.info(f'Effective number of parallel workers used: {jobs}') # Define progressbar - if self.print_colorized: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - fill_wrap=Fore.GREEN + '{}' + Fore.RESET, - marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] - else: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] + widgets = self.get_progressbar_widgets() with progressbar.ProgressBar( max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, widgets=widgets From 32e13d65c3bbba091eac4ae16259adaaf483a02a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:54:31 +0200 Subject: [PATCH 07/37] Refactor hyperopt to extract evaluate_result --- freqtrade/optimize/hyperopt.py | 49 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 749e608d3..3a117f356 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -290,7 +290,7 @@ class Hyperopt: # noinspection PyProtectedMember attr.value = params_dict[attr_name] - def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: + def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict[str, Any]: """ Used Optimize function. Called once per epoch to optimize whatever is configured. @@ -410,7 +410,9 @@ class Hyperopt: model_queue_size=SKOPT_MODEL_QUEUE_SIZE, ) - def run_optimizer_parallel(self, parallel: Parallel, asked: List[List], i: int) -> List: + def run_optimizer_parallel( + self, parallel: Parallel, asked: List[List], i: int) -> List[Dict[str, Any]]: + """ Start optimizer in a parallel way """ return parallel(delayed( wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @@ -514,6 +516,30 @@ class Hyperopt: ] return widgets + def evaluate_result(self, val: Dict[str, Any], current: int, is_random: bool): + """ + Evaluate results returned from generate_optimizer + """ + val['current_epoch'] = current + val['is_initial_point'] = current <= INITIAL_POINTS + + logger.debug("Optimizer epoch evaluated: %s", val) + + is_best = HyperoptTools.is_best_loss(val, self.current_best_loss) + # This value is assigned here and not in the optimization method + # to keep proper order in the list of results. That's because + # evaluations can take different time. Here they are aligned in the + # order they will be shown to the user. + val['is_best'] = is_best + val['is_random'] = is_random + self.print_results(val) + + if is_best: + self.current_best_loss = val['loss'] + self.current_best_epoch = val + + self._save_result(val) + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state')) logger.info(f"Using optimizer random state: {self.random_state}") @@ -569,25 +595,8 @@ class Hyperopt: for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 - val['current_epoch'] = current - val['is_initial_point'] = current <= INITIAL_POINTS - logger.debug(f"Optimizer epoch evaluated: {val}") - - is_best = HyperoptTools.is_best_loss(val, self.current_best_loss) - # This value is assigned here and not in the optimization method - # to keep proper order in the list of results. That's because - # evaluations can take different time. Here they are aligned in the - # order they will be shown to the user. - val['is_best'] = is_best - val['is_random'] = is_random[j] - self.print_results(val) - - if is_best: - self.current_best_loss = val['loss'] - self.current_best_epoch = val - - self._save_result(val) + self.evaluate_result(val, current, is_random[j]) pbar.update(current) From 78cd46ecd55e1042ef5f20f10ebd7e0c269d6ca0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:56:17 +0200 Subject: [PATCH 08/37] hyperopt Remove unnecessary arguments --- freqtrade/optimize/hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3a117f356..76fc84609 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -290,7 +290,7 @@ class Hyperopt: # noinspection PyProtectedMember attr.value = params_dict[attr_name] - def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict[str, Any]: + def generate_optimizer(self, raw_params: List[Any]) -> Dict[str, Any]: """ Used Optimize function. Called once per epoch to optimize whatever is configured. @@ -411,10 +411,10 @@ class Hyperopt: ) def run_optimizer_parallel( - self, parallel: Parallel, asked: List[List], i: int) -> List[Dict[str, Any]]: + self, parallel: Parallel, asked: List[List]) -> List[Dict[str, Any]]: """ Start optimizer in a parallel way """ return parallel(delayed( - wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) + wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) @@ -588,7 +588,7 @@ class Hyperopt: current_jobs = jobs - n_rest if n_rest > 0 else jobs asked, is_random = self.get_asked_points(n_points=current_jobs) - f_val = self.run_optimizer_parallel(parallel, asked, i) + f_val = self.run_optimizer_parallel(parallel, asked) self.opt.tell(asked, [v['loss'] for v in f_val]) # Calculate progressbar outputs From 72d197a99d0570ae9d3607042d7ec5a533907c8e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 15:42:27 +0200 Subject: [PATCH 09/37] Run first epoch in non-parallel mode this allows dataprovider to load it's cache. closes #7384 --- freqtrade/data/dataprovider.py | 4 +++- freqtrade/optimize/hyperopt.py | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index e639b3ae7..c6519d2b8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -196,7 +196,9 @@ class DataProvider: Clear pair dataframe cache. """ self.__cached_pairs = {} - self.__cached_pairs_backtesting = {} + # Don't reset backtesting pairs - + # otherwise they're reloaded each time during hyperopt due to with analyze_per_epoch + # self.__cached_pairs_backtesting = {} self.__slice_index = 0 # Exchange functions diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 76fc84609..b0119368f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -580,11 +580,24 @@ class Hyperopt: max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, widgets=widgets ) as pbar: - EVALS = ceil(self.total_epochs / jobs) - for i in range(EVALS): + start = 0 + + if self.analyze_per_epoch: + # First analysis not in parallel mode when using --analyze-per-epoch. + # This allows dataprovider to load it's informative cache. + asked, is_random = self.get_asked_points(n_points=1) + # print(asked) + f_val = self.generate_optimizer(asked[0]) + self.opt.tell(asked, [f_val['loss']]) + self.evaluate_result(f_val, 1, is_random[0]) + pbar.update(1) + start += 1 + + evals = ceil((self.total_epochs - start) / jobs) + for i in range(evals): # Correct the number of epochs to be processed for the last # iteration (should not exceed self.total_epochs in total) - n_rest = (i + 1) * jobs - self.total_epochs + n_rest = (i + 1) * jobs - (self.total_epochs - start) current_jobs = jobs - n_rest if n_rest > 0 else jobs asked, is_random = self.get_asked_points(n_points=current_jobs) @@ -594,7 +607,7 @@ class Hyperopt: # Calculate progressbar outputs for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) - current = i * jobs + j + 1 + current = i * jobs + j + 1 + start self.evaluate_result(val, current, is_random[j]) From 816c1f760373f6fc55710cb2e3f09ae39eb14fc5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 17:51:30 +0200 Subject: [PATCH 10/37] add test for per epoch hyperopt --- tests/optimize/test_hyperopt.py | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 0f615b7a3..eaea8aee7 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -922,6 +922,45 @@ def test_in_strategy_auto_hyperopt_with_parallel(mocker, hyperopt_conf, tmpdir, hyperopt.start() +def test_in_strategy_auto_hyperopt_per_epoch(mocker, hyperopt_conf, tmpdir, fee) -> None: + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) + + hyperopt_conf.update({ + 'strategy': 'HyperoptableStrategy', + 'user_data_dir': Path(tmpdir), + 'hyperopt_random_state': 42, + 'spaces': ['all'], + 'epochs': 3, + 'analyze_per_epoch': True, + }) + go = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.generate_optimizer', + return_value={ + 'loss': 0.05, + 'results_explanation': 'foo result', 'params': {}, + 'results_metrics': generate_result_metrics(), + }) + hyperopt = Hyperopt(hyperopt_conf) + hyperopt.backtesting.exchange.get_max_leverage = MagicMock(return_value=1.0) + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter) + assert hyperopt.backtesting.strategy.bot_loop_started is True + + assert hyperopt.backtesting.strategy.buy_rsi.in_space is True + assert hyperopt.backtesting.strategy.buy_rsi.value == 35 + assert hyperopt.backtesting.strategy.sell_rsi.value == 74 + assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value == 30 + buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range + assert isinstance(buy_rsi_range, range) + # Range from 0 - 50 (inclusive) + assert len(list(buy_rsi_range)) == 51 + + hyperopt.start() + # backtesting should be called 3 times (once per epoch) + assert go.call_count == 3 + + def test_SKDecimal(): space = SKDecimal(1, 2, decimals=2) assert 1.5 in space From 982c0315fa2ba909362173c2e892aa7bca2c836b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 19:31:11 +0200 Subject: [PATCH 11/37] Rename variable --- freqtrade/optimize/hyperopt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b0119368f..f15e0b7d8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -586,10 +586,9 @@ class Hyperopt: # First analysis not in parallel mode when using --analyze-per-epoch. # This allows dataprovider to load it's informative cache. asked, is_random = self.get_asked_points(n_points=1) - # print(asked) - f_val = self.generate_optimizer(asked[0]) - self.opt.tell(asked, [f_val['loss']]) - self.evaluate_result(f_val, 1, is_random[0]) + f_val0 = self.generate_optimizer(asked[0]) + self.opt.tell(asked, [f_val0['loss']]) + self.evaluate_result(f_val0, 1, is_random[0]) pbar.update(1) start += 1 From 6777d43aea7f169db1711999b4a15c3fedf21a5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:01:27 +0000 Subject: [PATCH 12/37] Bump sqlalchemy from 1.4.40 to 1.4.41 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.40 to 1.4.41. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdbe6ac28..d063410d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.93.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 -SQLAlchemy==1.4.40 +SQLAlchemy==1.4.41 python-telegram-bot==13.14 arrow==1.2.3 cachetools==4.2.2 From 6bfe99606179ebb1cbb236b8751da180ccd1b3ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:01:56 +0000 Subject: [PATCH 13/37] Bump mkdocs-material from 8.4.2 to 8.4.3 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.4.2 to 8.4.3. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.4.2...8.4.3) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 5a6c46471..d63e79004 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.3.1 -mkdocs-material==8.4.2 +mkdocs-material==8.4.3 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 From 1ef334411ed5c9ba0204acb41172089bfb5c38dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:02:01 +0000 Subject: [PATCH 14/37] Bump types-requests from 2.28.9 to 2.28.10 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.9 to 2.28.10. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 40ca4e154..69749f843 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -25,6 +25,6 @@ nbconvert==7.0.0 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.9 +types-requests==2.28.10 types-tabulate==0.8.11 types-python-dateutil==2.8.19 From a4b7e0a714c0f9624d693772467080c43012649e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:02:06 +0000 Subject: [PATCH 15/37] Bump fastapi from 0.82.0 to 0.83.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.82.0 to 0.83.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.82.0...0.83.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdbe6ac28..ea8cb39ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.8.0 sdnotify==0.3.2 # API Server -fastapi==0.82.0 +fastapi==0.83.0 uvicorn==0.18.3 pyjwt==2.4.0 aiofiles==0.8.0 From 6968fc333b584355c90cf9235ee743a23a239e9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:02:10 +0000 Subject: [PATCH 16/37] Bump jsonschema from 4.15.0 to 4.16.0 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.15.0 to 4.16.0. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.15.0...v4.16.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdbe6ac28..8a3701bb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.3 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.12 -jsonschema==4.15.0 +jsonschema==4.16.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 From c149c47afbc20b083c3f6077edcb6fdc1c223801 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 05:05:30 +0000 Subject: [PATCH 17/37] Bump ccxt from 1.93.3 to 1.93.35 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.93.3 to 1.93.35. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.93.3...1.93.35) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d063410d4..af9048c87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.2 pandas==1.4.4 pandas-ta==0.3.14b -ccxt==1.93.3 +ccxt==1.93.35 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From 4ac804f795227163d93c03438cddbd8d722fdd9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 06:05:04 +0000 Subject: [PATCH 18/37] Bump numpy from 1.23.2 to 1.23.3 Bumps [numpy](https://github.com/numpy/numpy) from 1.23.2 to 1.23.3. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.23.2...v1.23.3) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9bb8e6af0..551440cc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.23.2 +numpy==1.23.3 pandas==1.4.4 pandas-ta==0.3.14b From 00db473f102fb4c6cee1199930e9fe81c3f7792c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 06:05:22 +0000 Subject: [PATCH 19/37] Bump cryptography from 37.0.4 to 38.0.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 37.0.4 to 38.0.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/37.0.4...38.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9bb8e6af0..59bef6b11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b ccxt==1.93.35 # Pin cryptography for now due to rust build errors with piwheels -cryptography==37.0.4 +cryptography==38.0.1 aiohttp==3.8.1 SQLAlchemy==1.4.41 python-telegram-bot==13.14 From 7ee92db7a2c2f25b48917d285f11ab8f6d474418 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 06:07:40 +0000 Subject: [PATCH 20/37] Bump aiofiles from 0.8.0 to 22.1.0 Bumps [aiofiles](https://github.com/Tinche/aiofiles) from 0.8.0 to 22.1.0. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.8.0...v22.1.0) --- updated-dependencies: - dependency-name: aiofiles dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 345c336f5..e263b7914 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ sdnotify==0.3.2 fastapi==0.83.0 uvicorn==0.18.3 pyjwt==2.4.0 -aiofiles==0.8.0 +aiofiles==22.1.0 psutil==5.9.2 # Support for colorized terminal output From f45824acf536ccd7f56d78f0ea692a414709962c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Sep 2022 09:23:02 +0200 Subject: [PATCH 21/37] Bump precommit types/requests --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86c4ec1ad..6f2b3ba3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.9 + - types-requests==2.28.10 - types-tabulate==0.8.11 - types-python-dateutil==2.8.19 # stages: [push] From 7b6e465d57e392114d8cca2df24c18c5d4c3e4fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Sep 2022 21:28:11 +0200 Subject: [PATCH 22/37] Remove gate live test skip --- tests/exchange/test_ccxt_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index f57b0b366..82be6196a 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -297,7 +297,7 @@ class TestCCXTExchange(): def test_ccxt__async_get_candle_history(self, exchange): exchange, exchangename = exchange # For some weired reason, this test returns random lengths for bittrex. - if not exchange._ft_has['ohlcv_has_history'] or exchangename in ('bittrex', 'gateio'): + if not exchange._ft_has['ohlcv_has_history'] or exchangename in ('bittrex'): return pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] From 37dd22c89e2e50bce49ffce17277dc06d2abec8b Mon Sep 17 00:00:00 2001 From: initrv Date: Wed, 14 Sep 2022 03:40:13 +0300 Subject: [PATCH 23/37] Fixed a bug that prevents clearing old models Corrects the error of clearing old models when the model directory contains directories with names that do not match a regular expression --- freqtrade/freqai/data_drawer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 9eeabef8f..1c091f1be 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -355,7 +355,7 @@ class FreqaiDataDrawer: for dir in model_folders: result = pattern.match(str(dir.name)) if result is None: - break + continue coin = result.group(1) timestamp = result.group(2) From 49800e4cc300eb39b412287a05480a98962c354b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Sep 2022 06:55:05 +0200 Subject: [PATCH 24/37] pin ci python to 3.10.6 for now --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5bc209e..91d53044d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: os: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10.6"] steps: - uses: actions/checkout@v3 @@ -121,7 +121,7 @@ jobs: strategy: matrix: os: [ macos-latest ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10.6"] steps: - uses: actions/checkout@v3 @@ -205,7 +205,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10.6"] steps: - uses: actions/checkout@v3 From 91bc3d11618c3f5bf47a8a71166544cf4e0b7a0f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Sep 2022 07:04:14 +0200 Subject: [PATCH 25/37] Update docs aroudn use_exit_signal close #7413 --- docs/strategy-customization.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 2b6e1fb2f..b97bd6d23 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -264,7 +264,8 @@ def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFram ### Exit signal rules Edit the method `populate_exit_trend()` into your strategy file to update your exit strategy. -Please note that the exit-signal is only used if `use_exit_signal` is set to true in the configuration. +The exit-signal is only used for exits if `use_exit_signal` is set to true in the configuration. +`use_exit_signal` will not influence [signal collision rules](#colliding-signals) - which will still apply and can prevent entries. It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. From e5368f5a149bf0e330dfac53fdd0d0e44679b469 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Sep 2022 13:24:20 +0000 Subject: [PATCH 26/37] backtesting confirm_trade_entry should pass correct amount, not stake-amount closes #7423 --- freqtrade/optimize/backtesting.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 97418b72c..105851e60 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -812,14 +812,6 @@ class Backtesting: return trade time_in_force = self.strategy.order_time_in_force['entry'] - if not pos_adjust: - # Confirm trade entry: - if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=stake_amount, rate=propose_rate, - time_in_force=time_in_force, current_time=current_time, - entry_tag=entry_tag, side=direction): - return trade - if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): self.order_id_counter += 1 base_currency = self.exchange.get_pair_base_currency(pair) @@ -834,6 +826,15 @@ class Backtesting: # Backcalculate actual stake amount. stake_amount = amount * propose_rate / leverage + if not pos_adjust: + # Confirm trade entry: + if not strategy_safe_wrapper( + self.strategy.confirm_trade_entry, default_retval=True)( + pair=pair, order_type=order_type, amount=amount, rate=propose_rate, + time_in_force=time_in_force, current_time=current_time, + entry_tag=entry_tag, side=direction): + return trade + is_short = (direction == 'short') # Necessary for Margin trading. Disabled until support is enabled. # interest_rate = self.exchange.get_interest_rate() From 0aada271cafa282494b29318ca32f7e7b4a7309a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:11:04 +0200 Subject: [PATCH 27/37] Move informative_pairs for freqAI to backend --- freqtrade/strategy/interface.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9401ebebe..93988ac48 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -613,6 +613,22 @@ class IStrategy(ABC, HyperStrategyMixin): # END - Intended to be overridden by strategy ### + def __informative_pairs_freqai(self) -> ListPairsWithTimeframes: + """ + Create informative-pairs needed for FreqAI + """ + if self.config.get('freqai', {}).get('enabled', False): + whitelist_pairs = self.dp.current_whitelist() + candle_type = self.config.get('candle_type_def', CandleType.SPOT) + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in set(whitelist_pairs + corr_pairs): + informative_pairs.append((pair, tf, candle_type)) + return informative_pairs + + return [] + def gather_informative_pairs(self) -> ListPairsWithTimeframes: """ Internal method which gathers all informative pairs (user or automatically defined). @@ -637,6 +653,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: for pair in self.dp.current_whitelist(): informative_pairs.append((pair, inf_data.timeframe, candle_type)) + informative_pairs.extend(self.__informative_pairs_freqai()) return list(set(informative_pairs)) def get_strategy_name(self) -> str: From d62cef01beba253f4410f3ecf1b8e3a5fb22ea3f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:18:08 +0200 Subject: [PATCH 28/37] Add test for __informative_pairs_freqai --- tests/freqai/test_freqai_interface.py | 25 ++++++++++++++++++++++ tests/strategy/strats/freqai_test_strat.py | 13 ----------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index a66594d7f..6aa4f40c3 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -8,6 +8,7 @@ import pytest from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange, log_has_re from tests.freqai.conftest import get_patched_freqai_strategy @@ -315,3 +316,27 @@ def test_principal_component_analysis(mocker, freqai_conf): assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_pca_object.pkl") shutil.rmtree(Path(freqai.dk.full_path)) + + +@pytest.mark.parametrize('timeframes,corr_pairs', [ + (['5m'], ['ADA/BTC', 'DASH/BTC']), + (['5m'], ['ADA/BTC', 'DASH/BTC']), + (['5m', '15m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), +]) +def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs): + freqai_conf['freqai']['feature_parameters'].update({ + 'include_timeframes': timeframes, + 'include_corr_pairlist': corr_pairs, + + }) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + pairlists = PairListManager(exchange, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange, pairlists) + pairlist = strategy.dp.current_whitelist() + + pairs_a = strategy.informative_pairs() + assert len(pairs_a) == 0 + pairs_b = strategy.gather_informative_pairs() + # we expect unique pairs * timeframes + assert len(pairs_b) == len(set(pairlist + corr_pairs)) * len(timeframes) diff --git a/tests/strategy/strats/freqai_test_strat.py b/tests/strategy/strats/freqai_test_strat.py index 792a3952f..cdfb7f4d0 100644 --- a/tests/strategy/strats/freqai_test_strat.py +++ b/tests/strategy/strats/freqai_test_strat.py @@ -43,19 +43,6 @@ class freqai_test_strat(IStrategy): ) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_any_indicators( self, pair, df, tf, informative=None, set_generalized_indicators=False ): From 10ec681b30ad4826c5ec8a15d94a23820d1176fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:19:46 +0200 Subject: [PATCH 29/37] Clean up no longer needed informative sample code --- docs/freqai.md | 13 ------------- freqtrade/templates/FreqaiExampleStrategy.py | 13 ------------- freqtrade/templates/FreqaiHybridExampleStrategy.py | 14 -------------- .../strats/freqai_test_multimodel_strat.py | 13 ------------- 4 files changed, 53 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index 5f523f58a..33fac198c 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -190,19 +190,6 @@ The FreqAI strategy requires the user to include the following lines of code in # passed to any single indicator) startup_candle_count: int = 20 - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # the model will return all labels created by user in `populate_any_indicators` diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 15b2c6c83..907106453 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -47,19 +47,6 @@ class FreqaiExampleStrategy(IStrategy): std_dev_multiplier_sell = CategoricalParameter( [0.1, 0.25, 0.4], space="sell", default=0.2, optimize=True) - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_any_indicators( self, pair, df, tf, informative=None, set_generalized_indicators=False ): diff --git a/freqtrade/templates/FreqaiHybridExampleStrategy.py b/freqtrade/templates/FreqaiHybridExampleStrategy.py index 286ff012f..593a6062b 100644 --- a/freqtrade/templates/FreqaiHybridExampleStrategy.py +++ b/freqtrade/templates/FreqaiHybridExampleStrategy.py @@ -95,20 +95,6 @@ class FreqaiExampleHybridStrategy(IStrategy): short_rsi = IntParameter(low=51, high=100, default=70, space='sell', optimize=True, load=True) exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) - # FreqAI required function, leave as is or add additional informatives to existing structure. - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - # FreqAI required function, user can add or remove indicators, but general structure # must stay the same. def populate_any_indicators( diff --git a/tests/strategy/strats/freqai_test_multimodel_strat.py b/tests/strategy/strats/freqai_test_multimodel_strat.py index cd3327da9..ada4b25f0 100644 --- a/tests/strategy/strats/freqai_test_multimodel_strat.py +++ b/tests/strategy/strats/freqai_test_multimodel_strat.py @@ -43,19 +43,6 @@ class freqai_test_multimodel_strat(IStrategy): ) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_any_indicators( self, pair, df, tf, informative=None, set_generalized_indicators=False ): From 6682ae35b332539d7b0fdb72af9e5fccc04bb3e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:37:47 +0200 Subject: [PATCH 30/37] Update cached binance_leverage_tiers --- .../exchange/binance_leverage_tiers.json | 686 +++++++++++++++++- 1 file changed, 670 insertions(+), 16 deletions(-) diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index eace16c05..2fa326bb1 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -81,6 +81,104 @@ } } ], + "1000LUNC/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "1000SHIB/BUSD": [ { "tier": 1.0, @@ -1109,6 +1207,88 @@ } } ], + "AMB/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "ANC/BUSD": [ { "tier": 1.0, @@ -3300,13 +3480,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" @@ -4880,13 +5060,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386940.0" @@ -8333,6 +8513,104 @@ } } ], + "FOOTBALL/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "FTM/BUSD": [ { "tier": 1.0, @@ -12123,6 +12401,104 @@ } } ], + "LUNA2/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.015, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.015", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "50.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "675.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5675.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11925.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386925.0" + } + } + ], "MANA/USDT": [ { "tier": 1.0, @@ -13028,10 +13404,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.01, - "maxLeverage": 50.0, + "maxLeverage": 25.0, "info": { "bracket": "1", - "initialLeverage": "50", + "initialLeverage": "25", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.01", @@ -13805,6 +14181,88 @@ } } ], + "PHB/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "QTUM/USDT": [ { "tier": 1.0, @@ -14008,10 +14466,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.01, - "maxLeverage": 50.0, + "maxLeverage": 25.0, "info": { "bracket": "1", - "initialLeverage": "50", + "initialLeverage": "25", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.01", @@ -14478,13 +14936,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" @@ -14576,13 +15034,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" @@ -15487,6 +15945,104 @@ } } ], + "SPELL/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "SRM/USDT": [ { "tier": 1.0, @@ -15585,6 +16141,104 @@ } } ], + "STG/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "STMX/USDT": [ { "tier": 1.0, @@ -16176,13 +16830,13 @@ "tier": 5.0, "currency": "BUSD", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "5", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386875.0" @@ -16470,13 +17124,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" From 4182a7891a261e0dd7facd605a2667f9d2169412 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:41:48 +0200 Subject: [PATCH 31/37] Allow leverage tier cache to be 4 weeks old. we've seen from binance that it's not changing this often. --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_okx.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 33a56c530..a2ddc16e8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2304,7 +2304,7 @@ class Exchange: updated = tiers.get('updated') if updated: updated_dt = parser.parse(updated) - if updated_dt < datetime.now(timezone.utc) - timedelta(days=1): + if updated_dt < datetime.now(timezone.utc) - timedelta(weeks=4): logger.info("Cached leverage tiers are outdated. Will update.") return None return tiers['data'] diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 12322acae..ac5c81ebb 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -472,7 +472,7 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, api_mock.fetch_market_leverage_tiers.call_count == 0 # 2 day passes ... - time_machine.move_to(datetime.now() + timedelta(days=2)) + time_machine.move_to(datetime.now() + timedelta(weeks=5)) exchange.load_leverage_tiers() assert log_has(logmsg, caplog) From 38b28fc4da4caf437d8ccf2bcfd9bc95b79ebe08 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 14:19:20 +0200 Subject: [PATCH 32/37] Update duplicated test --- tests/freqai/test_freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 6aa4f40c3..998bce903 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -320,7 +320,7 @@ def test_principal_component_analysis(mocker, freqai_conf): @pytest.mark.parametrize('timeframes,corr_pairs', [ (['5m'], ['ADA/BTC', 'DASH/BTC']), - (['5m'], ['ADA/BTC', 'DASH/BTC']), + (['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), (['5m', '15m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), ]) def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs): From 8639c1f23d89121d97775daa5157939395f10e07 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:56:27 +0200 Subject: [PATCH 33/37] Reduce complexity in binance stoploss handling --- freqtrade/exchange/binance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 026ba1c65..a012aa3a3 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -48,13 +48,12 @@ class Binance(Exchange): Returns True if adjustment is necessary. :param side: "buy" or "sell" """ - - ordertype = 'stop' if self.trading_mode == TradingMode.FUTURES else 'stop_loss_limit' + order_types = ('stop_loss_limit', 'stop', 'stop_market') return ( order.get('stopPrice', None) is None or ( - order['type'] == ordertype + order['type'] in order_types and ( (side == "sell" and stop_loss > float(order['stopPrice'])) or (side == "buy" and stop_loss < float(order['stopPrice'])) From ca6dec3d4c41783c5a91046825301698c308bdb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 14:55:00 +0200 Subject: [PATCH 34/37] Binance spot also allows market orders closes #7426 --- freqtrade/exchange/binance.py | 2 +- tests/exchange/test_binance.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index a012aa3a3..faa780529 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -31,7 +31,7 @@ class Binance(Exchange): "ccxt_futures_name": "future" } _ft_has_futures: Dict = { - "stoploss_order_types": {"limit": "stop"}, + "stoploss_order_types": {"limit": "limit", "market": "market"}, "tickers_have_price": False, } diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 4d1c40647..e9f4dfa8a 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -23,7 +23,7 @@ from tests.exchange.test_exchange import ccxt_exceptionhandlers def test_stoploss_order_binance(default_conf, mocker, limitratio, expected, side, trademode): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss_limit' if trademode == TradingMode.SPOT else 'stop' + order_type = 'stop_loss_limit' if trademode == TradingMode.SPOT else 'limit' api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -45,12 +45,15 @@ def test_stoploss_order_binance(default_conf, mocker, limitratio, expected, side amount=1, stop_price=190, side=side, - order_types={'stoploss_on_exchange_limit_ratio': 1.05}, + order_types={'stoploss': 'limit', 'stoploss_on_exchange_limit_ratio': 1.05}, leverage=1.0 ) api_mock.create_order.reset_mock() - order_types = {} if limitratio is None else {'stoploss_on_exchange_limit_ratio': limitratio} + order_types = {'stoploss': 'limit'} + if limitratio is not None: + order_types.update({'stoploss_on_exchange_limit_ratio': limitratio}) + order = exchange.stoploss( pair='ETH/BTC', amount=1, From 9f266cbcb2232e7d3fc9219caaef4ea21aa85c48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 16:44:08 +0200 Subject: [PATCH 35/37] Allow safe_price for market stop orders --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ea60796a4..2e479066c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -83,7 +83,7 @@ class Order(_DECL_BASE): @property def safe_price(self) -> float: - return self.average or self.price + return self.average or self.price or self.stop_price @property def safe_filled(self) -> float: From 063511826c3c4ed87e1247552e7a676cc0db4afe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 17:07:47 +0200 Subject: [PATCH 36/37] Update stoploss on exchange logic closes #7424 --- freqtrade/freqtradebot.py | 7 ++++--- freqtrade/persistence/trade_model.py | 7 ++++++- tests/test_freqtradebot.py | 11 +++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6c001a8d6..3eaec5c98 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1072,6 +1072,7 @@ class FreqtradeBot(LoggingMixin): order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss') trade.orders.append(order_obj) trade.stoploss_order_id = str(stoploss_order['id']) + trade.stoploss_last_update = datetime.now(timezone.utc) return True except InsufficientFundsError as e: logger.warning(f"Unable to place stoploss order {e}.") @@ -1145,10 +1146,9 @@ class FreqtradeBot(LoggingMixin): if self.create_stoploss_order(trade=trade, stop_price=stop_price): # The above will return False if the placement failed and the trade was force-sold. # in which case the trade will be closed - which we must check below. - trade.stoploss_last_update = datetime.utcnow() return False - # If stoploss order is canceled for some reason we add it + # If stoploss order is canceled for some reason we add it again if (trade.is_open and stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled')): @@ -1186,7 +1186,8 @@ class FreqtradeBot(LoggingMixin): if self.exchange.stoploss_adjust(stoploss_norm, order, side=trade.exit_side): # we check if the update is necessary update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) - if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: + upd_req = datetime.now(timezone.utc) - timedelta(seconds=update_beat) + if trade.stoploss_last_update_utc and upd_req >= trade.stoploss_last_update_utc: # cancelling the current stoploss on exchange first logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} " f"(orderid:{order['id']}) in order to add another one ...") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 2e479066c..6e421f33e 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -376,6 +376,12 @@ class LocalTrade(): def open_date_utc(self): return self.open_date.replace(tzinfo=timezone.utc) + @property + def stoploss_last_update_utc(self): + if self.stoploss_last_update: + return self.stoploss_last_update.replace(tzinfo=timezone.utc) + return None + @property def close_date_utc(self): return self.close_date.replace(tzinfo=timezone.utc) @@ -560,7 +566,6 @@ class LocalTrade(): self.stop_loss = stop_loss_norm self.stop_loss_pct = -1 * abs(percent) - self.stoploss_last_update = datetime.utcnow() def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False, refresh: bool = False) -> None: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 565797d81..c1152ac09 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1427,6 +1427,7 @@ def test_handle_stoploss_on_exchange_trailing( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 + trade.stoploss_last_update = arrow.utcnow().shift(minutes=-20).datetime stoploss_order_hanging = MagicMock(return_value={ 'id': 100, @@ -1456,7 +1457,7 @@ def test_handle_stoploss_on_exchange_trailing( ) cancel_order_mock = MagicMock() - stoploss_order_mock = MagicMock(return_value={'id': 13434334}) + stoploss_order_mock = MagicMock(return_value={'id': 'so1'}) mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order', cancel_order_mock) mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock) @@ -1569,6 +1570,7 @@ def test_handle_stoploss_on_exchange_trailing_error( assert stoploss.call_count == 1 # Fail creating stoploss order + trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime caplog.clear() cancel_mock = mocker.patch("freqtrade.exchange.Binance.cancel_stoploss_order", MagicMock()) mocker.patch("freqtrade.exchange.Binance.stoploss", side_effect=ExchangeError()) @@ -1657,6 +1659,7 @@ def test_handle_stoploss_on_exchange_custom_stop( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 + trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime stoploss_order_hanging = MagicMock(return_value={ 'id': 100, @@ -1685,7 +1688,7 @@ def test_handle_stoploss_on_exchange_custom_stop( ) cancel_order_mock = MagicMock() - stoploss_order_mock = MagicMock(return_value={'id': 13434334}) + stoploss_order_mock = MagicMock(return_value={'id': 'so1'}) mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order', cancel_order_mock) mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock) @@ -1727,8 +1730,7 @@ def test_handle_stoploss_on_exchange_custom_stop( assert freqtrade.handle_trade(trade) is True -def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, - limit_order) -> None: +def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_order) -> None: enter_order = limit_order['buy'] exit_order = limit_order['sell'] @@ -1784,6 +1786,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 + trade.stoploss_last_update = arrow.utcnow() stoploss_order_hanging = MagicMock(return_value={ 'id': 100, From 92a32ab31b65f77bd63f9feab6b33b7b8b9028cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 17:14:45 +0200 Subject: [PATCH 37/37] Add documentation for stop-market on binance futures part of #7426 --- docs/exchanges.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b5470f65a..dc2003f9c 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -57,7 +57,8 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t Binance supports [time_in_force](configuration.md#understand-order_time_in_force). !!! Tip "Stoploss on Exchange" - Binance supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.. + Binance supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange. + On futures, Binance supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use. ### Binance Blacklist