From 60eb02bb62f5952695a69d9eef531d33d91727bf Mon Sep 17 00:00:00 2001 From: Emre Date: Sat, 10 Sep 2022 20:13:16 +0300 Subject: [PATCH 01/87] Add XGBoostClassifier --- .../prediction_models/XGBoostClassifier.py | 85 +++++++++++++++++++ tests/freqai/test_freqai_interface.py | 30 +++++++ 2 files changed, 115 insertions(+) create mode 100644 freqtrade/freqai/prediction_models/XGBoostClassifier.py diff --git a/freqtrade/freqai/prediction_models/XGBoostClassifier.py b/freqtrade/freqai/prediction_models/XGBoostClassifier.py new file mode 100644 index 000000000..8bf5d6281 --- /dev/null +++ b/freqtrade/freqai/prediction_models/XGBoostClassifier.py @@ -0,0 +1,85 @@ +import logging +from typing import Any, Dict, Tuple + +import numpy as np +import numpy.typing as npt +import pandas as pd +from pandas import DataFrame +from pandas.api.types import is_integer_dtype +from sklearn.preprocessing import LabelEncoder +from xgboost import XGBClassifier + +from freqtrade.freqai.base_models.BaseClassifierModel import BaseClassifierModel +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen + + +logger = logging.getLogger(__name__) + + +class XGBoostClassifier(BaseClassifierModel): + """ + User created prediction model. The class needs to override three necessary + functions, predict(), train(), fit(). The class inherits ModelHandler which + has its own DataHandler where data is held, saved, loaded, and managed. + """ + + def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any: + """ + User sets up the training and test data to fit their desired model here + :params: + :data_dictionary: the dictionary constructed by DataHandler to hold + all the training and test data/labels. + """ + + X = data_dictionary["train_features"].to_numpy() + y = data_dictionary["train_labels"].to_numpy()[:, 0] + + le = LabelEncoder() + if not is_integer_dtype(y): + y = pd.Series(le.fit_transform(y), dtype="int64") + + if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0: + eval_set = None + else: + test_features = data_dictionary["test_features"].to_numpy() + test_labels = data_dictionary["test_labels"].to_numpy()[:, 0] + + if not is_integer_dtype(test_labels): + test_labels = pd.Series(le.transform(test_labels), dtype="int64") + + eval_set = [(test_features, test_labels)] + + train_weights = data_dictionary["train_weights"] + + init_model = self.get_init_model(dk.pair) + + model = XGBClassifier(**self.model_training_parameters) + + model.fit(X=X, y=y, eval_set=eval_set, sample_weight=train_weights, + xgb_model=init_model) + + return model + + def predict( + self, unfiltered_df: DataFrame, dk: FreqaiDataKitchen, **kwargs + ) -> Tuple[DataFrame, npt.NDArray[np.int_]]: + """ + Filter the prediction features data and predict with it. + :param: unfiltered_df: Full dataframe for the current backtest period. + :return: + :pred_df: dataframe containing the predictions + :do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove + data (NaNs) or felt uncertain about data (PCA and DI index) + """ + + (pred_df, dk.do_predict) = super().predict(unfiltered_df, dk, **kwargs) + + le = LabelEncoder() + label = dk.label_list[0] + labels_before = list(dk.data['labels_std'].keys()) + labels_after = le.fit_transform(labels_before).tolist() + pred_df[label] = le.inverse_transform(pred_df[label]) + pred_df = pred_df.rename( + columns={labels_after[i]: labels_before[i] for i in range(len(labels_before))}) + + return (pred_df, dk.do_predict) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 5f8eeb086..afcc4fd37 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -174,6 +174,36 @@ def test_extract_data_and_train_model_LightGBMClassifier(mocker, freqai_conf): shutil.rmtree(Path(freqai.dk.full_path)) +def test_extract_data_and_train_model_XGBoostClassifier(mocker, freqai_conf): + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.update({"freqaimodel": "XGBoostClassifier"}) + freqai_conf.update({"strategy": "freqai_test_classifier"}) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.extract_data_and_train_model(new_timerange, "ADA/BTC", + strategy, freqai.dk, data_load_timerange) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_model.joblib").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_metadata.json").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_trained_df.pkl").exists() + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_svm_model.joblib").exists() + + def test_extract_data_and_train_model_XGBoostRegressor(mocker, freqai_conf): freqai_conf.update({"timerange": "20180110-20180130"}) freqai_conf.update({"freqaimodel": "XGBoostRegressor"}) From d598f4334efb965605f7af0918d358b9d89ec008 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 11 Sep 2022 13:28:14 +0200 Subject: [PATCH 02/87] add search share button to website --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 2e5e0c8c9..257db7867 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -49,6 +49,8 @@ theme: logo: "images/logo.png" favicon: "images/logo.png" custom_dir: "docs/overrides" + features: + - search.share palette: - scheme: default primary: "blue grey" From 4476b5a7f4ab0183197a8f0479b9e3ef0d0e7937 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 08:58:40 +0200 Subject: [PATCH 03/87] add user_data arg to test-pairlist --- docs/utils.md | 4 +++- freqtrade/commands/arguments.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index 5646365e4..174fa0527 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -525,12 +525,14 @@ Requires a configuration with specified `pairlists` attribute. Can be used to generate static pairlists to be used during backtesting / hyperopt. ``` -usage: freqtrade test-pairlist [-h] [-v] [-c PATH] +usage: freqtrade test-pairlist [-h] [--userdir PATH] [-v] [-c PATH] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-1] [--print-json] [--exchange EXCHANGE] optional arguments: -h, --help show this help message and exit + --userdir PATH, --user-data-dir PATH + Path to userdata directory. -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). -c PATH, --config PATH Specify configuration file (default: diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 37ce17f21..2835f8582 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -53,8 +53,8 @@ ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one "print_csv", "base_currencies", "quote_currencies", "list_pairs_all", "trading_mode"] -ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column", - "list_pairs_print_json", "exchange"] +ARGS_TEST_PAIRLIST = ["user_data_dir", "verbosity", "config", "quote_currencies", + "print_one_column", "list_pairs_print_json", "exchange"] ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] From ccc70a21f274a55b1f68b15a6e9ffbfb39844a5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 09:02:06 +0200 Subject: [PATCH 04/87] Update pairs_file cli argument description --- docs/data-download.md | 10 +++++----- freqtrade/commands/cli_options.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index b72e7f337..2b76d4f74 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -25,8 +25,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--include-inactive-pairs] [--timerange TIMERANGE] [--dl-trades] [--exchange EXCHANGE] - [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] - [--erase] + [-t TIMEFRAMES [TIMEFRAMES ...]] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] [--trading-mode {spot,margin,futures}] @@ -37,7 +36,8 @@ optional arguments: -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Limit command to these pairs. Pairs are space- separated. - --pairs-file FILE File containing a list of pairs to download. + --pairs-file FILE File containing a list of pairs. Takes precedence over + --pairs or pairs configured in the configuration. --days INT Download data for given number of days. --new-pairs-days INT Download data of new pairs for given number of days. Default: `None`. @@ -50,7 +50,7 @@ optional arguments: as --timeframes/-t. --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. - -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] + -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...] Specify which tickers to download. Space-separated list. Default: `1m 5m`. --erase Clean all existing data for the selected @@ -61,7 +61,7 @@ optional arguments: --data-format-trades {json,jsongz,hdf5} Storage format for downloaded trades data. (default: `jsongz`). - --trading-mode {spot,margin,futures} + --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures} Select Trading mode --prepend Allow data prepending. (Data-appending is disabled) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 4240a8014..9aacbcc97 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -393,7 +393,8 @@ AVAILABLE_CLI_OPTIONS = { # Download data "pairs_file": Arg( '--pairs-file', - help='File containing a list of pairs to download.', + help='File containing a list of pairs. ' + 'Takes precedence over --pairs or pairs configured in the configuration.', metavar='FILE', ), "days": Arg( From 9c8c7a03a160cdaad21b85d472879e5d5649d269 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:24:01 +0200 Subject: [PATCH 05/87] Improve typehint --- freqtrade/data/dataprovider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 4151b7419..e639b3ae7 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -86,7 +86,7 @@ class DataProvider: """ _candle_type = CandleType.from_string( candle_type) if candle_type != '' else self._config['candle_type_def'] - saved_pair = (pair, str(timeframe), _candle_type) + saved_pair: PairWithTimeframe = (pair, str(timeframe), _candle_type) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) From a48923c0e43eda833aced9afc8a13e98caa29c8f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:38:36 +0200 Subject: [PATCH 06/87] Extract widget colorization to separate function --- freqtrade/optimize/hyperopt.py | 46 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3becf857f..749e608d3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -410,7 +410,7 @@ class Hyperopt: model_queue_size=SKOPT_MODEL_QUEUE_SIZE, ) - def run_optimizer_parallel(self, parallel, asked, i) -> List: + def run_optimizer_parallel(self, parallel: Parallel, asked: List[List], i: int) -> List: return parallel(delayed( wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @@ -491,6 +491,29 @@ class Hyperopt: else: return self.opt.ask(n_points=n_points), [False for _ in range(n_points)] + def get_progressbar_widgets(self): + if self.print_colorized: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='\N{FULL BLOCK}', + fill_wrap=Fore.GREEN + '{}' + Fore.RESET, + marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + else: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='\N{FULL BLOCK}', + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + return widgets + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state')) logger.info(f"Using optimizer random state: {self.random_state}") @@ -526,26 +549,7 @@ class Hyperopt: logger.info(f'Effective number of parallel workers used: {jobs}') # Define progressbar - if self.print_colorized: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - fill_wrap=Fore.GREEN + '{}' + Fore.RESET, - marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] - else: - widgets = [ - ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), - ' (', progressbar.Percentage(), ')] ', - progressbar.Bar(marker=progressbar.AnimatedMarker( - fill='\N{FULL BLOCK}', - )), - ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', - ] + widgets = self.get_progressbar_widgets() with progressbar.ProgressBar( max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, widgets=widgets From 32e13d65c3bbba091eac4ae16259adaaf483a02a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:54:31 +0200 Subject: [PATCH 07/87] Refactor hyperopt to extract evaluate_result --- freqtrade/optimize/hyperopt.py | 49 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 749e608d3..3a117f356 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -290,7 +290,7 @@ class Hyperopt: # noinspection PyProtectedMember attr.value = params_dict[attr_name] - def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: + def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict[str, Any]: """ Used Optimize function. Called once per epoch to optimize whatever is configured. @@ -410,7 +410,9 @@ class Hyperopt: model_queue_size=SKOPT_MODEL_QUEUE_SIZE, ) - def run_optimizer_parallel(self, parallel: Parallel, asked: List[List], i: int) -> List: + def run_optimizer_parallel( + self, parallel: Parallel, asked: List[List], i: int) -> List[Dict[str, Any]]: + """ Start optimizer in a parallel way """ return parallel(delayed( wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @@ -514,6 +516,30 @@ class Hyperopt: ] return widgets + def evaluate_result(self, val: Dict[str, Any], current: int, is_random: bool): + """ + Evaluate results returned from generate_optimizer + """ + val['current_epoch'] = current + val['is_initial_point'] = current <= INITIAL_POINTS + + logger.debug("Optimizer epoch evaluated: %s", val) + + is_best = HyperoptTools.is_best_loss(val, self.current_best_loss) + # This value is assigned here and not in the optimization method + # to keep proper order in the list of results. That's because + # evaluations can take different time. Here they are aligned in the + # order they will be shown to the user. + val['is_best'] = is_best + val['is_random'] = is_random + self.print_results(val) + + if is_best: + self.current_best_loss = val['loss'] + self.current_best_epoch = val + + self._save_result(val) + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state')) logger.info(f"Using optimizer random state: {self.random_state}") @@ -569,25 +595,8 @@ class Hyperopt: for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) current = i * jobs + j + 1 - val['current_epoch'] = current - val['is_initial_point'] = current <= INITIAL_POINTS - logger.debug(f"Optimizer epoch evaluated: {val}") - - is_best = HyperoptTools.is_best_loss(val, self.current_best_loss) - # This value is assigned here and not in the optimization method - # to keep proper order in the list of results. That's because - # evaluations can take different time. Here they are aligned in the - # order they will be shown to the user. - val['is_best'] = is_best - val['is_random'] = is_random[j] - self.print_results(val) - - if is_best: - self.current_best_loss = val['loss'] - self.current_best_epoch = val - - self._save_result(val) + self.evaluate_result(val, current, is_random[j]) pbar.update(current) From 78cd46ecd55e1042ef5f20f10ebd7e0c269d6ca0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 11:56:17 +0200 Subject: [PATCH 08/87] hyperopt Remove unnecessary arguments --- freqtrade/optimize/hyperopt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3a117f356..76fc84609 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -290,7 +290,7 @@ class Hyperopt: # noinspection PyProtectedMember attr.value = params_dict[attr_name] - def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict[str, Any]: + def generate_optimizer(self, raw_params: List[Any]) -> Dict[str, Any]: """ Used Optimize function. Called once per epoch to optimize whatever is configured. @@ -411,10 +411,10 @@ class Hyperopt: ) def run_optimizer_parallel( - self, parallel: Parallel, asked: List[List], i: int) -> List[Dict[str, Any]]: + self, parallel: Parallel, asked: List[List]) -> List[Dict[str, Any]]: """ Start optimizer in a parallel way """ return parallel(delayed( - wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) + wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) @@ -588,7 +588,7 @@ class Hyperopt: current_jobs = jobs - n_rest if n_rest > 0 else jobs asked, is_random = self.get_asked_points(n_points=current_jobs) - f_val = self.run_optimizer_parallel(parallel, asked, i) + f_val = self.run_optimizer_parallel(parallel, asked) self.opt.tell(asked, [v['loss'] for v in f_val]) # Calculate progressbar outputs From 72d197a99d0570ae9d3607042d7ec5a533907c8e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 15:42:27 +0200 Subject: [PATCH 09/87] Run first epoch in non-parallel mode this allows dataprovider to load it's cache. closes #7384 --- freqtrade/data/dataprovider.py | 4 +++- freqtrade/optimize/hyperopt.py | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index e639b3ae7..c6519d2b8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -196,7 +196,9 @@ class DataProvider: Clear pair dataframe cache. """ self.__cached_pairs = {} - self.__cached_pairs_backtesting = {} + # Don't reset backtesting pairs - + # otherwise they're reloaded each time during hyperopt due to with analyze_per_epoch + # self.__cached_pairs_backtesting = {} self.__slice_index = 0 # Exchange functions diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 76fc84609..b0119368f 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -580,11 +580,24 @@ class Hyperopt: max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, widgets=widgets ) as pbar: - EVALS = ceil(self.total_epochs / jobs) - for i in range(EVALS): + start = 0 + + if self.analyze_per_epoch: + # First analysis not in parallel mode when using --analyze-per-epoch. + # This allows dataprovider to load it's informative cache. + asked, is_random = self.get_asked_points(n_points=1) + # print(asked) + f_val = self.generate_optimizer(asked[0]) + self.opt.tell(asked, [f_val['loss']]) + self.evaluate_result(f_val, 1, is_random[0]) + pbar.update(1) + start += 1 + + evals = ceil((self.total_epochs - start) / jobs) + for i in range(evals): # Correct the number of epochs to be processed for the last # iteration (should not exceed self.total_epochs in total) - n_rest = (i + 1) * jobs - self.total_epochs + n_rest = (i + 1) * jobs - (self.total_epochs - start) current_jobs = jobs - n_rest if n_rest > 0 else jobs asked, is_random = self.get_asked_points(n_points=current_jobs) @@ -594,7 +607,7 @@ class Hyperopt: # Calculate progressbar outputs for j, val in enumerate(f_val): # Use human-friendly indexes here (starting from 1) - current = i * jobs + j + 1 + current = i * jobs + j + 1 + start self.evaluate_result(val, current, is_random[j]) From 816c1f760373f6fc55710cb2e3f09ae39eb14fc5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 17:51:30 +0200 Subject: [PATCH 10/87] add test for per epoch hyperopt --- tests/optimize/test_hyperopt.py | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 0f615b7a3..eaea8aee7 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -922,6 +922,45 @@ def test_in_strategy_auto_hyperopt_with_parallel(mocker, hyperopt_conf, tmpdir, hyperopt.start() +def test_in_strategy_auto_hyperopt_per_epoch(mocker, hyperopt_conf, tmpdir, fee) -> None: + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) + + hyperopt_conf.update({ + 'strategy': 'HyperoptableStrategy', + 'user_data_dir': Path(tmpdir), + 'hyperopt_random_state': 42, + 'spaces': ['all'], + 'epochs': 3, + 'analyze_per_epoch': True, + }) + go = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.generate_optimizer', + return_value={ + 'loss': 0.05, + 'results_explanation': 'foo result', 'params': {}, + 'results_metrics': generate_result_metrics(), + }) + hyperopt = Hyperopt(hyperopt_conf) + hyperopt.backtesting.exchange.get_max_leverage = MagicMock(return_value=1.0) + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter) + assert hyperopt.backtesting.strategy.bot_loop_started is True + + assert hyperopt.backtesting.strategy.buy_rsi.in_space is True + assert hyperopt.backtesting.strategy.buy_rsi.value == 35 + assert hyperopt.backtesting.strategy.sell_rsi.value == 74 + assert hyperopt.backtesting.strategy.protection_cooldown_lookback.value == 30 + buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range + assert isinstance(buy_rsi_range, range) + # Range from 0 - 50 (inclusive) + assert len(list(buy_rsi_range)) == 51 + + hyperopt.start() + # backtesting should be called 3 times (once per epoch) + assert go.call_count == 3 + + def test_SKDecimal(): space = SKDecimal(1, 2, decimals=2) assert 1.5 in space From 982c0315fa2ba909362173c2e892aa7bca2c836b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 11 Sep 2022 19:31:11 +0200 Subject: [PATCH 11/87] Rename variable --- freqtrade/optimize/hyperopt.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b0119368f..f15e0b7d8 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -586,10 +586,9 @@ class Hyperopt: # First analysis not in parallel mode when using --analyze-per-epoch. # This allows dataprovider to load it's informative cache. asked, is_random = self.get_asked_points(n_points=1) - # print(asked) - f_val = self.generate_optimizer(asked[0]) - self.opt.tell(asked, [f_val['loss']]) - self.evaluate_result(f_val, 1, is_random[0]) + f_val0 = self.generate_optimizer(asked[0]) + self.opt.tell(asked, [f_val0['loss']]) + self.evaluate_result(f_val0, 1, is_random[0]) pbar.update(1) start += 1 From 6777d43aea7f169db1711999b4a15c3fedf21a5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:01:27 +0000 Subject: [PATCH 12/87] Bump sqlalchemy from 1.4.40 to 1.4.41 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.40 to 1.4.41. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdbe6ac28..d063410d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.93.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 -SQLAlchemy==1.4.40 +SQLAlchemy==1.4.41 python-telegram-bot==13.14 arrow==1.2.3 cachetools==4.2.2 From 6bfe99606179ebb1cbb236b8751da180ccd1b3ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:01:56 +0000 Subject: [PATCH 13/87] Bump mkdocs-material from 8.4.2 to 8.4.3 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.4.2 to 8.4.3. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.4.2...8.4.3) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 5a6c46471..d63e79004 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.3.1 -mkdocs-material==8.4.2 +mkdocs-material==8.4.3 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 From 1ef334411ed5c9ba0204acb41172089bfb5c38dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:02:01 +0000 Subject: [PATCH 14/87] Bump types-requests from 2.28.9 to 2.28.10 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.9 to 2.28.10. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 40ca4e154..69749f843 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -25,6 +25,6 @@ nbconvert==7.0.0 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.9 +types-requests==2.28.10 types-tabulate==0.8.11 types-python-dateutil==2.8.19 From a4b7e0a714c0f9624d693772467080c43012649e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:02:06 +0000 Subject: [PATCH 15/87] Bump fastapi from 0.82.0 to 0.83.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.82.0 to 0.83.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.82.0...0.83.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdbe6ac28..ea8cb39ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.8.0 sdnotify==0.3.2 # API Server -fastapi==0.82.0 +fastapi==0.83.0 uvicorn==0.18.3 pyjwt==2.4.0 aiofiles==0.8.0 From 6968fc333b584355c90cf9235ee743a23a239e9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 03:02:10 +0000 Subject: [PATCH 16/87] Bump jsonschema from 4.15.0 to 4.16.0 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.15.0 to 4.16.0. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.15.0...v4.16.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fdbe6ac28..8a3701bb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.3 cachetools==4.2.2 requests==2.28.1 urllib3==1.26.12 -jsonschema==4.15.0 +jsonschema==4.16.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.10 From c149c47afbc20b083c3f6077edcb6fdc1c223801 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 05:05:30 +0000 Subject: [PATCH 17/87] Bump ccxt from 1.93.3 to 1.93.35 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.93.3 to 1.93.35. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.93.3...1.93.35) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d063410d4..af9048c87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.2 pandas==1.4.4 pandas-ta==0.3.14b -ccxt==1.93.3 +ccxt==1.93.35 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 From 4ac804f795227163d93c03438cddbd8d722fdd9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 06:05:04 +0000 Subject: [PATCH 18/87] Bump numpy from 1.23.2 to 1.23.3 Bumps [numpy](https://github.com/numpy/numpy) from 1.23.2 to 1.23.3. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.23.2...v1.23.3) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9bb8e6af0..551440cc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.23.2 +numpy==1.23.3 pandas==1.4.4 pandas-ta==0.3.14b From 00db473f102fb4c6cee1199930e9fe81c3f7792c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 06:05:22 +0000 Subject: [PATCH 19/87] Bump cryptography from 37.0.4 to 38.0.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 37.0.4 to 38.0.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/37.0.4...38.0.1) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9bb8e6af0..59bef6b11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b ccxt==1.93.35 # Pin cryptography for now due to rust build errors with piwheels -cryptography==37.0.4 +cryptography==38.0.1 aiohttp==3.8.1 SQLAlchemy==1.4.41 python-telegram-bot==13.14 From 7ee92db7a2c2f25b48917d285f11ab8f6d474418 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 06:07:40 +0000 Subject: [PATCH 20/87] Bump aiofiles from 0.8.0 to 22.1.0 Bumps [aiofiles](https://github.com/Tinche/aiofiles) from 0.8.0 to 22.1.0. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.8.0...v22.1.0) --- updated-dependencies: - dependency-name: aiofiles dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 345c336f5..e263b7914 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ sdnotify==0.3.2 fastapi==0.83.0 uvicorn==0.18.3 pyjwt==2.4.0 -aiofiles==0.8.0 +aiofiles==22.1.0 psutil==5.9.2 # Support for colorized terminal output From f45824acf536ccd7f56d78f0ea692a414709962c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Sep 2022 09:23:02 +0200 Subject: [PATCH 21/87] Bump precommit types/requests --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86c4ec1ad..6f2b3ba3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.9 + - types-requests==2.28.10 - types-tabulate==0.8.11 - types-python-dateutil==2.8.19 # stages: [push] From 7b6e465d57e392114d8cca2df24c18c5d4c3e4fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Sep 2022 21:28:11 +0200 Subject: [PATCH 22/87] Remove gate live test skip --- tests/exchange/test_ccxt_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index f57b0b366..82be6196a 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -297,7 +297,7 @@ class TestCCXTExchange(): def test_ccxt__async_get_candle_history(self, exchange): exchange, exchangename = exchange # For some weired reason, this test returns random lengths for bittrex. - if not exchange._ft_has['ohlcv_has_history'] or exchangename in ('bittrex', 'gateio'): + if not exchange._ft_has['ohlcv_has_history'] or exchangename in ('bittrex'): return pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] From 37dd22c89e2e50bce49ffce17277dc06d2abec8b Mon Sep 17 00:00:00 2001 From: initrv Date: Wed, 14 Sep 2022 03:40:13 +0300 Subject: [PATCH 23/87] Fixed a bug that prevents clearing old models Corrects the error of clearing old models when the model directory contains directories with names that do not match a regular expression --- freqtrade/freqai/data_drawer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 9eeabef8f..1c091f1be 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -355,7 +355,7 @@ class FreqaiDataDrawer: for dir in model_folders: result = pattern.match(str(dir.name)) if result is None: - break + continue coin = result.group(1) timestamp = result.group(2) From 49800e4cc300eb39b412287a05480a98962c354b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Sep 2022 06:55:05 +0200 Subject: [PATCH 24/87] pin ci python to 3.10.6 for now --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5bc209e..91d53044d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: os: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10.6"] steps: - uses: actions/checkout@v3 @@ -121,7 +121,7 @@ jobs: strategy: matrix: os: [ macos-latest ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10.6"] steps: - uses: actions/checkout@v3 @@ -205,7 +205,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10.6"] steps: - uses: actions/checkout@v3 From 91bc3d11618c3f5bf47a8a71166544cf4e0b7a0f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Sep 2022 07:04:14 +0200 Subject: [PATCH 25/87] Update docs aroudn use_exit_signal close #7413 --- docs/strategy-customization.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 2b6e1fb2f..b97bd6d23 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -264,7 +264,8 @@ def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFram ### Exit signal rules Edit the method `populate_exit_trend()` into your strategy file to update your exit strategy. -Please note that the exit-signal is only used if `use_exit_signal` is set to true in the configuration. +The exit-signal is only used for exits if `use_exit_signal` is set to true in the configuration. +`use_exit_signal` will not influence [signal collision rules](#colliding-signals) - which will still apply and can prevent entries. It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. From e5368f5a149bf0e330dfac53fdd0d0e44679b469 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 16 Sep 2022 13:24:20 +0000 Subject: [PATCH 26/87] backtesting confirm_trade_entry should pass correct amount, not stake-amount closes #7423 --- freqtrade/optimize/backtesting.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 97418b72c..105851e60 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -812,14 +812,6 @@ class Backtesting: return trade time_in_force = self.strategy.order_time_in_force['entry'] - if not pos_adjust: - # Confirm trade entry: - if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=stake_amount, rate=propose_rate, - time_in_force=time_in_force, current_time=current_time, - entry_tag=entry_tag, side=direction): - return trade - if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): self.order_id_counter += 1 base_currency = self.exchange.get_pair_base_currency(pair) @@ -834,6 +826,15 @@ class Backtesting: # Backcalculate actual stake amount. stake_amount = amount * propose_rate / leverage + if not pos_adjust: + # Confirm trade entry: + if not strategy_safe_wrapper( + self.strategy.confirm_trade_entry, default_retval=True)( + pair=pair, order_type=order_type, amount=amount, rate=propose_rate, + time_in_force=time_in_force, current_time=current_time, + entry_tag=entry_tag, side=direction): + return trade + is_short = (direction == 'short') # Necessary for Margin trading. Disabled until support is enabled. # interest_rate = self.exchange.get_interest_rate() From b707a6da35f8af7719e256bc3dfda300e9844a4c Mon Sep 17 00:00:00 2001 From: initrv Date: Fri, 16 Sep 2022 19:17:41 +0300 Subject: [PATCH 27/87] Add ability to plot feature importance --- config_examples/config_freqai.example.json | 3 +- freqtrade/freqai/freqai_interface.py | 9 +++ freqtrade/freqai/utils.py | 71 ++++++++++++++++++++++ requirements-plot.txt | 1 + 4 files changed, 83 insertions(+), 1 deletion(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 12eb30128..9494ba0e1 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -77,7 +77,8 @@ "indicator_periods_candles": [ 10, 20 - ] + ], + "plot_feature_importance": true }, "data_split_parameters": { "test_size": 0.33, diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 78931bed4..0cc51fdab 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -20,6 +20,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.freqai.utils import plot_feature_importance from freqtrade.strategy.interface import IStrategy @@ -555,6 +556,14 @@ class IFreqaiModel(ABC): model = self.train(unfiltered_dataframe, pair, dk) + if self.freqai_info["feature_parameters"].get("plot_feature_importance", False): + plot_feature_importance( + model=model, + feature_names=dk.training_features_list, + pair=pair, + train_dir=dk.data_path + ) + self.dd.pair_dict[pair]["trained_timestamp"] = new_trained_timerange.stopts dk.set_new_model_names(pair, new_trained_timerange) self.dd.pair_dict[pair]["first"] = False diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 6a70f050f..86d89d4b0 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,5 +1,13 @@ import logging from datetime import datetime, timezone +# for plot_feature_importance +from pathlib import Path + +import numpy as np +import pandas as pd +import plotly.graph_objects as go +import plotly.io as pio +from plotly.subplots import make_subplots from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider @@ -132,3 +140,66 @@ def get_required_data_timerange( # trading_mode=config.get("trading_mode", "spot"), # prepend=config.get("prepend_data", False), # ) + + +def plot_feature_importance(model, feature_names, pair, train_dir, count_max=25) -> None: + """ + Plot Best and Worst Features by importance for CatBoost model. + Called once per sub-train. + + Required: pip install kaleido + + Usage: plot_feature_importance( + model=model, + feature_names=dk.training_features_list, + pair=pair, + train_dir=dk.data_path) + """ + + # Gather feature importance from model + if "catboost.core" in str(model.__class__): + fi = model.get_feature_importance() + + elif "lightgbm.sklearn" in str(model.__class__): + fi = model.feature_importances_ + + else: + raise NotImplementedError(f"Cannot extract feature importance for {model.__class__}") + + # Data preparation + fi_df = pd.DataFrame({ + "feature_names": np.array(feature_names), + "feature_importance": np.array(fi) + }) + fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1] + fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1] + + # Plotting + fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5) + fig.add_trace( + go.Bar( + x=fi_df_top["feature_importance"], + y=fi_df_top["feature_names"], + orientation='h', showlegend=False + ), row=1, col=1 + ) + fig.add_trace( + go.Bar( + x=fi_df_worst["feature_importance"], + y=fi_df_worst["feature_names"], + orientation='h', showlegend=False + ), row=1, col=2 + ) + fig.update_layout( + title_text=f"Best and Worst Features {pair}", + width=1000, height=600 + ) + + # Create directory and save image + model_dir, train_name = str(train_dir).rsplit("/", 1) + fi_dir = Path(f"{model_dir}/feature_importance/{pair.split('/')[0]}") + fi_dir.mkdir(parents=True, exist_ok=True) + + pio.write_image(fig, f"{fi_dir}/{train_name}.png", format="png") + + logger.info(f"Freqai saving feature importance plot {fi_dir}/{train_name}.png") diff --git a/requirements-plot.txt b/requirements-plot.txt index 80cd3f4f2..ef3cf9f24 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -2,3 +2,4 @@ -r requirements.txt plotly==5.10.0 +kaleido==0.2.1 From 86aa875bc9d5edeba04f908fe45b011e52045c83 Mon Sep 17 00:00:00 2001 From: initrv Date: Fri, 16 Sep 2022 21:47:12 +0300 Subject: [PATCH 28/87] plot features as html instead of png --- freqtrade/freqai/utils.py | 62 ++++++++++++++++----------------------- requirements-plot.txt | 1 - 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 86d89d4b0..3f6b8b053 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,13 +1,9 @@ import logging from datetime import datetime, timezone -# for plot_feature_importance from pathlib import Path import numpy as np import pandas as pd -import plotly.graph_objects as go -import plotly.io as pio -from plotly.subplots import make_subplots from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider @@ -142,64 +138,56 @@ def get_required_data_timerange( # ) -def plot_feature_importance(model, feature_names, pair, train_dir, count_max=25) -> None: +def plot_feature_importance(model, feature_names, pair, train_dir, count_max=50) -> None: """ Plot Best and Worst Features by importance for CatBoost model. Called once per sub-train. - - Required: pip install kaleido - Usage: plot_feature_importance( model=model, feature_names=dk.training_features_list, pair=pair, train_dir=dk.data_path) """ + try: + import plotly.graph_objects as go + from plotly.subplots import make_subplots + except ImportError: + logger.exception("Module plotly not found \n Please install using `pip3 install plotly`") + exit(1) + + from freqtrade.plot.plotting import store_plot_file # Gather feature importance from model if "catboost.core" in str(model.__class__): - fi = model.get_feature_importance() - + feature_importance = model.get_feature_importance() elif "lightgbm.sklearn" in str(model.__class__): - fi = model.feature_importances_ - + feature_importance = model.feature_importances_ else: raise NotImplementedError(f"Cannot extract feature importance for {model.__class__}") # Data preparation fi_df = pd.DataFrame({ "feature_names": np.array(feature_names), - "feature_importance": np.array(fi) + "feature_importance": np.array(feature_importance) }) fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1] fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1] # Plotting + def add_feature_trace(fig, fi_df, col): + return fig.add_trace( + go.Bar( + x=fi_df["feature_importance"], + y=fi_df["feature_names"], + orientation='h', showlegend=False + ), row=1, col=col + ) fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5) - fig.add_trace( - go.Bar( - x=fi_df_top["feature_importance"], - y=fi_df_top["feature_names"], - orientation='h', showlegend=False - ), row=1, col=1 - ) - fig.add_trace( - go.Bar( - x=fi_df_worst["feature_importance"], - y=fi_df_worst["feature_names"], - orientation='h', showlegend=False - ), row=1, col=2 - ) - fig.update_layout( - title_text=f"Best and Worst Features {pair}", - width=1000, height=600 - ) + fig = add_feature_trace(fig, fi_df_top, 1) + fig = add_feature_trace(fig, fi_df_worst, 2) + fig.update_layout(title_text=f"Best and Worst Features {pair}") - # Create directory and save image + # Store plot file model_dir, train_name = str(train_dir).rsplit("/", 1) fi_dir = Path(f"{model_dir}/feature_importance/{pair.split('/')[0]}") - fi_dir.mkdir(parents=True, exist_ok=True) - - pio.write_image(fig, f"{fi_dir}/{train_name}.png", format="png") - - logger.info(f"Freqai saving feature importance plot {fi_dir}/{train_name}.png") + store_plot_file(fig, f"{train_name}.html", fi_dir) diff --git a/requirements-plot.txt b/requirements-plot.txt index ef3cf9f24..80cd3f4f2 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -2,4 +2,3 @@ -r requirements.txt plotly==5.10.0 -kaleido==0.2.1 From 0aada271cafa282494b29318ca32f7e7b4a7309a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:11:04 +0200 Subject: [PATCH 29/87] Move informative_pairs for freqAI to backend --- freqtrade/strategy/interface.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9401ebebe..93988ac48 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -613,6 +613,22 @@ class IStrategy(ABC, HyperStrategyMixin): # END - Intended to be overridden by strategy ### + def __informative_pairs_freqai(self) -> ListPairsWithTimeframes: + """ + Create informative-pairs needed for FreqAI + """ + if self.config.get('freqai', {}).get('enabled', False): + whitelist_pairs = self.dp.current_whitelist() + candle_type = self.config.get('candle_type_def', CandleType.SPOT) + corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] + informative_pairs = [] + for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: + for pair in set(whitelist_pairs + corr_pairs): + informative_pairs.append((pair, tf, candle_type)) + return informative_pairs + + return [] + def gather_informative_pairs(self) -> ListPairsWithTimeframes: """ Internal method which gathers all informative pairs (user or automatically defined). @@ -637,6 +653,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: for pair in self.dp.current_whitelist(): informative_pairs.append((pair, inf_data.timeframe, candle_type)) + informative_pairs.extend(self.__informative_pairs_freqai()) return list(set(informative_pairs)) def get_strategy_name(self) -> str: From d62cef01beba253f4410f3ecf1b8e3a5fb22ea3f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:18:08 +0200 Subject: [PATCH 30/87] Add test for __informative_pairs_freqai --- tests/freqai/test_freqai_interface.py | 25 ++++++++++++++++++++++ tests/strategy/strats/freqai_test_strat.py | 13 ----------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index a66594d7f..6aa4f40c3 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -8,6 +8,7 @@ import pytest from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange, log_has_re from tests.freqai.conftest import get_patched_freqai_strategy @@ -315,3 +316,27 @@ def test_principal_component_analysis(mocker, freqai_conf): assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_pca_object.pkl") shutil.rmtree(Path(freqai.dk.full_path)) + + +@pytest.mark.parametrize('timeframes,corr_pairs', [ + (['5m'], ['ADA/BTC', 'DASH/BTC']), + (['5m'], ['ADA/BTC', 'DASH/BTC']), + (['5m', '15m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), +]) +def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs): + freqai_conf['freqai']['feature_parameters'].update({ + 'include_timeframes': timeframes, + 'include_corr_pairlist': corr_pairs, + + }) + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + pairlists = PairListManager(exchange, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange, pairlists) + pairlist = strategy.dp.current_whitelist() + + pairs_a = strategy.informative_pairs() + assert len(pairs_a) == 0 + pairs_b = strategy.gather_informative_pairs() + # we expect unique pairs * timeframes + assert len(pairs_b) == len(set(pairlist + corr_pairs)) * len(timeframes) diff --git a/tests/strategy/strats/freqai_test_strat.py b/tests/strategy/strats/freqai_test_strat.py index 792a3952f..cdfb7f4d0 100644 --- a/tests/strategy/strats/freqai_test_strat.py +++ b/tests/strategy/strats/freqai_test_strat.py @@ -43,19 +43,6 @@ class freqai_test_strat(IStrategy): ) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_any_indicators( self, pair, df, tf, informative=None, set_generalized_indicators=False ): From 10ec681b30ad4826c5ec8a15d94a23820d1176fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:19:46 +0200 Subject: [PATCH 31/87] Clean up no longer needed informative sample code --- docs/freqai.md | 13 ------------- freqtrade/templates/FreqaiExampleStrategy.py | 13 ------------- freqtrade/templates/FreqaiHybridExampleStrategy.py | 14 -------------- .../strats/freqai_test_multimodel_strat.py | 13 ------------- 4 files changed, 53 deletions(-) diff --git a/docs/freqai.md b/docs/freqai.md index 5f523f58a..33fac198c 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -190,19 +190,6 @@ The FreqAI strategy requires the user to include the following lines of code in # passed to any single indicator) startup_candle_count: int = 20 - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # the model will return all labels created by user in `populate_any_indicators` diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 15b2c6c83..907106453 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -47,19 +47,6 @@ class FreqaiExampleStrategy(IStrategy): std_dev_multiplier_sell = CategoricalParameter( [0.1, 0.25, 0.4], space="sell", default=0.2, optimize=True) - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_any_indicators( self, pair, df, tf, informative=None, set_generalized_indicators=False ): diff --git a/freqtrade/templates/FreqaiHybridExampleStrategy.py b/freqtrade/templates/FreqaiHybridExampleStrategy.py index 286ff012f..593a6062b 100644 --- a/freqtrade/templates/FreqaiHybridExampleStrategy.py +++ b/freqtrade/templates/FreqaiHybridExampleStrategy.py @@ -95,20 +95,6 @@ class FreqaiExampleHybridStrategy(IStrategy): short_rsi = IntParameter(low=51, high=100, default=70, space='sell', optimize=True, load=True) exit_short_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) - # FreqAI required function, leave as is or add additional informatives to existing structure. - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - # FreqAI required function, user can add or remove indicators, but general structure # must stay the same. def populate_any_indicators( diff --git a/tests/strategy/strats/freqai_test_multimodel_strat.py b/tests/strategy/strats/freqai_test_multimodel_strat.py index cd3327da9..ada4b25f0 100644 --- a/tests/strategy/strats/freqai_test_multimodel_strat.py +++ b/tests/strategy/strats/freqai_test_multimodel_strat.py @@ -43,19 +43,6 @@ class freqai_test_multimodel_strat(IStrategy): ) max_roi_time_long = IntParameter(0, 800, default=400, space="sell", optimize=False, load=True) - def informative_pairs(self): - whitelist_pairs = self.dp.current_whitelist() - corr_pairs = self.config["freqai"]["feature_parameters"]["include_corr_pairlist"] - informative_pairs = [] - for tf in self.config["freqai"]["feature_parameters"]["include_timeframes"]: - for pair in whitelist_pairs: - informative_pairs.append((pair, tf)) - for pair in corr_pairs: - if pair in whitelist_pairs: - continue # avoid duplication - informative_pairs.append((pair, tf)) - return informative_pairs - def populate_any_indicators( self, pair, df, tf, informative=None, set_generalized_indicators=False ): From 6682ae35b332539d7b0fdb72af9e5fccc04bb3e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:37:47 +0200 Subject: [PATCH 32/87] Update cached binance_leverage_tiers --- .../exchange/binance_leverage_tiers.json | 686 +++++++++++++++++- 1 file changed, 670 insertions(+), 16 deletions(-) diff --git a/freqtrade/exchange/binance_leverage_tiers.json b/freqtrade/exchange/binance_leverage_tiers.json index eace16c05..2fa326bb1 100644 --- a/freqtrade/exchange/binance_leverage_tiers.json +++ b/freqtrade/exchange/binance_leverage_tiers.json @@ -81,6 +81,104 @@ } } ], + "1000LUNC/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "1000SHIB/BUSD": [ { "tier": 1.0, @@ -1109,6 +1207,88 @@ } } ], + "AMB/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "ANC/BUSD": [ { "tier": 1.0, @@ -3300,13 +3480,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" @@ -4880,13 +5060,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386940.0" @@ -8333,6 +8513,104 @@ } } ], + "FOOTBALL/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "FTM/BUSD": [ { "tier": 1.0, @@ -12123,6 +12401,104 @@ } } ], + "LUNA2/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.015, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.015", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "50.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "675.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5675.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11925.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386925.0" + } + } + ], "MANA/USDT": [ { "tier": 1.0, @@ -13028,10 +13404,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.01, - "maxLeverage": 50.0, + "maxLeverage": 25.0, "info": { "bracket": "1", - "initialLeverage": "50", + "initialLeverage": "25", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.01", @@ -13805,6 +14181,88 @@ } } ], + "PHB/BUSD": [ + { + "tier": 1.0, + "currency": "BUSD", + "minNotional": 0.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "1", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "0", + "maintMarginRatio": "0.025", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "BUSD", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "2", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "625.0" + } + }, + { + "tier": 3.0, + "currency": "BUSD", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "3", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5625.0" + } + }, + { + "tier": 4.0, + "currency": "BUSD", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "4", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11875.0" + } + }, + { + "tier": 5.0, + "currency": "BUSD", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "5", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386875.0" + } + } + ], "QTUM/USDT": [ { "tier": 1.0, @@ -14008,10 +14466,10 @@ "minNotional": 0.0, "maxNotional": 5000.0, "maintenanceMarginRate": 0.01, - "maxLeverage": 50.0, + "maxLeverage": 25.0, "info": { "bracket": "1", - "initialLeverage": "50", + "initialLeverage": "25", "notionalCap": "5000", "notionalFloor": "0", "maintMarginRatio": "0.01", @@ -14478,13 +14936,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" @@ -14576,13 +15034,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" @@ -15487,6 +15945,104 @@ } } ], + "SPELL/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "SRM/USDT": [ { "tier": 1.0, @@ -15585,6 +16141,104 @@ } } ], + "STG/USDT": [ + { + "tier": 1.0, + "currency": "USDT", + "minNotional": 0.0, + "maxNotional": 5000.0, + "maintenanceMarginRate": 0.01, + "maxLeverage": 25.0, + "info": { + "bracket": "1", + "initialLeverage": "25", + "notionalCap": "5000", + "notionalFloor": "0", + "maintMarginRatio": "0.01", + "cum": "0.0" + } + }, + { + "tier": 2.0, + "currency": "USDT", + "minNotional": 5000.0, + "maxNotional": 25000.0, + "maintenanceMarginRate": 0.025, + "maxLeverage": 20.0, + "info": { + "bracket": "2", + "initialLeverage": "20", + "notionalCap": "25000", + "notionalFloor": "5000", + "maintMarginRatio": "0.025", + "cum": "75.0" + } + }, + { + "tier": 3.0, + "currency": "USDT", + "minNotional": 25000.0, + "maxNotional": 100000.0, + "maintenanceMarginRate": 0.05, + "maxLeverage": 10.0, + "info": { + "bracket": "3", + "initialLeverage": "10", + "notionalCap": "100000", + "notionalFloor": "25000", + "maintMarginRatio": "0.05", + "cum": "700.0" + } + }, + { + "tier": 4.0, + "currency": "USDT", + "minNotional": 100000.0, + "maxNotional": 250000.0, + "maintenanceMarginRate": 0.1, + "maxLeverage": 5.0, + "info": { + "bracket": "4", + "initialLeverage": "5", + "notionalCap": "250000", + "notionalFloor": "100000", + "maintMarginRatio": "0.1", + "cum": "5700.0" + } + }, + { + "tier": 5.0, + "currency": "USDT", + "minNotional": 250000.0, + "maxNotional": 1000000.0, + "maintenanceMarginRate": 0.125, + "maxLeverage": 2.0, + "info": { + "bracket": "5", + "initialLeverage": "2", + "notionalCap": "1000000", + "notionalFloor": "250000", + "maintMarginRatio": "0.125", + "cum": "11950.0" + } + }, + { + "tier": 6.0, + "currency": "USDT", + "minNotional": 1000000.0, + "maxNotional": 5000000.0, + "maintenanceMarginRate": 0.5, + "maxLeverage": 1.0, + "info": { + "bracket": "6", + "initialLeverage": "1", + "notionalCap": "5000000", + "notionalFloor": "1000000", + "maintMarginRatio": "0.5", + "cum": "386950.0" + } + } + ], "STMX/USDT": [ { "tier": 1.0, @@ -16176,13 +16830,13 @@ "tier": 5.0, "currency": "BUSD", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "5", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386875.0" @@ -16470,13 +17124,13 @@ "tier": 6.0, "currency": "USDT", "minNotional": 1000000.0, - "maxNotional": 30000000.0, + "maxNotional": 5000000.0, "maintenanceMarginRate": 0.5, "maxLeverage": 1.0, "info": { "bracket": "6", "initialLeverage": "1", - "notionalCap": "30000000", + "notionalCap": "5000000", "notionalFloor": "1000000", "maintMarginRatio": "0.5", "cum": "386950.0" From 4182a7891a261e0dd7facd605a2667f9d2169412 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:41:48 +0200 Subject: [PATCH 33/87] Allow leverage tier cache to be 4 weeks old. we've seen from binance that it's not changing this often. --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_okx.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 33a56c530..a2ddc16e8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -2304,7 +2304,7 @@ class Exchange: updated = tiers.get('updated') if updated: updated_dt = parser.parse(updated) - if updated_dt < datetime.now(timezone.utc) - timedelta(days=1): + if updated_dt < datetime.now(timezone.utc) - timedelta(weeks=4): logger.info("Cached leverage tiers are outdated. Will update.") return None return tiers['data'] diff --git a/tests/exchange/test_okx.py b/tests/exchange/test_okx.py index 12322acae..ac5c81ebb 100644 --- a/tests/exchange/test_okx.py +++ b/tests/exchange/test_okx.py @@ -472,7 +472,7 @@ def test_load_leverage_tiers_okx(default_conf, mocker, markets, tmpdir, caplog, api_mock.fetch_market_leverage_tiers.call_count == 0 # 2 day passes ... - time_machine.move_to(datetime.now() + timedelta(days=2)) + time_machine.move_to(datetime.now() + timedelta(weeks=5)) exchange.load_leverage_tiers() assert log_has(logmsg, caplog) From 38b28fc4da4caf437d8ccf2bcfd9bc95b79ebe08 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 14:19:20 +0200 Subject: [PATCH 34/87] Update duplicated test --- tests/freqai/test_freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 6aa4f40c3..998bce903 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -320,7 +320,7 @@ def test_principal_component_analysis(mocker, freqai_conf): @pytest.mark.parametrize('timeframes,corr_pairs', [ (['5m'], ['ADA/BTC', 'DASH/BTC']), - (['5m'], ['ADA/BTC', 'DASH/BTC']), + (['5m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), (['5m', '15m'], ['ADA/BTC', 'DASH/BTC', 'ETH/USDT']), ]) def test_freqai_informative_pairs(mocker, freqai_conf, timeframes, corr_pairs): From 8639c1f23d89121d97775daa5157939395f10e07 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 10:56:27 +0200 Subject: [PATCH 35/87] Reduce complexity in binance stoploss handling --- freqtrade/exchange/binance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 026ba1c65..a012aa3a3 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -48,13 +48,12 @@ class Binance(Exchange): Returns True if adjustment is necessary. :param side: "buy" or "sell" """ - - ordertype = 'stop' if self.trading_mode == TradingMode.FUTURES else 'stop_loss_limit' + order_types = ('stop_loss_limit', 'stop', 'stop_market') return ( order.get('stopPrice', None) is None or ( - order['type'] == ordertype + order['type'] in order_types and ( (side == "sell" and stop_loss > float(order['stopPrice'])) or (side == "buy" and stop_loss < float(order['stopPrice'])) From ca6dec3d4c41783c5a91046825301698c308bdb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 14:55:00 +0200 Subject: [PATCH 36/87] Binance spot also allows market orders closes #7426 --- freqtrade/exchange/binance.py | 2 +- tests/exchange/test_binance.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index a012aa3a3..faa780529 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -31,7 +31,7 @@ class Binance(Exchange): "ccxt_futures_name": "future" } _ft_has_futures: Dict = { - "stoploss_order_types": {"limit": "stop"}, + "stoploss_order_types": {"limit": "limit", "market": "market"}, "tickers_have_price": False, } diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 4d1c40647..e9f4dfa8a 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -23,7 +23,7 @@ from tests.exchange.test_exchange import ccxt_exceptionhandlers def test_stoploss_order_binance(default_conf, mocker, limitratio, expected, side, trademode): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop_loss_limit' if trademode == TradingMode.SPOT else 'stop' + order_type = 'stop_loss_limit' if trademode == TradingMode.SPOT else 'limit' api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -45,12 +45,15 @@ def test_stoploss_order_binance(default_conf, mocker, limitratio, expected, side amount=1, stop_price=190, side=side, - order_types={'stoploss_on_exchange_limit_ratio': 1.05}, + order_types={'stoploss': 'limit', 'stoploss_on_exchange_limit_ratio': 1.05}, leverage=1.0 ) api_mock.create_order.reset_mock() - order_types = {} if limitratio is None else {'stoploss_on_exchange_limit_ratio': limitratio} + order_types = {'stoploss': 'limit'} + if limitratio is not None: + order_types.update({'stoploss_on_exchange_limit_ratio': limitratio}) + order = exchange.stoploss( pair='ETH/BTC', amount=1, From 9f266cbcb2232e7d3fc9219caaef4ea21aa85c48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 16:44:08 +0200 Subject: [PATCH 37/87] Allow safe_price for market stop orders --- freqtrade/persistence/trade_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ea60796a4..2e479066c 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -83,7 +83,7 @@ class Order(_DECL_BASE): @property def safe_price(self) -> float: - return self.average or self.price + return self.average or self.price or self.stop_price @property def safe_filled(self) -> float: From 063511826c3c4ed87e1247552e7a676cc0db4afe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 17:07:47 +0200 Subject: [PATCH 38/87] Update stoploss on exchange logic closes #7424 --- freqtrade/freqtradebot.py | 7 ++++--- freqtrade/persistence/trade_model.py | 7 ++++++- tests/test_freqtradebot.py | 11 +++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6c001a8d6..3eaec5c98 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1072,6 +1072,7 @@ class FreqtradeBot(LoggingMixin): order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss') trade.orders.append(order_obj) trade.stoploss_order_id = str(stoploss_order['id']) + trade.stoploss_last_update = datetime.now(timezone.utc) return True except InsufficientFundsError as e: logger.warning(f"Unable to place stoploss order {e}.") @@ -1145,10 +1146,9 @@ class FreqtradeBot(LoggingMixin): if self.create_stoploss_order(trade=trade, stop_price=stop_price): # The above will return False if the placement failed and the trade was force-sold. # in which case the trade will be closed - which we must check below. - trade.stoploss_last_update = datetime.utcnow() return False - # If stoploss order is canceled for some reason we add it + # If stoploss order is canceled for some reason we add it again if (trade.is_open and stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled')): @@ -1186,7 +1186,8 @@ class FreqtradeBot(LoggingMixin): if self.exchange.stoploss_adjust(stoploss_norm, order, side=trade.exit_side): # we check if the update is necessary update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) - if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: + upd_req = datetime.now(timezone.utc) - timedelta(seconds=update_beat) + if trade.stoploss_last_update_utc and upd_req >= trade.stoploss_last_update_utc: # cancelling the current stoploss on exchange first logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} " f"(orderid:{order['id']}) in order to add another one ...") diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 2e479066c..6e421f33e 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -376,6 +376,12 @@ class LocalTrade(): def open_date_utc(self): return self.open_date.replace(tzinfo=timezone.utc) + @property + def stoploss_last_update_utc(self): + if self.stoploss_last_update: + return self.stoploss_last_update.replace(tzinfo=timezone.utc) + return None + @property def close_date_utc(self): return self.close_date.replace(tzinfo=timezone.utc) @@ -560,7 +566,6 @@ class LocalTrade(): self.stop_loss = stop_loss_norm self.stop_loss_pct = -1 * abs(percent) - self.stoploss_last_update = datetime.utcnow() def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False, refresh: bool = False) -> None: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 565797d81..c1152ac09 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1427,6 +1427,7 @@ def test_handle_stoploss_on_exchange_trailing( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 + trade.stoploss_last_update = arrow.utcnow().shift(minutes=-20).datetime stoploss_order_hanging = MagicMock(return_value={ 'id': 100, @@ -1456,7 +1457,7 @@ def test_handle_stoploss_on_exchange_trailing( ) cancel_order_mock = MagicMock() - stoploss_order_mock = MagicMock(return_value={'id': 13434334}) + stoploss_order_mock = MagicMock(return_value={'id': 'so1'}) mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order', cancel_order_mock) mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock) @@ -1569,6 +1570,7 @@ def test_handle_stoploss_on_exchange_trailing_error( assert stoploss.call_count == 1 # Fail creating stoploss order + trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime caplog.clear() cancel_mock = mocker.patch("freqtrade.exchange.Binance.cancel_stoploss_order", MagicMock()) mocker.patch("freqtrade.exchange.Binance.stoploss", side_effect=ExchangeError()) @@ -1657,6 +1659,7 @@ def test_handle_stoploss_on_exchange_custom_stop( trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 + trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime stoploss_order_hanging = MagicMock(return_value={ 'id': 100, @@ -1685,7 +1688,7 @@ def test_handle_stoploss_on_exchange_custom_stop( ) cancel_order_mock = MagicMock() - stoploss_order_mock = MagicMock(return_value={'id': 13434334}) + stoploss_order_mock = MagicMock(return_value={'id': 'so1'}) mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order', cancel_order_mock) mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock) @@ -1727,8 +1730,7 @@ def test_handle_stoploss_on_exchange_custom_stop( assert freqtrade.handle_trade(trade) is True -def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, - limit_order) -> None: +def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_order) -> None: enter_order = limit_order['buy'] exit_order = limit_order['sell'] @@ -1784,6 +1786,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, trade.is_open = True trade.open_order_id = None trade.stoploss_order_id = 100 + trade.stoploss_last_update = arrow.utcnow() stoploss_order_hanging = MagicMock(return_value={ 'id': 100, From 92a32ab31b65f77bd63f9feab6b33b7b8b9028cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Sep 2022 17:14:45 +0200 Subject: [PATCH 39/87] Add documentation for stop-market on binance futures part of #7426 --- docs/exchanges.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index b5470f65a..dc2003f9c 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -57,7 +57,8 @@ This configuration enables kraken, as well as rate-limiting to avoid bans from t Binance supports [time_in_force](configuration.md#understand-order_time_in_force). !!! Tip "Stoploss on Exchange" - Binance supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange.. + Binance supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange. + On futures, Binance supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use. ### Binance Blacklist From 1c92734f39074eb7078f45dc296d18c338467959 Mon Sep 17 00:00:00 2001 From: initrv Date: Sat, 17 Sep 2022 18:53:43 +0300 Subject: [PATCH 40/87] simplify plot_feature_importance call --- freqtrade/freqai/freqai_interface.py | 11 +++----- freqtrade/freqai/utils.py | 38 +++++++++++++--------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 0cc51fdab..3fa8a801b 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -556,14 +556,6 @@ class IFreqaiModel(ABC): model = self.train(unfiltered_dataframe, pair, dk) - if self.freqai_info["feature_parameters"].get("plot_feature_importance", False): - plot_feature_importance( - model=model, - feature_names=dk.training_features_list, - pair=pair, - train_dir=dk.data_path - ) - self.dd.pair_dict[pair]["trained_timestamp"] = new_trained_timerange.stopts dk.set_new_model_names(pair, new_trained_timerange) self.dd.pair_dict[pair]["first"] = False @@ -571,6 +563,9 @@ class IFreqaiModel(ABC): self.dd.pair_to_end_of_training_queue(pair) self.dd.save_data(model, pair, dk) + if self.freqai_info["feature_parameters"].get("plot_feature_importance", False): + plot_feature_importance(model, pair, dk) + if self.freqai_info.get("purge_old_models", False): self.dd.purge_old_models() diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 3f6b8b053..34528acdd 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,6 +1,7 @@ import logging from datetime import datetime, timezone from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -11,6 +12,7 @@ from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.exchange.exchange import market_is_active +from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -138,36 +140,30 @@ def get_required_data_timerange( # ) -def plot_feature_importance(model, feature_names, pair, train_dir, count_max=50) -> None: +def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, + count_max: int = 25) -> None: """ - Plot Best and Worst Features by importance for CatBoost model. - Called once per sub-train. - Usage: plot_feature_importance( - model=model, - feature_names=dk.training_features_list, - pair=pair, - train_dir=dk.data_path) + Plot Best and worst features by importance for a single sub-train. + :param model: Any = A model which was `fit` using a common library + such as catboost or lightgbm + :param pair: str = pair e.g. BTC/USD + :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop + :param count_max: int = the amount of features to be loaded per column """ - try: - import plotly.graph_objects as go - from plotly.subplots import make_subplots - except ImportError: - logger.exception("Module plotly not found \n Please install using `pip3 install plotly`") - exit(1) + from freqtrade.plot.plotting import go, make_subplots, store_plot_file - from freqtrade.plot.plotting import store_plot_file - - # Gather feature importance from model + # Extract feature importance from model if "catboost.core" in str(model.__class__): feature_importance = model.get_feature_importance() elif "lightgbm.sklearn" in str(model.__class__): feature_importance = model.feature_importances_ else: - raise NotImplementedError(f"Cannot extract feature importance for {model.__class__}") + # TODO: Add support for more libraries + raise NotImplementedError(f"Cannot extract feature importance from {model.__class__}") # Data preparation fi_df = pd.DataFrame({ - "feature_names": np.array(feature_names), + "feature_names": np.array(dk.training_features_list), "feature_importance": np.array(feature_importance) }) fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1] @@ -185,9 +181,9 @@ def plot_feature_importance(model, feature_names, pair, train_dir, count_max=50) fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5) fig = add_feature_trace(fig, fi_df_top, 1) fig = add_feature_trace(fig, fi_df_worst, 2) - fig.update_layout(title_text=f"Best and Worst Features {pair}") + fig.update_layout(title_text=f"Best and worst features by importance {pair}") # Store plot file - model_dir, train_name = str(train_dir).rsplit("/", 1) + model_dir, train_name = str(dk.data_path).rsplit("/", 1) fi_dir = Path(f"{model_dir}/feature_importance/{pair.split('/')[0]}") store_plot_file(fig, f"{train_name}.html", fi_dir) From 2c23effbf27adefa9949fe21c462dc3ef5e36b8d Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 17 Sep 2022 19:17:44 +0200 Subject: [PATCH 41/87] allow plot to plot multitargets, add test --- freqtrade/freqai/freqai_interface.py | 2 +- freqtrade/freqai/utils.py | 68 ++++++++++++++------------- freqtrade/plot/plotting.py | 5 +- tests/freqai/test_freqai_interface.py | 35 ++++++++++++++ 4 files changed, 75 insertions(+), 35 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 3fa8a801b..c6b45d61b 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -563,7 +563,7 @@ class IFreqaiModel(ABC): self.dd.pair_to_end_of_training_queue(pair) self.dd.save_data(model, pair, dk) - if self.freqai_info["feature_parameters"].get("plot_feature_importance", False): + if self.freqai_info["feature_parameters"].get("plot_feature_importance", True): plot_feature_importance(model, pair, dk) if self.freqai_info.get("purge_old_models", False): diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 34528acdd..3f278d436 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,6 +1,5 @@ import logging from datetime import datetime, timezone -from pathlib import Path from typing import Any import numpy as np @@ -153,37 +152,42 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, from freqtrade.plot.plotting import go, make_subplots, store_plot_file # Extract feature importance from model - if "catboost.core" in str(model.__class__): - feature_importance = model.get_feature_importance() - elif "lightgbm.sklearn" in str(model.__class__): - feature_importance = model.feature_importances_ - else: - # TODO: Add support for more libraries - raise NotImplementedError(f"Cannot extract feature importance from {model.__class__}") + models = {} + if 'FreqaiMultiOutputRegressor' in str(model.__class__): + for estimator, label in zip(model.estimators_, dk.label_list): + models[label] = estimator - # Data preparation - fi_df = pd.DataFrame({ - "feature_names": np.array(dk.training_features_list), - "feature_importance": np.array(feature_importance) - }) - fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1] - fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1] + for label in models: + mdl = models[label] + if "catboost.core" in str(mdl.__class__): + feature_importance = mdl.get_feature_importance() + elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__): + feature_importance = mdl.feature_importances_ + else: + # TODO: Add support for more libraries + raise NotImplementedError(f"Cannot extract feature importance from {mdl.__class__}") - # Plotting - def add_feature_trace(fig, fi_df, col): - return fig.add_trace( - go.Bar( - x=fi_df["feature_importance"], - y=fi_df["feature_names"], - orientation='h', showlegend=False - ), row=1, col=col - ) - fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5) - fig = add_feature_trace(fig, fi_df_top, 1) - fig = add_feature_trace(fig, fi_df_worst, 2) - fig.update_layout(title_text=f"Best and worst features by importance {pair}") + # Data preparation + fi_df = pd.DataFrame({ + "feature_names": np.array(dk.training_features_list), + "feature_importance": np.array(feature_importance) + }) + fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1] + fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1] - # Store plot file - model_dir, train_name = str(dk.data_path).rsplit("/", 1) - fi_dir = Path(f"{model_dir}/feature_importance/{pair.split('/')[0]}") - store_plot_file(fig, f"{train_name}.html", fi_dir) + # Plotting + def add_feature_trace(fig, fi_df, col): + return fig.add_trace( + go.Bar( + x=fi_df["feature_importance"], + y=fi_df["feature_names"], + orientation='h', showlegend=False + ), row=1, col=col + ) + fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5) + fig = add_feature_trace(fig, fi_df_top, 1) + fig = add_feature_trace(fig, fi_df_worst, 2) + fig.update_layout(title_text=f"Best and worst features by importance {pair}") + + store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path, + include_plotlyjs="cdn") diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f8e95300a..8a00c7899 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -601,7 +601,8 @@ def generate_plot_filename(pair: str, timeframe: str) -> str: return file_name -def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False) -> None: +def store_plot_file(fig, filename: str, directory: Path, + auto_open: bool = False, include_plotlyjs=True) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot @@ -614,7 +615,7 @@ def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False _filename = directory.joinpath(filename) plot(fig, filename=str(_filename), - auto_open=auto_open) + auto_open=auto_open, include_plotlyjs=include_plotlyjs) logger.info(f"Stored plot as {_filename}") diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index a66594d7f..b600cc535 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -315,3 +315,38 @@ def test_principal_component_analysis(mocker, freqai_conf): assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}_pca_object.pkl") shutil.rmtree(Path(freqai.dk.full_path)) + + +def test_plot_feature_importance(mocker, freqai_conf): + + from freqtrade.freqai.utils import plot_feature_importance + + freqai_conf.update({"timerange": "20180110-20180130"}) + freqai_conf.get("freqai", {}).get("feature_parameters", {}).update( + {"princpial_component_analysis": "true"}) + + strategy = get_patched_freqai_strategy(mocker, freqai_conf) + exchange = get_patched_exchange(mocker, freqai_conf) + strategy.dp = DataProvider(freqai_conf, exchange) + strategy.freqai_info = freqai_conf.get("freqai", {}) + freqai = strategy.freqai + freqai.live = True + freqai.dk = FreqaiDataKitchen(freqai_conf) + timerange = TimeRange.parse_timerange("20180110-20180130") + freqai.dd.load_all_pair_histories(timerange, freqai.dk) + + freqai.dd.pair_dict = MagicMock() + + data_load_timerange = TimeRange.parse_timerange("20180110-20180130") + new_timerange = TimeRange.parse_timerange("20180120-20180130") + + freqai.extract_data_and_train_model( + new_timerange, "ADA/BTC", strategy, freqai.dk, data_load_timerange) + + model = freqai.dd.load_data("ADA/BTC", freqai.dk) + + plot_feature_importance(model, "ADA/BTC", freqai.dk) + + assert Path(freqai.dk.data_path / f"{freqai.dk.model_filename}.html") + + shutil.rmtree(Path(freqai.dk.full_path)) From 68f7a315048703285e978dc58e023506153069f2 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 18 Sep 2022 00:00:14 +0200 Subject: [PATCH 42/87] ensure continued operation despite not being able to plot --- freqtrade/freqai/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 3f278d436..10afbaf52 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -156,6 +156,8 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, if 'FreqaiMultiOutputRegressor' in str(model.__class__): for estimator, label in zip(model.estimators_, dk.label_list): models[label] = estimator + else: + models[dk.label_list[0]] for label in models: mdl = models[label] @@ -164,8 +166,8 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__): feature_importance = mdl.feature_importances_ else: - # TODO: Add support for more libraries - raise NotImplementedError(f"Cannot extract feature importance from {mdl.__class__}") + logger.info('Model type not support for generating feature importances.') + return # Data preparation fi_df = pd.DataFrame({ From 1ef875901aec7fabd11f794669d62c1047c009dd Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 18 Sep 2022 00:01:42 +0200 Subject: [PATCH 43/87] maintian user privacy by keeping plotly offline --- freqtrade/freqai/utils.py | 3 +-- freqtrade/plot/plotting.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 10afbaf52..50303019a 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -191,5 +191,4 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, fig = add_feature_trace(fig, fi_df_worst, 2) fig.update_layout(title_text=f"Best and worst features by importance {pair}") - store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path, - include_plotlyjs="cdn") + store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 8a00c7899..752a25d2d 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -602,7 +602,7 @@ def generate_plot_filename(pair: str, timeframe: str) -> str: def store_plot_file(fig, filename: str, directory: Path, - auto_open: bool = False, include_plotlyjs=True) -> None: + auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot @@ -615,7 +615,7 @@ def store_plot_file(fig, filename: str, directory: Path, _filename = directory.joinpath(filename) plot(fig, filename=str(_filename), - auto_open=auto_open, include_plotlyjs=include_plotlyjs) + auto_open=auto_open) logger.info(f"Stored plot as {_filename}") From fa3d4b58ab283bd5d5c490994eceaeec0a747aa3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 08:30:59 +0200 Subject: [PATCH 44/87] Revert unnecessary formatting --- freqtrade/plot/plotting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 752a25d2d..f8e95300a 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -601,8 +601,7 @@ def generate_plot_filename(pair: str, timeframe: str) -> str: return file_name -def store_plot_file(fig, filename: str, directory: Path, - auto_open: bool = False) -> None: +def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False) -> None: """ Generate a plot html file from pre populated fig plotly object :param fig: Plotly Figure to plot From 4634936265cd5d72d609bdb588129f2fa648b07b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 08:39:03 +0200 Subject: [PATCH 45/87] additional support for --data-dir --- freqtrade/commands/cli_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 9aacbcc97..f383f0768 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -69,7 +69,7 @@ AVAILABLE_CLI_OPTIONS = { metavar='PATH', ), "datadir": Arg( - '-d', '--datadir', + '-d', '--datadir', '--data-dir', help='Path to directory with historical backtesting data.', metavar='PATH', ), From ab78fb373af68d1cf39ed11310a00aa051724a6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 08:45:24 +0200 Subject: [PATCH 46/87] Improve freqAI strategy formatting and readability --- freqtrade/templates/FreqaiExampleStrategy.py | 34 +++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 907106453..d71fd91e8 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -170,25 +170,31 @@ class FreqaiExampleStrategy(IStrategy): dataframe = self.freqai.start(dataframe, metadata, self) for val in self.std_dev_multiplier_buy.range: - dataframe[f'target_roi_{val}'] = dataframe["&-s_close_mean"] + \ - dataframe["&-s_close_std"] * val + dataframe[f'target_roi_{val}'] = ( + dataframe["&-s_close_mean"] + dataframe["&-s_close_std"] * val + ) for val in self.std_dev_multiplier_sell.range: - dataframe[f'sell_roi_{val}'] = dataframe["&-s_close_mean"] - \ - dataframe["&-s_close_std"] * val + dataframe[f'sell_roi_{val}'] = ( + dataframe["&-s_close_mean"] - dataframe["&-s_close_std"] * val + ) return dataframe def populate_entry_trend(self, df: DataFrame, metadata: dict) -> DataFrame: - enter_long_conditions = [df["do_predict"] == 1, df["&-s_close"] - > df[f"target_roi_{self.std_dev_multiplier_buy.value}"]] + enter_long_conditions = [ + df["do_predict"] == 1, + df["&-s_close"] > df[f"target_roi_{self.std_dev_multiplier_buy.value}"], + ] if enter_long_conditions: df.loc[ reduce(lambda x, y: x & y, enter_long_conditions), ["enter_long", "enter_tag"] ] = (1, "long") - enter_short_conditions = [df["do_predict"] == 1, df["&-s_close"] - < df[f"sell_roi_{self.std_dev_multiplier_sell.value}"]] + enter_short_conditions = [ + df["do_predict"] == 1, + df["&-s_close"] < df[f"sell_roi_{self.std_dev_multiplier_sell.value}"], + ] if enter_short_conditions: df.loc[ @@ -198,13 +204,17 @@ class FreqaiExampleStrategy(IStrategy): return df def populate_exit_trend(self, df: DataFrame, metadata: dict) -> DataFrame: - exit_long_conditions = [df["do_predict"] == 1, df["&-s_close"] < - df[f"sell_roi_{self.std_dev_multiplier_sell.value}"] * 0.25] + exit_long_conditions = [ + df["do_predict"] == 1, + df["&-s_close"] < df[f"sell_roi_{self.std_dev_multiplier_sell.value}"] * 0.25, + ] if exit_long_conditions: df.loc[reduce(lambda x, y: x & y, exit_long_conditions), "exit_long"] = 1 - exit_short_conditions = [df["do_predict"] == 1, df["&-s_close"] > - df[f"target_roi_{self.std_dev_multiplier_buy.value}"] * 0.25] + exit_short_conditions = [ + df["do_predict"] == 1, + df["&-s_close"] > df[f"target_roi_{self.std_dev_multiplier_buy.value}"] * 0.25, + ] if exit_short_conditions: df.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1 From faf84295a521dd2ff2ea1b87d317039b8a0bce20 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 08:50:41 +0200 Subject: [PATCH 47/87] Separate strategy subtemplates for better overview --- freqtrade/commands/deploy_commands.py | 20 +++++++++---------- .../buy_trend_full.j2 | 0 .../buy_trend_minimal.j2 | 0 .../indicators_full.j2 | 0 .../indicators_minimal.j2 | 0 .../plot_config_full.j2 | 0 .../plot_config_minimal.j2 | 0 .../sell_trend_full.j2 | 0 .../sell_trend_minimal.j2 | 0 .../strategy_methods_advanced.j2 | 0 .../strategy_methods_empty.j2 | 0 tests/test_misc.py | 4 ++-- 12 files changed, 12 insertions(+), 12 deletions(-) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/buy_trend_full.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/buy_trend_minimal.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/indicators_full.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/indicators_minimal.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/plot_config_full.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/plot_config_minimal.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/sell_trend_full.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/sell_trend_minimal.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/strategy_methods_advanced.j2 (100%) rename freqtrade/templates/{subtemplates => strategy_subtemplates}/strategy_methods_empty.j2 (100%) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 92c9adf66..9ec33eac4 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -36,24 +36,24 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st """ fallback = 'full' indicators = render_template_with_fallback( - templatefile=f"subtemplates/indicators_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/indicators_{fallback}.j2", + templatefile=f"strategy_subtemplates/indicators_{subtemplate}.j2", + templatefallbackfile=f"strategy_subtemplates/indicators_{fallback}.j2", ) buy_trend = render_template_with_fallback( - templatefile=f"subtemplates/buy_trend_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2", + templatefile=f"strategy_subtemplates/buy_trend_{subtemplate}.j2", + templatefallbackfile=f"strategy_subtemplates/buy_trend_{fallback}.j2", ) sell_trend = render_template_with_fallback( - templatefile=f"subtemplates/sell_trend_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2", + templatefile=f"strategy_subtemplates/sell_trend_{subtemplate}.j2", + templatefallbackfile=f"strategy_subtemplates/sell_trend_{fallback}.j2", ) plot_config = render_template_with_fallback( - templatefile=f"subtemplates/plot_config_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2", + templatefile=f"strategy_subtemplates/plot_config_{subtemplate}.j2", + templatefallbackfile=f"strategy_subtemplates/plot_config_{fallback}.j2", ) additional_methods = render_template_with_fallback( - templatefile=f"subtemplates/strategy_methods_{subtemplate}.j2", - templatefallbackfile="subtemplates/strategy_methods_empty.j2", + templatefile=f"strategy_subtemplates/strategy_methods_{subtemplate}.j2", + templatefallbackfile="strategy_subtemplates/strategy_methods_empty.j2", ) strategy_text = render_template(templatefile='base_strategy.py.j2', diff --git a/freqtrade/templates/subtemplates/buy_trend_full.j2 b/freqtrade/templates/strategy_subtemplates/buy_trend_full.j2 similarity index 100% rename from freqtrade/templates/subtemplates/buy_trend_full.j2 rename to freqtrade/templates/strategy_subtemplates/buy_trend_full.j2 diff --git a/freqtrade/templates/subtemplates/buy_trend_minimal.j2 b/freqtrade/templates/strategy_subtemplates/buy_trend_minimal.j2 similarity index 100% rename from freqtrade/templates/subtemplates/buy_trend_minimal.j2 rename to freqtrade/templates/strategy_subtemplates/buy_trend_minimal.j2 diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/strategy_subtemplates/indicators_full.j2 similarity index 100% rename from freqtrade/templates/subtemplates/indicators_full.j2 rename to freqtrade/templates/strategy_subtemplates/indicators_full.j2 diff --git a/freqtrade/templates/subtemplates/indicators_minimal.j2 b/freqtrade/templates/strategy_subtemplates/indicators_minimal.j2 similarity index 100% rename from freqtrade/templates/subtemplates/indicators_minimal.j2 rename to freqtrade/templates/strategy_subtemplates/indicators_minimal.j2 diff --git a/freqtrade/templates/subtemplates/plot_config_full.j2 b/freqtrade/templates/strategy_subtemplates/plot_config_full.j2 similarity index 100% rename from freqtrade/templates/subtemplates/plot_config_full.j2 rename to freqtrade/templates/strategy_subtemplates/plot_config_full.j2 diff --git a/freqtrade/templates/subtemplates/plot_config_minimal.j2 b/freqtrade/templates/strategy_subtemplates/plot_config_minimal.j2 similarity index 100% rename from freqtrade/templates/subtemplates/plot_config_minimal.j2 rename to freqtrade/templates/strategy_subtemplates/plot_config_minimal.j2 diff --git a/freqtrade/templates/subtemplates/sell_trend_full.j2 b/freqtrade/templates/strategy_subtemplates/sell_trend_full.j2 similarity index 100% rename from freqtrade/templates/subtemplates/sell_trend_full.j2 rename to freqtrade/templates/strategy_subtemplates/sell_trend_full.j2 diff --git a/freqtrade/templates/subtemplates/sell_trend_minimal.j2 b/freqtrade/templates/strategy_subtemplates/sell_trend_minimal.j2 similarity index 100% rename from freqtrade/templates/subtemplates/sell_trend_minimal.j2 rename to freqtrade/templates/strategy_subtemplates/sell_trend_minimal.j2 diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 similarity index 100% rename from freqtrade/templates/subtemplates/strategy_methods_advanced.j2 rename to freqtrade/templates/strategy_subtemplates/strategy_methods_advanced.j2 diff --git a/freqtrade/templates/subtemplates/strategy_methods_empty.j2 b/freqtrade/templates/strategy_subtemplates/strategy_methods_empty.j2 similarity index 100% rename from freqtrade/templates/subtemplates/strategy_methods_empty.j2 rename to freqtrade/templates/strategy_subtemplates/strategy_methods_empty.j2 diff --git a/tests/test_misc.py b/tests/test_misc.py index 107932be4..4b52079bf 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -184,8 +184,8 @@ def test_render_template_fallback(mocker): templatefile='subtemplates/indicators_does-not-exist.j2',) val = render_template_with_fallback( - templatefile='subtemplates/indicators_does-not-exist.j2', - templatefallbackfile='subtemplates/indicators_minimal.j2', + templatefile='strategy_subtemplates/indicators_does-not-exist.j2', + templatefallbackfile='strategy_subtemplates/indicators_minimal.j2', ) assert isinstance(val, str) assert 'if self.dp' in val From 7a73adb95500f8a2852a113aecfe71cc7071a07b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 08:57:26 +0200 Subject: [PATCH 48/87] Improve default strategy template --- freqtrade/templates/base_strategy.py.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 5a4504687..d8930f65e 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -1,6 +1,6 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement # flake8: noqa: F401 - +# isort: skip_file # --- Do not remove these libs --- import numpy as np # noqa import pandas as pd # noqa @@ -9,13 +9,13 @@ from datetime import datetime # noqa from typing import Optional, Union # noqa from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, - IStrategy, IntParameter) + IntParameter, IStrategy, merge_informative_pair) # -------------------------------- # Add your lib to import here import talib.abstract as ta import pandas_ta as pta -import freqtrade.vendor.qtpylib.indicators as qtpylib +from technical import qtpylib class {{ strategy }}(IStrategy): From 9f23588154b24bb1d35cf0d5e7ef0e7db82a2cd5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 08:58:33 +0200 Subject: [PATCH 49/87] strategy template - remove pointless noqa's --- freqtrade/templates/base_strategy.py.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index d8930f65e..53426b211 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -2,11 +2,11 @@ # flake8: noqa: F401 # isort: skip_file # --- Do not remove these libs --- -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame # noqa -from datetime import datetime # noqa -from typing import Optional, Union # noqa +import numpy as np +import pandas as pd +from pandas import DataFrame +from datetime import datetime +from typing import Optional, Union from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter, IntParameter, IStrategy, merge_informative_pair) From 188f75d8ec312e6d2687fc2442ef2517abfbd62e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 18 Sep 2022 12:49:08 +0200 Subject: [PATCH 50/87] set model in models dict --- freqtrade/freqai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 50303019a..57bc17cdf 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -157,7 +157,7 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, for estimator, label in zip(model.estimators_, dk.label_list): models[label] = estimator else: - models[dk.label_list[0]] + models[dk.label_list[0]] = model for label in models: mdl = models[label] From 667853c50444be2768b7e43d9ab58ec25540d0cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 13:20:36 +0200 Subject: [PATCH 51/87] Use Alias to type config objects --- freqtrade/configuration/check_exchange.py | 4 +-- freqtrade/configuration/configuration.py | 31 ++++++++++--------- .../configuration/deprecated_settings.py | 11 ++++--- .../configuration/directory_operations.py | 6 ++-- freqtrade/configuration/load_config.py | 4 +-- freqtrade/constants.py | 4 ++- freqtrade/data/converter.py | 4 +-- freqtrade/data/dataprovider.py | 4 +-- freqtrade/edge/edge_positioning.py | 5 ++- freqtrade/exchange/exchange.py | 7 +++-- freqtrade/freqai/data_drawer.py | 3 +- freqtrade/freqai/data_kitchen.py | 3 +- freqtrade/freqai/freqai_interface.py | 6 ++-- freqtrade/freqai/utils.py | 9 +++--- freqtrade/freqtradebot.py | 4 +-- freqtrade/loggers.py | 4 +-- freqtrade/optimize/backtesting.py | 4 +-- freqtrade/optimize/edge_cli.py | 4 +-- freqtrade/optimize/hyperopt.py | 6 ++-- freqtrade/optimize/hyperopt_interface.py | 3 +- freqtrade/optimize/hyperopt_tools.py | 4 +-- freqtrade/plugins/pairlist/SpreadFilter.py | 3 +- freqtrade/plugins/pairlist/StaticPairList.py | 3 +- .../plugins/pairlist/VolatilityFilter.py | 4 +-- freqtrade/plugins/pairlist/VolumePairList.py | 4 +-- freqtrade/plugins/protections/iprotection.py | 4 +-- .../plugins/protections/low_profit_pairs.py | 4 +-- .../protections/max_drawdown_protection.py | 4 +-- .../plugins/protections/stoploss_guard.py | 4 +-- freqtrade/rpc/api_server/webserver.py | 7 +++-- freqtrade/rpc/rpc.py | 8 ++--- freqtrade/strategy/hyper.py | 3 +- freqtrade/wallets.py | 4 +-- freqtrade/worker.py | 11 ++++--- 34 files changed, 102 insertions(+), 91 deletions(-) diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 2be13ce4f..c3d859275 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -1,6 +1,6 @@ import logging -from typing import Any, Dict +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, @@ -10,7 +10,7 @@ from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, logger = logging.getLogger(__name__) -def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: +def check_exchange(config: Config, check_for_bad: bool = True) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade :param check_for_bad: if True, check the exchange against the list of known 'bad' diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 7c68ac46c..76105cc4d 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -13,6 +13,7 @@ from freqtrade.configuration.deprecated_settings import process_temporary_deprec from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.environment_vars import enironment_vars_to_dict from freqtrade.configuration.load_config import load_file, load_from_files +from freqtrade.constants import Config from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging @@ -30,10 +31,10 @@ class Configuration: def __init__(self, args: Dict[str, Any], runmode: RunMode = None) -> None: self.args = args - self.config: Optional[Dict[str, Any]] = None + self.config: Optional[Config] = None self.runmode = runmode - def get_config(self) -> Dict[str, Any]: + def get_config(self) -> Config: """ Return the config. Use this method to get the bot config :return: Dict: Bot config @@ -65,7 +66,7 @@ class Configuration: :return: Configuration dictionary """ # Load all configs - config: Dict[str, Any] = load_from_files(self.args.get("config", [])) + config: Config = load_from_files(self.args.get("config", [])) # Load environment variables env_data = enironment_vars_to_dict() @@ -108,7 +109,7 @@ class Configuration: return config - def _process_logging_options(self, config: Dict[str, Any]) -> None: + def _process_logging_options(self, config: Config) -> None: """ Extract information for sys.argv and load logging configuration: the -v/--verbose, --logfile options @@ -121,7 +122,7 @@ class Configuration: setup_logging(config) - def _process_trading_options(self, config: Dict[str, Any]) -> None: + def _process_trading_options(self, config: Config) -> None: if config['runmode'] not in TRADING_MODES: return @@ -137,7 +138,7 @@ class Configuration: logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"') - def _process_common_options(self, config: Dict[str, Any]) -> None: + def _process_common_options(self, config: Config) -> None: # Set strategy if not specified in config and or if it's non default if self.args.get('strategy') or not config.get('strategy'): @@ -161,7 +162,7 @@ class Configuration: if 'sd_notify' in self.args and self.args['sd_notify']: config['internals'].update({'sd_notify': True}) - def _process_datadir_options(self, config: Dict[str, Any]) -> None: + def _process_datadir_options(self, config: Config) -> None: """ Extract information for sys.argv and load directory configurations --user-data, --datadir @@ -195,7 +196,7 @@ class Configuration: config['exportfilename'] = (config['user_data_dir'] / 'backtest_results') - def _process_optimize_options(self, config: Dict[str, Any]) -> None: + def _process_optimize_options(self, config: Config) -> None: # This will override the strategy configuration self._args_to_config(config, argname='timeframe', @@ -380,7 +381,7 @@ class Configuration: self._args_to_config(config, argname="hyperopt_ignore_missing_space", logstring="Paramter --ignore-missing-space detected: {}") - def _process_plot_options(self, config: Dict[str, Any]) -> None: + def _process_plot_options(self, config: Config) -> None: self._args_to_config(config, argname='pairs', logstring='Using pairs {}') @@ -432,7 +433,7 @@ class Configuration: self._args_to_config(config, argname='show_timerange', logstring='Detected --show-timerange') - def _process_data_options(self, config: Dict[str, Any]) -> None: + def _process_data_options(self, config: Config) -> None: self._args_to_config(config, argname='new_pairs_days', logstring='Detected --new-pairs-days: {}') self._args_to_config(config, argname='trading_mode', @@ -443,7 +444,7 @@ class Configuration: self._args_to_config(config, argname='candle_types', logstring='Detected --candle-types: {}') - def _process_analyze_options(self, config: Dict[str, Any]) -> None: + def _process_analyze_options(self, config: Config) -> None: self._args_to_config(config, argname='analysis_groups', logstring='Analysis reason groups: {}') @@ -456,7 +457,7 @@ class Configuration: self._args_to_config(config, argname='indicator_list', logstring='Analysis indicator list: {}') - def _process_runmode(self, config: Dict[str, Any]) -> None: + def _process_runmode(self, config: Config) -> None: self._args_to_config(config, argname='dry_run', logstring='Parameter --dry-run detected, ' @@ -469,7 +470,7 @@ class Configuration: config.update({'runmode': self.runmode}) - def _process_freqai_options(self, config: Dict[str, Any]) -> None: + def _process_freqai_options(self, config: Config) -> None: self._args_to_config(config, argname='freqaimodel', logstring='Using freqaimodel class name: {}') @@ -479,7 +480,7 @@ class Configuration: return - def _args_to_config(self, config: Dict[str, Any], argname: str, + def _args_to_config(self, config: Config, argname: str, logstring: str, logfun: Optional[Callable] = None, deprecated_msg: Optional[str] = None) -> None: """ @@ -502,7 +503,7 @@ class Configuration: if deprecated_msg: warnings.warn(f"DEPRECATED: {deprecated_msg}", DeprecationWarning) - def _resolve_pairs_list(self, config: Dict[str, Any]) -> None: + def _resolve_pairs_list(self, config: Config) -> None: """ Helper for download script. Takes first found: diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index e88383785..46c19a5b2 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -3,15 +3,16 @@ Functions to handle deprecated settings """ import logging -from typing import Any, Dict, Optional +from typing import Optional +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) -def check_conflicting_settings(config: Dict[str, Any], +def check_conflicting_settings(config: Config, section_old: Optional[str], name_old: str, section_new: Optional[str], name_new: str) -> None: section_new_config = config.get(section_new, {}) if section_new else config @@ -28,7 +29,7 @@ def check_conflicting_settings(config: Dict[str, Any], ) -def process_removed_setting(config: Dict[str, Any], +def process_removed_setting(config: Config, section1: str, name1: str, section2: Optional[str], name2: str) -> None: """ @@ -47,7 +48,7 @@ def process_removed_setting(config: Dict[str, Any], ) -def process_deprecated_setting(config: Dict[str, Any], +def process_deprecated_setting(config: Config, section_old: Optional[str], name_old: str, section_new: Optional[str], name_new: str ) -> None: @@ -69,7 +70,7 @@ def process_deprecated_setting(config: Dict[str, Any], del section_old_config[name_old] -def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: +def process_temporary_deprecated_settings(config: Config) -> None: # Kept for future deprecated / moved settings # check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal', diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 771fd53cc..f70310ee1 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -1,16 +1,16 @@ import logging import shutil from pathlib import Path -from typing import Any, Dict, Optional +from typing import Optional -from freqtrade.constants import USER_DATA_FILES +from freqtrade.constants import USER_DATA_FILES, Config from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) -def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Path: +def create_datadir(config: Config, datadir: Optional[str] = None) -> Path: folder = Path(datadir) if datadir else Path(f"{config['user_data_dir']}/data") if not datadir: diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 3fcbd1f2f..6d0321ba0 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List import rapidjson -from freqtrade.constants import MINIMAL_CONFIG +from freqtrade.constants import MINIMAL_CONFIG, Config from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts @@ -80,7 +80,7 @@ def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> Recursively load configuration files if specified. Sub-files are assumed to be relative to the initial config. """ - config: Dict[str, Any] = {} + config: Config = {} if level > 5: raise OperationalException("Config loop detected.") diff --git a/freqtrade/constants.py b/freqtrade/constants.py index bab8c4816..077c4ecbf 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -3,7 +3,7 @@ """ bot constants """ -from typing import List, Literal, Tuple +from typing import Any, Dict, List, Literal, Tuple from freqtrade.enums import CandleType @@ -603,3 +603,5 @@ LongShort = Literal['long', 'short'] EntryExit = Literal['entry', 'exit'] BuySell = Literal['buy', 'sell'] MakerTaker = Literal['maker', 'taker'] + +Config = Dict[str, Any] diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 84c57be41..bdd010af3 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, Config, TradeList from freqtrade.enums import CandleType @@ -263,7 +263,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: def convert_ohlcv_format( - config: Dict[str, Any], + config: Config, convert_from: str, convert_to: str, erase: bool, diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index c6519d2b8..43850ddd9 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -12,7 +12,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe +from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import ExchangeError, OperationalException @@ -28,7 +28,7 @@ MAX_DATAFRAME_CANDLES = 1000 class DataProvider: - def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None: + def __init__(self, config: Config, exchange: Optional[Exchange], pairlists=None) -> None: self._config = config self._exchange = exchange self._pairlists = pairlists diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index af20e1645..45b4cd8f1 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -11,7 +11,7 @@ import utils_find_1st as utf1st from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT +from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT, Config from freqtrade.data.history import get_timerange, load_data, refresh_data from freqtrade.enums import CandleType, ExitType, RunMode from freqtrade.exceptions import OperationalException @@ -42,10 +42,9 @@ class Edge: Author: https://github.com/mishaker """ - config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: + def __init__(self, config: Config, exchange, strategy) -> None: self.config = config self.exchange = exchange diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index a2ddc16e8..c68fc5873 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -21,7 +21,8 @@ from dateutil import parser from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell, - EntryExit, ListPairsWithTimeframes, MakerTaker, PairWithTimeframe) + Config, EntryExit, ListPairsWithTimeframes, MakerTaker, + PairWithTimeframe) from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, @@ -91,7 +92,7 @@ class Exchange: # TradingMode.SPOT always supported and not required in this list ] - def __init__(self, config: Dict[str, Any], validate: bool = True, + def __init__(self, config: Config, validate: bool = True, load_leverage_tiers: bool = False) -> None: """ Initializes this module with the given config, @@ -108,7 +109,7 @@ class Exchange: self._loop_lock = Lock() self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) - self._config: Dict = {} + self._config: Config = {} self._config.update(config) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 1c091f1be..a9fe8bdc9 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -16,6 +16,7 @@ from numpy.typing import NDArray from pandas import DataFrame from freqtrade.configuration import TimeRange +from freqtrade.constants import Config from freqtrade.data.history import load_pair_history from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen @@ -58,7 +59,7 @@ class FreqaiDataDrawer: Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert """ - def __init__(self, full_path: Path, config: dict, follow_mode: bool = False): + def __init__(self, full_path: Path, config: Config, follow_mode: bool = False): self.config = config self.freqai_info = config.get("freqai", {}) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index e96a945eb..d2abd0ad2 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -18,6 +18,7 @@ from sklearn.model_selection import train_test_split from sklearn.neighbors import NearestNeighbors from freqtrade.configuration import TimeRange +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds from freqtrade.strategy.interface import IStrategy @@ -57,7 +58,7 @@ class FreqaiDataKitchen: def __init__( self, - config: Dict[str, Any], + config: Config, live: bool = False, pair: str = "", ): diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 78931bed4..9a2c64cc3 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -14,7 +14,7 @@ from numpy.typing import NDArray from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.constants import DATETIME_PRINT_FORMAT, Config from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds @@ -50,7 +50,7 @@ class IFreqaiModel(ABC): Juha Nykänen @suikula, Wagner Costa @wagnercosta, Johan Vlugt @Jooopieeert """ - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Config) -> None: self.config = config self.assert_config(self.config) @@ -99,7 +99,7 @@ class IFreqaiModel(ABC): """ return ({}) - def assert_config(self, config: Dict[str, Any]) -> None: + def assert_config(self, config: Config) -> None: if not config.get("freqai", {}): raise OperationalException("No freqai parameters found in configuration file.") diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 6a70f050f..063965ded 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -2,6 +2,7 @@ import logging from datetime import datetime, timezone from freqtrade.configuration import TimeRange +from freqtrade.constants import Config from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException @@ -13,7 +14,7 @@ from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist logger = logging.getLogger(__name__) -def download_all_data_for_training(dp: DataProvider, config: dict) -> None: +def download_all_data_for_training(dp: DataProvider, config: Config) -> None: """ Called only once upon start of bot to download the necessary data for populating indicators and training the model. @@ -47,9 +48,7 @@ def download_all_data_for_training(dp: DataProvider, config: dict) -> None: ) -def get_required_data_timerange( - config: dict -) -> TimeRange: +def get_required_data_timerange(config: Config) -> TimeRange: """ Used to compute the required data download time range for auto data-download in FreqAI @@ -86,7 +85,7 @@ def get_required_data_timerange( # Keep below for when we wish to download heterogeneously lengthed data for FreqAI. -# def download_all_data_for_training(dp: DataProvider, config: dict) -> None: +# def download_all_data_for_training(dp: DataProvider, config: Config) -> None: # """ # Called only once upon start of bot to download the necessary data for # populating indicators and training a FreqAI model. diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3eaec5c98..83abc9bc5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -13,7 +13,7 @@ from schedule import Scheduler from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency -from freqtrade.constants import BuySell, LongShort +from freqtrade.constants import BuySell, Config, LongShort from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge @@ -44,7 +44,7 @@ class FreqtradeBot(LoggingMixin): This is from here the bot start its logic. """ - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Config) -> None: """ Init all variables and objects the bot needs to work :param config: configuration dict, you can use Configuration.get_config() diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index e5b6ddbe9..f365053c9 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -2,8 +2,8 @@ import logging import sys from logging import Formatter from logging.handlers import BufferingHandler, RotatingFileHandler, SysLogHandler -from typing import Any, Dict +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException @@ -73,7 +73,7 @@ def setup_logging_pre() -> None: ) -def setup_logging(config: Dict[str, Any]) -> None: +def setup_logging(config: Config) -> None: """ Process -v/--verbose, --logfile options """ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 105851e60..0a05d740d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,7 +15,7 @@ from pandas import DataFrame from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency -from freqtrade.constants import DATETIME_PRINT_FORMAT, LongShort +from freqtrade.constants import DATETIME_PRINT_FORMAT, Config, LongShort from freqtrade.data import history from freqtrade.data.btanalysis import find_existing_backtest_stats, trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes @@ -70,7 +70,7 @@ class Backtesting: backtesting.start() """ - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Config) -> None: LoggingMixin.show_output = False self.config = config diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index aa3b02529..2eb1c53f5 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -4,10 +4,10 @@ This module contains the edge backtesting interface """ import logging -from typing import Any, Dict from freqtrade import constants from freqtrade.configuration import TimeRange, validate_config_consistency +from freqtrade.constants import Config from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.optimize.optimize_reports import generate_edge_table @@ -26,7 +26,7 @@ class EdgeCli: edge.start() """ - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Config) -> None: self.config = config # Ensure using dry-run diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f15e0b7d8..aef8405d5 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -21,7 +21,7 @@ from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_ from joblib.externals import cloudpickle from pandas import DataFrame -from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN +from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange from freqtrade.enums import HyperoptState @@ -66,7 +66,7 @@ class Hyperopt: hyperopt.start() """ - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Config) -> None: self.buy_space: List[Dimension] = [] self.sell_space: List[Dimension] = [] self.protection_space: List[Dimension] = [] @@ -132,7 +132,7 @@ class Hyperopt: self.print_json = self.config.get('print_json', False) @staticmethod - def get_lock_filename(config: Dict[str, Any]) -> str: + def get_lock_filename(config: Config) -> str: return str(config['user_data_dir'] / 'hyperopt.lock') diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index b1c68caca..a7c64ffb0 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -10,6 +10,7 @@ from typing import Dict, List, Union from sklearn.base import RegressorMixin from skopt.space import Categorical, Dimension, Integer +from freqtrade.constants import Config from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict from freqtrade.optimize.space import SKDecimal @@ -32,7 +33,7 @@ class IHyperOpt(ABC): timeframe: str strategy: IStrategy - def __init__(self, config: dict) -> None: + def __init__(self, config: Config) -> None: self.config = config # Assign timeframe to be used in hyperopt diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 9b022d519..d1f776e3d 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -12,7 +12,7 @@ import tabulate from colorama import Fore, Style from pandas import isna, json_normalize -from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES +from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES, Config from freqtrade.enums import HyperoptState from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 @@ -45,7 +45,7 @@ class HyperoptStateContainer(): class HyperoptTools(): @staticmethod - def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]: + def get_strategy_filename(config: Config, strategy_name: str) -> Optional[Path]: """ Get Strategy-location (filename) from strategy_name """ diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 43856b451..1f20af305 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -4,6 +4,7 @@ Spread pair list filter import logging from typing import Any, Dict +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -14,7 +15,7 @@ logger = logging.getLogger(__name__) class SpreadFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index 30fa474e4..83a0fa0c8 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -7,6 +7,7 @@ import logging from copy import deepcopy from typing import Any, Dict, List +from freqtrade.constants import Config from freqtrade.plugins.pairlist.IPairList import IPairList @@ -16,7 +17,7 @@ logger = logging.getLogger(__name__) class StaticPairList(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index bab44bdd1..c9af3a7b3 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -11,7 +11,7 @@ import numpy as np from cachetools import TTLCache from pandas import DataFrame -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -26,7 +26,7 @@ class VolatilityFilter(IPairList): """ def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index d7cc6e5ec..9dcada291 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List from cachetools import TTLCache -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.misc import format_ms_time @@ -25,7 +25,7 @@ SORT_VALUES = ['quoteVolume'] class VolumePairList(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 890988226..8e1589217 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional -from freqtrade.constants import LongShort +from freqtrade.constants import Config, LongShort from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import plural from freqtrade.mixins import LoggingMixin @@ -30,7 +30,7 @@ class IProtection(LoggingMixin, ABC): # Can stop trading for one pair has_local_stop: bool = False - def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + def __init__(self, config: Config, protection_config: Dict[str, Any]) -> None: self._config = config self._protection_config = protection_config self._stop_duration_candles: Optional[int] = None diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 099242b8d..f638673fa 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -3,7 +3,7 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict, Optional -from freqtrade.constants import LongShort +from freqtrade.constants import Config, LongShort from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -16,7 +16,7 @@ class LowProfitPairs(IProtection): has_global_stop: bool = False has_local_stop: bool = True - def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + def __init__(self, config: Config, protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) self._trade_limit = protection_config.get('trade_limit', 1) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index e0b016cb8..8193dc7e4 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Optional import pandas as pd -from freqtrade.constants import LongShort +from freqtrade.constants import Config, LongShort from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -19,7 +19,7 @@ class MaxDrawdown(IProtection): has_global_stop: bool = True has_local_stop: bool = False - def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + def __init__(self, config: Config, protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) self._trade_limit = protection_config.get('trade_limit', 1) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index e80d13e9d..23ceebbc9 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,7 +3,7 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict, Optional -from freqtrade.constants import LongShort +from freqtrade.constants import Config, LongShort from freqtrade.enums import ExitType from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -17,7 +17,7 @@ class StoplossGuard(IProtection): has_global_stop: bool = True has_local_stop: bool = True - def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + def __init__(self, config: Config, protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) self._trade_limit = protection_config.get('trade_limit', 10) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 0da129583..642f25e47 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.responses import JSONResponse +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler @@ -37,10 +38,10 @@ class ApiServer(RPCHandler): _bt = None _bt_data = None _bt_timerange = None - _bt_last_config: Dict[str, Any] = {} + _bt_last_config: Config = {} _has_rpc: bool = False _bgtask_running: bool = False - _config: Dict[str, Any] = {} + _config: Config = {} # Exchange - only available in webserver mode. _exchange = None @@ -54,7 +55,7 @@ class ApiServer(RPCHandler): ApiServer.__initialized = False return ApiServer.__instance - def __init__(self, config: Dict[str, Any], standalone: bool = False) -> None: + def __init__(self, config: Config, standalone: bool = False) -> None: ApiServer._config = config if self.__initialized and (standalone or self._standalone): return diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 05599074c..6602cdd35 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -16,7 +16,7 @@ from pandas import DataFrame, NaT from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange -from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT +from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT, Config from freqtrade.data.history import load_data from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State, @@ -58,7 +58,7 @@ class RPCException(Exception): class RPCHandler: - def __init__(self, rpc: 'RPC', config: Dict[str, Any]) -> None: + def __init__(self, rpc: 'RPC', config: Config) -> None: """ Initializes RPCHandlers :param rpc: instance of RPC Helper class @@ -66,7 +66,7 @@ class RPCHandler: :return: None """ self._rpc = rpc - self._config: Dict[str, Any] = config + self._config: Config = config @property def name(self) -> str: @@ -96,7 +96,7 @@ class RPC: :return: None """ self._freqtrade = freqtrade - self._config: Dict[str, Any] = freqtrade.config + self._config: Config = freqtrade.config if self._config.get('fiat_display_currency'): self._fiat_converter = CryptoToFiatConverter() diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 47377f238..6f62c9d3d 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -6,6 +6,7 @@ import logging from pathlib import Path from typing import Any, Dict, Iterator, List, Tuple, Type, Union +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.optimize.hyperopt_tools import HyperoptTools @@ -21,7 +22,7 @@ class HyperStrategyMixin: strategy logic. """ - def __init__(self, config: Dict[str, Any], *args, **kwargs): + def __init__(self, config: Config, *args, **kwargs): """ Initialize hyperoptable strategy mixin. """ diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 41115c72e..0a9ecc638 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -7,7 +7,7 @@ from typing import Dict, NamedTuple, Optional import arrow -from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.constants import UNLIMITED_STAKE_AMOUNT, Config from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange @@ -35,7 +35,7 @@ class PositionWallet(NamedTuple): class Wallets: - def __init__(self, config: dict, exchange: Exchange, log: bool = True) -> None: + def __init__(self, config: Config, exchange: Exchange, log: bool = True) -> None: self._config = config self._log = log self._exchange = exchange diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 66f718af0..dea0acc44 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -9,8 +9,9 @@ from typing import Any, Callable, Dict, Optional import sdnotify -from freqtrade import __version__, constants +from freqtrade import __version__ from freqtrade.configuration import Configuration +from freqtrade.constants import PROCESS_THROTTLE_SECS, RETRY_TIMEOUT, Config from freqtrade.enums import State from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot @@ -24,7 +25,7 @@ class Worker: Freqtradebot worker class """ - def __init__(self, args: Dict[str, Any], config: Dict[str, Any] = None) -> None: + def __init__(self, args: Dict[str, Any], config: Config = None) -> None: """ Init all variables and objects the bot needs to work """ @@ -53,7 +54,7 @@ class Worker: internals_config = self._config.get('internals', {}) self._throttle_secs = internals_config.get('process_throttle_secs', - constants.PROCESS_THROTTLE_SECS) + PROCESS_THROTTLE_SECS) self._heartbeat_interval = internals_config.get('heartbeat_interval', 60) self._sd_notify = sdnotify.SystemdNotifier() if \ @@ -151,8 +152,8 @@ class Worker: try: self.freqtrade.process() except TemporaryError as error: - logger.warning(f"Error: {error}, retrying in {constants.RETRY_TIMEOUT} seconds...") - time.sleep(constants.RETRY_TIMEOUT) + logger.warning(f"Error: {error}, retrying in {RETRY_TIMEOUT} seconds...") + time.sleep(RETRY_TIMEOUT) except OperationalException: tb = traceback.format_exc() hint = 'Issue `/start` if you think it is safe to restart.' From 994c1c5ea09b77a046c696eb03496389694aa378 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 13:31:52 +0200 Subject: [PATCH 52/87] use Config typing in more places --- docs/advanced-hyperopt.md | 3 ++- freqtrade/data/converter.py | 4 ++-- .../optimize/hyperopt_loss/hyperopt_loss_calmar.py | 3 ++- .../hyperopt_loss_max_drawdown_relative.py | 5 ++--- freqtrade/optimize/hyperopt_loss_interface.py | 4 +++- freqtrade/optimize/hyperopt_tools.py | 10 +++++----- freqtrade/optimize/optimize_reports.py | 7 ++++--- freqtrade/plot/plotting.py | 7 ++++--- freqtrade/plugins/pairlist/AgeFilter.py | 4 ++-- freqtrade/plugins/pairlist/IPairList.py | 3 ++- freqtrade/plugins/pairlist/OffsetFilter.py | 3 ++- freqtrade/plugins/pairlist/PerformanceFilter.py | 3 ++- freqtrade/plugins/pairlist/PrecisionFilter.py | 3 ++- freqtrade/plugins/pairlist/PriceFilter.py | 3 ++- freqtrade/plugins/pairlist/ShuffleFilter.py | 3 ++- freqtrade/plugins/pairlist/pairlist_helpers.py | 6 ++++-- freqtrade/plugins/pairlist/rangestabilityfilter.py | 4 ++-- freqtrade/plugins/pairlistmanager.py | 4 ++-- freqtrade/plugins/protectionmanager.py | 4 ++-- freqtrade/resolvers/exchange_resolver.py | 3 ++- freqtrade/resolvers/freqaimodel_resolver.py | 5 ++--- freqtrade/resolvers/hyperopt_resolver.py | 5 ++--- freqtrade/resolvers/iresolver.py | 5 +++-- freqtrade/resolvers/pairlist_resolver.py | 3 ++- freqtrade/resolvers/protection_resolver.py | 4 +++- freqtrade/resolvers/strategy_resolver.py | 11 +++++------ freqtrade/rpc/discord.py | 4 ++-- freqtrade/rpc/rpc_manager.py | 3 ++- freqtrade/rpc/telegram.py | 4 ++-- freqtrade/rpc/webhook.py | 3 ++- freqtrade/strategy/interface.py | 4 ++-- freqtrade/templates/sample_hyperopt_loss.py | 3 ++- 32 files changed, 79 insertions(+), 61 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 8a1ebaff3..9933628d1 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -17,6 +17,7 @@ from typing import Any, Dict from pandas import DataFrame +from freqtrade.constants import Config from freqtrade.optimize.hyperopt import IHyperOptLoss TARGET_TRADES = 600 @@ -31,7 +32,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, - config: Dict, processed: Dict[str, DataFrame], + config: Config, processed: Dict[str, DataFrame], backtest_stats: Dict[str, Any], *args, **kwargs) -> float: """ diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index bdd010af3..bf19c1310 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -5,7 +5,7 @@ import itertools import logging from datetime import datetime, timezone from operator import itemgetter -from typing import Any, Dict, List +from typing import Dict, List import pandas as pd from pandas import DataFrame, to_datetime @@ -237,7 +237,7 @@ def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS] -def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): +def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): """ Convert trades from one format to another format. :param config: Config dictionary diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py index ea6c151e5..2b591824f 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_calmar.py @@ -10,6 +10,7 @@ from typing import Any, Dict from pandas import DataFrame +from freqtrade.constants import Config from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -27,7 +28,7 @@ class CalmarHyperOptLoss(IHyperOptLoss): trade_count: int, min_date: datetime, max_date: datetime, - config: Dict, + config: Config, processed: Dict[str, DataFrame], backtest_stats: Dict[str, Any], *args, diff --git a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py index 3182afb47..669d12ddf 100644 --- a/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py +++ b/freqtrade/optimize/hyperopt_loss/hyperopt_loss_max_drawdown_relative.py @@ -4,10 +4,9 @@ MaxDrawDownRelativeHyperOptLoss This module defines the alternative HyperOptLoss class which can be used for Hyperoptimization. """ -from typing import Dict - from pandas import DataFrame +from freqtrade.constants import Config from freqtrade.data.metrics import calculate_underwater from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -22,7 +21,7 @@ class MaxDrawDownRelativeHyperOptLoss(IHyperOptLoss): """ @staticmethod - def hyperopt_loss_function(results: DataFrame, config: Dict, + def hyperopt_loss_function(results: DataFrame, config: Config, *args, **kwargs) -> float: """ diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index 8366dcc4f..d7b30dfd3 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -9,6 +9,8 @@ from typing import Any, Dict from pandas import DataFrame +from freqtrade.constants import Config + class IHyperOptLoss(ABC): """ @@ -21,7 +23,7 @@ class IHyperOptLoss(ABC): @abstractmethod def hyperopt_loss_function(*, results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, - config: Dict, processed: Dict[str, DataFrame], + config: Config, processed: Dict[str, DataFrame], backtest_stats: Dict[str, Any], **kwargs) -> float: """ diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index d1f776e3d..65bdc4db5 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -81,7 +81,7 @@ class HyperoptTools(): ) @staticmethod - def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict): + def try_export_params(config: Config, strategy_name: str, params: Dict): if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): # Export parameters ... fn = HyperoptTools.get_strategy_filename(config, strategy_name) @@ -91,7 +91,7 @@ class HyperoptTools(): logger.warning("Strategy not found, not exporting parameter file.") @staticmethod - def has_space(config: Dict[str, Any], space: str) -> bool: + def has_space(config: Config, space: str) -> bool: """ Tell if the space value is contained in the configuration """ @@ -131,7 +131,7 @@ class HyperoptTools(): return False @staticmethod - def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]: + def load_filtered_results(results_file: Path, config: Config) -> Tuple[List, int]: filteroptions = { 'only_best': config.get('hyperopt_list_best', False), 'only_profitable': config.get('hyperopt_list_profitable', False), @@ -346,7 +346,7 @@ class HyperoptTools(): return trials @staticmethod - def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + def get_result_table(config: Config, results: list, total_epochs: int, highlight_best: bool, print_colorized: bool, remove_header: int) -> str: """ Log result table @@ -444,7 +444,7 @@ class HyperoptTools(): return table @staticmethod - def export_csv_file(config: dict, results: list, csv_file: str) -> None: + def export_csv_file(config: Config, results: list, csv_file: str) -> None: """ Log result to csv-file """ diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index fa6c3f161..6c4dbcfef 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -7,7 +7,8 @@ from typing import Any, Dict, List, Union from pandas import DataFrame, to_datetime from tabulate import tabulate -from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT +from freqtrade.constants import (DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT, + Config) from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change, calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value @@ -898,7 +899,7 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: print() -def show_backtest_results(config: Dict, backtest_stats: Dict): +def show_backtest_results(config: Config, backtest_stats: Dict): stake_currency = config['stake_currency'] for strategy, results in backtest_stats['strategy'].items(): @@ -918,7 +919,7 @@ def show_backtest_results(config: Dict, backtest_stats: Dict): print('\nFor more details, please look at the detail tables above') -def show_sorted_pairlist(config: Dict, backtest_stats: Dict): +def show_sorted_pairlist(config: Config, backtest_stats: Dict): if config.get('backtest_show_pair_list', False): for strategy, results in backtest_stats['strategy'].items(): print(f"Pairs for Strategy {strategy}: \n[") diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f8e95300a..9c8787242 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -1,10 +1,11 @@ import logging from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional import pandas as pd from freqtrade.configuration import TimeRange +from freqtrade.constants import Config from freqtrade.data.btanalysis import (analyze_trade_parallelism, extract_trades_of_period, load_trades) from freqtrade.data.converter import trim_dataframe @@ -618,7 +619,7 @@ def store_plot_file(fig, filename: str, directory: Path, auto_open: bool = False logger.info(f"Stored plot as {_filename}") -def load_and_plot_trades(config: Dict[str, Any]): +def load_and_plot_trades(config: Config): """ From configuration provided - Initializes plot-script @@ -666,7 +667,7 @@ def load_and_plot_trades(config: Dict[str, Any]): logger.info('End of plotting process. %s plots generated', pair_counter) -def plot_profit(config: Dict[str, Any]) -> None: +def plot_profit(config: Config) -> None: """ Plots the total profit for all pairs. Note, the profit calculation isn't realistic. diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 13c992c87..70638936a 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional import arrow from pandas import DataFrame -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) class AgeFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 0155f918b..c02ba5ef5 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, Dict, List +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange, market_is_active from freqtrade.mixins import LoggingMixin @@ -17,7 +18,7 @@ logger = logging.getLogger(__name__) class IPairList(LoggingMixin, ABC): def __init__(self, exchange: Exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: """ :param exchange: Exchange instance diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index e0f8414ef..4200443c7 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -3,6 +3,7 @@ Offset pair list filter """ import logging from typing import Any, Dict, List +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -14,7 +15,7 @@ logger = logging.getLogger(__name__) class OffsetFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 8e0b407c3..4ba96231e 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -5,6 +5,7 @@ import logging from typing import Any, Dict, List import pandas as pd +from freqtrade.constants import Config from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.IPairList import IPairList @@ -16,7 +17,7 @@ logger = logging.getLogger(__name__) class PerformanceFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 61150f03d..7b8b1a30d 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -3,6 +3,7 @@ Precision pair list filter """ import logging from typing import Any, Dict +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -14,7 +15,7 @@ logger = logging.getLogger(__name__) class PrecisionFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 009789eaf..e38090e56 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -3,6 +3,7 @@ Price pair list filter """ import logging from typing import Any, Dict +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -14,7 +15,7 @@ logger = logging.getLogger(__name__) class PriceFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 663bba49b..098108949 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -4,6 +4,7 @@ Shuffle pair list filter import logging import random from typing import Any, Dict, List +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.plugins.pairlist.IPairList import IPairList @@ -15,7 +16,7 @@ logger = logging.getLogger(__name__) class ShuffleFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 0cec734fb..9ef3e4614 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -1,5 +1,7 @@ import re -from typing import Any, Dict, List +from typing import List + +from freqtrade.constants import Config def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], @@ -42,7 +44,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], return result -def dynamic_expand_pairlist(config: Dict[str, Any], markets: List[str]) -> List[str]: +def dynamic_expand_pairlist(config: Config, markets: List[str]) -> List[str]: expanded_pairs = expand_pairlist(config['pairs'], markets) if config.get('freqai', {}).get('enabled', False): corr_pairlist = config['freqai']['feature_parameters']['include_corr_pairlist'] diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f3e7bc0d6..0bc2cdb47 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -9,7 +9,7 @@ import arrow from cachetools import TTLCache from pandas import DataFrame -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) class RangeStabilityFilter(IPairList): def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], + config: Config, pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 3ddad4a5e..e01abb297 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -7,7 +7,7 @@ from typing import Dict, List from cachetools import TTLCache, cached -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.mixins import LoggingMixin @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) class PairListManager(LoggingMixin): - def __init__(self, exchange, config: dict) -> None: + def __init__(self, exchange, config: Config) -> None: self._exchange = exchange self._config = config self._whitelist = self._config['exchange'].get('pair_whitelist') diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index d33294fa7..54432e677 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -5,7 +5,7 @@ import logging from datetime import datetime, timezone from typing import Dict, List, Optional -from freqtrade.constants import LongShort +from freqtrade.constants import Config, LongShort from freqtrade.persistence import PairLocks from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) class ProtectionManager(): - def __init__(self, config: Dict, protections: List) -> None: + def __init__(self, config: Config, protections: List) -> None: self._config = config self._protection_handlers: List[IProtection] = [] diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index a2f572ff2..d7a1a22e2 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -2,6 +2,7 @@ This module loads custom exchanges """ import logging +from freqtrade.constants import Config import freqtrade.exchange as exchanges from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, Exchange @@ -18,7 +19,7 @@ class ExchangeResolver(IResolver): object_type = Exchange @staticmethod - def load_exchange(exchange_name: str, config: dict, validate: bool = True, + def load_exchange(exchange_name: str, config: Config, validate: bool = True, load_leverage_tiers: bool = False) -> Exchange: """ Load the custom class from config parameter diff --git a/freqtrade/resolvers/freqaimodel_resolver.py b/freqtrade/resolvers/freqaimodel_resolver.py index 5a847bb2b..aa5228ca1 100644 --- a/freqtrade/resolvers/freqaimodel_resolver.py +++ b/freqtrade/resolvers/freqaimodel_resolver.py @@ -5,9 +5,8 @@ This module load a custom model for freqai """ import logging from pathlib import Path -from typing import Dict -from freqtrade.constants import USERPATH_FREQAIMODELS +from freqtrade.constants import USERPATH_FREQAIMODELS, Config from freqtrade.exceptions import OperationalException from freqtrade.freqai.freqai_interface import IFreqaiModel from freqtrade.resolvers import IResolver @@ -29,7 +28,7 @@ class FreqaiModelResolver(IResolver): ) @staticmethod - def load_freqaimodel(config: Dict) -> IFreqaiModel: + def load_freqaimodel(config: Config) -> IFreqaiModel: """ Load the custom class from config parameter :param config: configuration dictionary diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index bcfe5e1d8..d050c6fbc 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -5,9 +5,8 @@ This module load custom hyperopt """ import logging from pathlib import Path -from typing import Dict -from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS +from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS, Config from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver @@ -26,7 +25,7 @@ class HyperOptLossResolver(IResolver): initial_search_path = Path(__file__).parent.parent.joinpath('optimize/hyperopt_loss').resolve() @staticmethod - def load_hyperoptloss(config: Dict) -> IHyperOptLoss: + def load_hyperoptloss(config: Config) -> IHyperOptLoss: """ Load the custom class from config parameter :param config: configuration dictionary diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index b99e7a94b..9682e1c2b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -10,6 +10,7 @@ import sys from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException @@ -43,7 +44,7 @@ class IResolver: initial_search_path: Optional[Path] @classmethod - def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, + def build_search_paths(cls, config: Config, user_subdir: Optional[str] = None, extra_dirs: List[str] = []) -> List[Path]: abs_paths: List[Path] = [] @@ -153,7 +154,7 @@ class IResolver: return None @classmethod - def load_object(cls, object_name: str, config: dict, *, kwargs: dict, + def load_object(cls, object_name: str, config: Config, *, kwargs: dict, extra_dir: Optional[str] = None) -> Any: """ Search and loads the specified object as configured in hte child class. diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 72a3cc1dd..f492bcb54 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -6,6 +6,7 @@ This module load custom pairlists import logging from pathlib import Path +from freqtrade.constants import Config from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.resolvers import IResolver @@ -24,7 +25,7 @@ class PairListResolver(IResolver): @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, - config: dict, pairlistconfig: dict, pairlist_pos: int) -> IPairList: + config: Config, pairlistconfig: dict, pairlist_pos: int) -> IPairList: """ Load the pairlist with pairlist_name :param pairlist_name: Classname of the pairlist diff --git a/freqtrade/resolvers/protection_resolver.py b/freqtrade/resolvers/protection_resolver.py index c54ae1011..11cd6f224 100644 --- a/freqtrade/resolvers/protection_resolver.py +++ b/freqtrade/resolvers/protection_resolver.py @@ -5,6 +5,7 @@ import logging from pathlib import Path from typing import Dict +from freqtrade.constants import Config from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import IResolver @@ -22,7 +23,8 @@ class ProtectionResolver(IResolver): initial_search_path = Path(__file__).parent.parent.joinpath('plugins/protections').resolve() @staticmethod - def load_protection(protection_name: str, config: Dict, protection_config: Dict) -> IProtection: + def load_protection(protection_name: str, config: Config, + protection_config: Dict) -> IProtection: """ Load the protection with protection_name :param protection_name: Classname of the pairlist diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 8b01980ce..c574246ac 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -9,10 +9,10 @@ from base64 import urlsafe_b64decode from inspect import getfullargspec from os import walk from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, List, Optional from freqtrade.configuration.config_validation import validate_migrated_strategy_settings -from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES +from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES, Config from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from freqtrade.resolvers import IResolver @@ -32,7 +32,7 @@ class StrategyResolver(IResolver): initial_search_path = None @staticmethod - def load_strategy(config: Dict[str, Any] = None) -> IStrategy: + def load_strategy(config: Config = None) -> IStrategy: """ Load the custom class from config parameter :param config: configuration dictionary or None @@ -91,8 +91,7 @@ class StrategyResolver(IResolver): return strategy @staticmethod - def _override_attribute_helper(strategy, config: Dict[str, Any], - attribute: str, default: Any): + def _override_attribute_helper(strategy, config: Config, attribute: str, default: Any): """ Override attributes in the strategy. Prevalence: @@ -215,7 +214,7 @@ class StrategyResolver(IResolver): @staticmethod def _load_strategy(strategy_name: str, - config: dict, extra_dir: Optional[str] = None) -> IStrategy: + config: Config, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index 85acfae4e..9efe6f427 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -1,6 +1,6 @@ import logging -from typing import Any, Dict +from freqtrade.constants import Config from freqtrade.enums import RPCMessageType from freqtrade.rpc import RPC from freqtrade.rpc.webhook import Webhook @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) class Discord(Webhook): - def __init__(self, rpc: 'RPC', config: Dict[str, Any]): + def __init__(self, rpc: 'RPC', config: Config): # super().__init__(rpc, config) self.rpc = rpc self.config = config diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 3ccf23228..3b60077ad 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -5,6 +5,7 @@ import logging from collections import deque from typing import Any, Dict, List +from freqtrade.constants import Config from freqtrade.enums import RPCMessageType from freqtrade.rpc import RPC, RPCHandler @@ -89,7 +90,7 @@ class RPCManager: 'msg': msg, }) - def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None: + def startup_messages(self, config: Config, pairlist, protections) -> None: if config['dry_run']: self.send_msg({ 'type': RPCMessageType.WARNING, diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4a759f6ec..c40bdb963 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -24,7 +24,7 @@ from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ -from freqtrade.constants import DUST_PER_COIN +from freqtrade.constants import DUST_PER_COIN, Config from freqtrade.enums import RPCMessageType, SignalDirection, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.misc import chunks, plural, round_coin_value @@ -88,7 +88,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: class Telegram(RPCHandler): """ This class handles all telegram communication """ - def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: + def __init__(self, rpc: RPC, config: Config) -> None: """ Init the Telegram call, and init the super class RPCHandler :param rpc: instance of RPC Helper class diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 1b39a29b7..6109e80bc 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -7,6 +7,7 @@ from typing import Any, Dict from requests import RequestException, post +from freqtrade.constants import Config from freqtrade.enums import RPCMessageType from freqtrade.rpc import RPC, RPCHandler @@ -19,7 +20,7 @@ logger.debug('Included module rpc.webhook ...') class Webhook(RPCHandler): """ This class handles all webhook communication """ - def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: + def __init__(self, rpc: RPC, config: Config) -> None: """ Init the Webhook class, and init the super class RPCHandler :param rpc: instance of RPC Helper class diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 93988ac48..5e765e85b 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -10,7 +10,7 @@ from typing import Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame -from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.constants import Config, ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RunMode, SignalDirection, SignalTagType, SignalType, TradingMode) @@ -118,7 +118,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Definition of plot_config. See plotting documentation for more details. plot_config: Dict = {} - def __init__(self, config: dict) -> None: + def __init__(self, config: Config) -> None: self.config = config # Dict to determine if analysis is necessary self._last_candle_seen_per_pair: Dict[str, datetime] = {} diff --git a/freqtrade/templates/sample_hyperopt_loss.py b/freqtrade/templates/sample_hyperopt_loss.py index 343349508..5eab92a0c 100644 --- a/freqtrade/templates/sample_hyperopt_loss.py +++ b/freqtrade/templates/sample_hyperopt_loss.py @@ -4,6 +4,7 @@ from typing import Dict from pandas import DataFrame +from freqtrade.constants import Config from freqtrade.optimize.hyperopt import IHyperOptLoss @@ -36,7 +37,7 @@ class SampleHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, - config: Dict, processed: Dict[str, DataFrame], + config: Config, processed: Dict[str, DataFrame], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results From 95457d23cae87da5fe11c4f053bb0e1c2b399541 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 18 Sep 2022 13:59:30 +0200 Subject: [PATCH 53/87] escape freqai-specific characters from file naming --- freqtrade/freqai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index 57bc17cdf..19a3098cc 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -190,5 +190,5 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen, fig = add_feature_trace(fig, fi_df_top, 1) fig = add_feature_trace(fig, fi_df_worst, 2) fig.update_layout(title_text=f"Best and worst features by importance {pair}") - + label = label.replace('&', '').replace('%', '') # escape two FreqAI specific characters store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path) From eaa43337d2d7c13eeeb8c809d212e047f5935470 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 18 Sep 2022 17:00:55 +0200 Subject: [PATCH 54/87] improve train queue system, ensure crash resilience in train queue. --- freqtrade/freqai/data_drawer.py | 14 +--- freqtrade/freqai/freqai_interface.py | 71 +++++++++++++------- freqtrade/templates/FreqaiExampleStrategy.py | 2 +- 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 1c091f1be..67daa626d 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -27,9 +27,7 @@ logger = logging.getLogger(__name__) class pair_info(TypedDict): model_filename: str - first: bool trained_timestamp: int - priority: int data_path: str extras: dict @@ -91,7 +89,7 @@ class FreqaiDataDrawer: self.old_DBSCAN_eps: Dict[str, float] = {} self.empty_pair_dict: pair_info = { "model_filename": "", "trained_timestamp": 0, - "priority": 1, "first": True, "data_path": "", "extras": {}} + "data_path": "", "extras": {}} def load_drawer_from_disk(self): """ @@ -216,7 +214,6 @@ class FreqaiDataDrawer: self.pair_dict[pair] = self.empty_pair_dict.copy() model_filename = "" trained_timestamp = 0 - self.pair_dict[pair]["priority"] = len(self.pair_dict) if not data_path_set and self.follow_mode: logger.warning( @@ -236,18 +233,9 @@ class FreqaiDataDrawer: return else: self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy() - self.pair_dict[metadata["pair"]]["priority"] = len(self.pair_dict) return - def pair_to_end_of_training_queue(self, pair: str) -> None: - # march all pairs up in the queue - with self.pair_dict_lock: - for p in self.pair_dict: - self.pair_dict[p]["priority"] -= 1 - # send pair to end of queue - self.pair_dict[pair]["priority"] = len(self.pair_dict) - def set_initial_return_values(self, pair: str, pred_df: DataFrame) -> None: """ Set the initial return values to the historical predictions dataframe. This avoids needing diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 78931bed4..85768fcf8 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -3,6 +3,7 @@ import shutil import threading import time from abc import ABC, abstractmethod +from collections import deque from datetime import datetime, timezone from pathlib import Path from threading import Lock @@ -80,6 +81,7 @@ class IFreqaiModel(ABC): self.pair_it = 0 self.pair_it_train = 0 self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist")) + self.train_queue = self._set_train_queue() self.last_trade_database_summary: DataFrame = {} self.current_trade_database_summary: DataFrame = {} self.analysis_lock = Lock() @@ -180,30 +182,36 @@ class IFreqaiModel(ABC): :param strategy: IStrategy = The user defined strategy class """ while not self._stop_event.is_set(): - time.sleep(1) - for pair in self.config.get("exchange", {}).get("pair_whitelist"): + pair = self.train_queue[0] - (_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair) + # ensure pair is avaialble in dp + if pair not in strategy.dp.current_whitelist(): + self.train_queue.popleft() + logger.warning(f'{pair} not in current whitelist, removing from train queue.') + continue - if self.dd.pair_dict[pair]["priority"] != 1: - continue - dk = FreqaiDataKitchen(self.config, self.live, pair) - dk.set_paths(pair, trained_timestamp) - ( - retrain, - new_trained_timerange, - data_load_timerange, - ) = dk.check_if_new_training_required(trained_timestamp) - dk.set_paths(pair, new_trained_timerange.stopts) + (_, trained_timestamp, _) = self.dd.get_pair_dict_info(pair) - if retrain: - self.train_timer('start') - self.extract_data_and_train_model( - new_trained_timerange, pair, strategy, dk, data_load_timerange - ) - self.train_timer('stop') + dk = FreqaiDataKitchen(self.config, self.live, pair) + dk.set_paths(pair, trained_timestamp) + ( + retrain, + new_trained_timerange, + data_load_timerange, + ) = dk.check_if_new_training_required(trained_timestamp) + dk.set_paths(pair, new_trained_timerange.stopts) - self.dd.save_historic_predictions_to_disk() + if retrain: + self.train_timer('start') + self.extract_data_and_train_model( + new_trained_timerange, pair, strategy, dk, data_load_timerange + ) + self.train_timer('stop') + + # only rotate the queue after the first has been trained. + self.train_queue.rotate(-1) + + self.dd.save_historic_predictions_to_disk() def start_backtesting( self, dataframe: DataFrame, metadata: dict, dk: FreqaiDataKitchen @@ -557,9 +565,6 @@ class IFreqaiModel(ABC): self.dd.pair_dict[pair]["trained_timestamp"] = new_trained_timerange.stopts dk.set_new_model_names(pair, new_trained_timerange) - self.dd.pair_dict[pair]["first"] = False - if self.dd.pair_dict[pair]["priority"] == 1 and self.scanning: - self.dd.pair_to_end_of_training_queue(pair) self.dd.save_data(model, pair, dk) if self.freqai_info.get("purge_old_models", False): @@ -685,6 +690,26 @@ class IFreqaiModel(ABC): return init_model + def _set_train_queue(self): + """ + Sets train queue from existing train timestamps if they exist + otherwise it sets the train queue based on the provided whitelist. + """ + current_pairlist = self.config.get("exchange", {}).get("pair_whitelist") + if not self.dd.pair_dict: + logger.info('Set fresh train queue from whitelist.') + return deque(current_pairlist) + + best_queue = deque() + + pair_dict_sorted = sorted(self.dd.pair_dict.items(), + key=lambda k: k[1]['trained_timestamp']) + for pair in pair_dict_sorted: + if pair[0] in current_pairlist: + best_queue.appendleft(pair[0]) + logger.info('Set existing queue from trained timestamps.') + return best_queue + # Following methods which are overridden by user made prediction models. # See freqai/prediction_models/CatboostPredictionModel.py for an example. diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index 15b2c6c83..0498ea564 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -45,7 +45,7 @@ class FreqaiExampleStrategy(IStrategy): std_dev_multiplier_buy = CategoricalParameter( [0.75, 1, 1.25, 1.5, 1.75], default=1.25, space="buy", optimize=True) std_dev_multiplier_sell = CategoricalParameter( - [0.1, 0.25, 0.4], space="sell", default=0.2, optimize=True) + [0.75, 1, 1.25, 1.5, 1.75], space="sell", default=1.25, optimize=True) def informative_pairs(self): whitelist_pairs = self.dp.current_whitelist() From 470d5d84058e0b000e659bfbd11b7a38a31a8c2b Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 18 Sep 2022 17:08:07 +0200 Subject: [PATCH 55/87] ensure full new pairlist is in the queue --- freqtrade/freqai/freqai_interface.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 85768fcf8..2cec96059 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -707,6 +707,10 @@ class IFreqaiModel(ABC): for pair in pair_dict_sorted: if pair[0] in current_pairlist: best_queue.appendleft(pair[0]) + for pair in current_pairlist: + if pair not in best_queue: + best_queue.appendleft(pair) + logger.info('Set existing queue from trained timestamps.') return best_queue From 584b2381d1ac09ae43f172d21b937c4d199e8958 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 19:36:11 +0200 Subject: [PATCH 56/87] Fix Imports --- freqtrade/plugins/pairlist/OffsetFilter.py | 2 +- freqtrade/plugins/pairlist/PerformanceFilter.py | 2 +- freqtrade/plugins/pairlist/PrecisionFilter.py | 2 +- freqtrade/plugins/pairlist/PriceFilter.py | 2 +- freqtrade/plugins/pairlist/ShuffleFilter.py | 2 +- freqtrade/resolvers/exchange_resolver.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py index 4200443c7..149befdeb 100644 --- a/freqtrade/plugins/pairlist/OffsetFilter.py +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -3,8 +3,8 @@ Offset pair list filter """ import logging from typing import Any, Dict, List -from freqtrade.constants import Config +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 4ba96231e..c29b4f337 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -5,8 +5,8 @@ import logging from typing import Any, Dict, List import pandas as pd -from freqtrade.constants import Config +from freqtrade.constants import Config from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.IPairList import IPairList diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 7b8b1a30d..8f1c9b839 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -3,8 +3,8 @@ Precision pair list filter """ import logging from typing import Any, Dict -from freqtrade.constants import Config +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index e38090e56..f2952001a 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -3,8 +3,8 @@ Price pair list filter """ import logging from typing import Any, Dict -from freqtrade.constants import Config +from freqtrade.constants import Config from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 098108949..b6b5fc3c8 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -4,8 +4,8 @@ Shuffle pair list filter import logging import random from typing import Any, Dict, List -from freqtrade.constants import Config +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.plugins.pairlist.IPairList import IPairList diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index d7a1a22e2..54a488e8d 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -2,9 +2,9 @@ This module loads custom exchanges """ import logging -from freqtrade.constants import Config import freqtrade.exchange as exchanges +from freqtrade.constants import Config from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, Exchange from freqtrade.resolvers import IResolver From a06eee300a552828214217a1024e72931ba444be Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 16:18:27 +0200 Subject: [PATCH 57/87] move ohlcv_get_pairs to parent class --- freqtrade/data/history/hdf5datahandler.py | 20 -------------------- freqtrade/data/history/idatahandler.py | 10 +++++++++- freqtrade/data/history/jsondatahandler.py | 20 -------------------- 3 files changed, 9 insertions(+), 41 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 135d97c79..bc44f5f8b 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -20,26 +20,6 @@ class HDF5DataHandler(IDataHandler): _columns = DEFAULT_DATAFRAME_COLUMNS - @classmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: - """ - Returns a list of all pairs with ohlcv data available in this datadir - for the specified timeframe - :param datadir: Directory to search for ohlcv files - :param timeframe: Timeframe to search pairs for - :param candle_type: Any of the enum CandleType (must match trading mode!) - :return: List of Pairs - """ - candle = "" - if candle_type != CandleType.SPOT: - datadir = datadir.joinpath('futures') - candle = f"-{candle_type}" - - _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.h5)', p.name) - for p in datadir.glob(f"*{timeframe}{candle}.h5")] - # Check if regex found something and only return these results - return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] - def ohlcv_store( self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType) -> None: """ diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 846bcc607..08e591c5c 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -61,7 +61,6 @@ class IDataHandler(ABC): ) for match in _tmp if match and len(match.groups()) > 1] @classmethod - @abstractmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir @@ -71,6 +70,15 @@ class IDataHandler(ABC): :param candle_type: Any of the enum CandleType (must match trading mode!) :return: List of Pairs """ + candle = "" + if candle_type != CandleType.SPOT: + datadir = datadir.joinpath('futures') + candle = f"-{candle_type}" + ext = cls._get_file_extension() + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + f'.{ext})', p.name) + for p in datadir.glob(f"*{timeframe}{candle}.{ext}")] + # Check if regex found something and only return these results + return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod def ohlcv_store( diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index a62e5e381..a54b44601 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -23,26 +23,6 @@ class JsonDataHandler(IDataHandler): _use_zip = False _columns = DEFAULT_DATAFRAME_COLUMNS - @classmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: - """ - Returns a list of all pairs with ohlcv data available in this datadir - for the specified timeframe - :param datadir: Directory to search for ohlcv files - :param timeframe: Timeframe to search pairs for - :param candle_type: Any of the enum CandleType (must match trading mode!) - :return: List of Pairs - """ - candle = "" - if candle_type != CandleType.SPOT: - datadir = datadir.joinpath('futures') - candle = f"-{candle_type}" - - _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.json)', p.name) - for p in datadir.glob(f"*{timeframe}{candle}.{cls._get_file_extension()}")] - # Check if regex found something and only return these results - return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] - def ohlcv_store( self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType) -> None: """ From 8116ca847ba942fcc4876ca92fc429c3104503a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 18 Sep 2022 16:57:03 +0200 Subject: [PATCH 58/87] move trades_get_pairs to parent class --- freqtrade/data/history/hdf5datahandler.py | 16 +--------------- freqtrade/data/history/idatahandler.py | 6 +++++- freqtrade/data/history/jsondatahandler.py | 16 +--------------- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index bc44f5f8b..01b7af7e7 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -1,7 +1,5 @@ import logging -import re -from pathlib import Path -from typing import List, Optional +from typing import Optional import numpy as np import pandas as pd @@ -101,18 +99,6 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - @classmethod - def trades_get_pairs(cls, datadir: Path) -> List[str]: - """ - Returns a list of all pairs for which trade data is available in this - :param datadir: Directory to search for ohlcv files - :return: List of Pairs - """ - _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name) - for p in datadir.glob("*trades.h5")] - # Check if regex found something and only return these results to avoid exceptions. - return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] - def trades_store(self, pair: str, data: TradeList) -> None: """ Store trades data (list of Dicts) to file diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 08e591c5c..8abccacdc 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -152,13 +152,17 @@ class IDataHandler(ABC): """ @classmethod - @abstractmethod def trades_get_pairs(cls, datadir: Path) -> List[str]: """ Returns a list of all pairs for which trade data is available in this :param datadir: Directory to search for ohlcv files :return: List of Pairs """ + _ext = cls._get_file_extension() + _tmp = [re.search(r'^(\S+)(?=\-trades.' + _ext + ')', p.name) + for p in datadir.glob(f"*trades.{_ext}")] + # Check if regex found something and only return these results to avoid exceptions. + return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod def trades_store(self, pair: str, data: TradeList) -> None: diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index a54b44601..f016c0ec1 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -1,7 +1,5 @@ import logging -import re -from pathlib import Path -from typing import List, Optional +from typing import Optional import numpy as np from pandas import DataFrame, read_json, to_datetime @@ -99,18 +97,6 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - @classmethod - def trades_get_pairs(cls, datadir: Path) -> List[str]: - """ - Returns a list of all pairs for which trade data is available in this - :param datadir: Directory to search for ohlcv files - :return: List of Pairs - """ - _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) - for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] - # Check if regex found something and only return these results to avoid exceptions. - return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] - def trades_store(self, pair: str, data: TradeList) -> None: """ Store trades data (list of Dicts) to file From 4cdc89706e4b951d6140e288d43256f5bc5ec461 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 03:02:04 +0000 Subject: [PATCH 59/87] Bump joblib from 1.1.0 to 1.2.0 Bumps [joblib](https://github.com/joblib/joblib) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/joblib/joblib/releases) - [Changelog](https://github.com/joblib/joblib/blob/master/CHANGES.rst) - [Commits](https://github.com/joblib/joblib/compare/1.1.0...1.2.0) --- updated-dependencies: - dependency-name: joblib dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-freqai.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-freqai.txt b/requirements-freqai.txt index e8d950382..9cdd431fe 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -3,7 +3,7 @@ # Required for freqai scikit-learn==1.1.2 -joblib==1.1.0 +joblib==1.2.0 catboost==1.0.6; platform_machine != 'aarch64' lightgbm==3.3.2 xgboost==1.6.2 diff --git a/requirements.txt b/requirements.txt index 91d3d3c8c..b6a1b87a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ pycoingecko==3.0.0 jinja2==3.1.2 tables==3.7.0 blosc==1.10.6 -joblib==1.1.0 +joblib==1.2.0 # find first, C search in arrays py_find_1st==1.1.5 From f512717943415032663f46be578c32145b0439f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 03:02:16 +0000 Subject: [PATCH 60/87] Bump ccxt from 1.93.35 to 1.93.66 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.93.35 to 1.93.66. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.93.35...1.93.66) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91d3d3c8c..09b3558a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.3 pandas==1.4.4 pandas-ta==0.3.14b -ccxt==1.93.35 +ccxt==1.93.66 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1 aiohttp==3.8.1 From cbdb0ce3e74d8b9da8f553750ce031fafa8e8951 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 03:02:23 +0000 Subject: [PATCH 61/87] Bump pyjwt from 2.4.0 to 2.5.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.4.0 to 2.5.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.4.0...2.5.0) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91d3d3c8c..c1cb2fcf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ sdnotify==0.3.2 # API Server fastapi==0.83.0 uvicorn==0.18.3 -pyjwt==2.4.0 +pyjwt==2.5.0 aiofiles==22.1.0 psutil==5.9.2 From 15c9b6bf4139045dfb97e41c3df637afce32f547 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 03:02:33 +0000 Subject: [PATCH 62/87] Bump mkdocs-material from 8.4.3 to 8.5.2 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.4.3 to 8.5.2. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.4.3...8.5.2) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index d63e79004..da6713b76 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.3.1 -mkdocs-material==8.4.3 +mkdocs-material==8.5.2 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 jinja2==3.1.2 From b5f51b5ec215a089e2c10abf2d40557fb0cb4273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 04:34:24 +0000 Subject: [PATCH 63/87] Bump fastapi from 0.83.0 to 0.85.0 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.83.0 to 0.85.0. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.83.0...0.85.0) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1cb2fcf2..541bfb266 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ orjson==3.8.0 sdnotify==0.3.2 # API Server -fastapi==0.83.0 +fastapi==0.85.0 uvicorn==0.18.3 pyjwt==2.5.0 aiofiles==22.1.0 From d93093100058db73e2c4a9939b716fa344e4436f Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Sep 2022 07:14:33 +0200 Subject: [PATCH 64/87] Bring back sleep - it'll ensure we give up control over the thread. --- freqtrade/freqai/freqai_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 2cec96059..5425f576c 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -182,6 +182,7 @@ class IFreqaiModel(ABC): :param strategy: IStrategy = The user defined strategy class """ while not self._stop_event.is_set(): + time.sleep(1) pair = self.train_queue[0] # ensure pair is avaialble in dp From 4a0a0c307c5f7366503515d696468ee7b32c5104 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Sep 2022 07:23:26 +0200 Subject: [PATCH 65/87] Use json_load to load leverage tiers --- freqtrade/exchange/binance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index faa780529..f9fb4a8b1 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,5 +1,4 @@ """ Binance exchange subclass """ -import json import logging from datetime import datetime from pathlib import Path @@ -12,7 +11,7 @@ from freqtrade.enums import CandleType, MarginMode, TradingMode from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, json_load logger = logging.getLogger(__name__) @@ -200,7 +199,7 @@ class Binance(Exchange): Path(__file__).parent / 'binance_leverage_tiers.json' ) with open(leverage_tiers_path) as json_file: - return json.load(json_file) + return json_load(json_file) else: try: return self._api.fetch_leverage_tiers() From ea58c29ded45033eb175d51f7041facb3ec33fa7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Sep 2022 08:11:25 +0000 Subject: [PATCH 66/87] Add plot_feature_importance to schema definition --- config_examples/config_freqai.example.json | 4 ++-- docs/freqai.md | 19 ++++++++++--------- freqtrade/constants.py | 1 + freqtrade/freqai/freqai_interface.py | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config_examples/config_freqai.example.json b/config_examples/config_freqai.example.json index 9494ba0e1..3a8a3b273 100644 --- a/config_examples/config_freqai.example.json +++ b/config_examples/config_freqai.example.json @@ -78,7 +78,7 @@ 10, 20 ], - "plot_feature_importance": true + "plot_feature_importance": false }, "data_split_parameters": { "test_size": 0.33, @@ -94,4 +94,4 @@ "internals": { "process_throttle_secs": 5 } -} +} \ No newline at end of file diff --git a/docs/freqai.md b/docs/freqai.md index 33fac198c..a03162b45 100644 --- a/docs/freqai.md +++ b/docs/freqai.md @@ -109,11 +109,12 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `indicator_max_period_candles` | **No longer used**. User must use the strategy set `startup_candle_count` which defines the maximum *period* used in `populate_any_indicators()` for indicator creation (timeframe independent). FreqAI uses this information in combination with the maximum timeframe to calculate how many data points it should download so that the first data point does not have a NaN
**Datatype:** positive integer. | `indicator_periods_candles` | Calculate indicators for `indicator_periods_candles` time periods and add them to the feature set.
**Datatype:** List of positive integers. | `stratify_training_data` | This value is used to indicate the grouping of the data. For example, 2 would set every 2nd data point into a separate dataset to be pulled from during training/testing. See details about how it works [here](#stratifying-the-data-for-training-and-testing-the-model)
**Datatype:** Positive integer. -| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis)
**Datatype:** Boolean. +| `principal_component_analysis` | Automatically reduce the dimensionality of the data set using Principal Component Analysis. See details about how it works [here](#reducing-data-dimensionality-with-principal-component-analysis) +| `plot_feature_importance` | Create an interactive feature importance plot for each model.
**Datatype:** Boolean.
**Datatype:** Boolean, defaults to `False` | `DI_threshold` | Activates the Dissimilarity Index for outlier detection when > 0. See details about how it works [here](#removing-outliers-with-the-dissimilarity-index).
**Datatype:** Positive float (typically < 1). | `use_SVM_to_remove_outliers` | Train a support vector machine to detect and remove outliers from the training data set, as well as from incoming data points. See details about how it works [here](#removing-outliers-using-a-support-vector-machine-svm).
**Datatype:** Boolean. | `svm_params` | All parameters available in Sklearn's `SGDOneClassSVM()`. See details about some select parameters [here](#removing-outliers-using-a-support-vector-machine-svm).
**Datatype:** Dictionary. -| `use_DBSCAN_to_remove_outliers` | Cluster data using DBSCAN to identify and remove outliers from training and prediction data. See details about how it works [here](#removing-outliers-with-dbscan).
**Datatype:** Boolean. +| `use_DBSCAN_to_remove_outliers` | Cluster data using DBSCAN to identify and remove outliers from training and prediction data. See details about how it works [here](#removing-outliers-with-dbscan).
**Datatype:** Boolean. | `inlier_metric_window` | If set, FreqAI will add the `inlier_metric` to the training feature set and set the lookback to be the `inlier_metric_window`. Details of how the `inlier_metric` is computed can be found [here](#using-the-inliermetric)
**Datatype:** int. Default: 0 | `noise_standard_deviation` | If > 0, FreqAI adds noise to the training features. FreqAI generates random deviates from a gaussian distribution with a standard deviation of `noise_standard_deviation` and adds them to all data points. Value should be kept relative to the normalized space between -1 and 1). In other words, since data is always normalized between -1 and 1 in FreqAI, the user can expect a `noise_standard_deviation: 0.05` to see 32% of data randomly increased/decreased by more than 2.5% (i.e. the percent of data falling within the first standard deviation). Good for preventing overfitting.
**Datatype:** int. Default: 0 | `outlier_protection_percentage` | If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection while keeping the original dataset intact. If the outlier protection is triggered, no predictions will be made based on the training data.
**Datatype:** Float. Default: `30` @@ -510,7 +511,7 @@ The FreqAI backtesting module can be executed with the following command: freqtrade backtesting --strategy FreqaiExampleStrategy --strategy-path freqtrade/templates --config config_examples/config_freqai.example.json --freqaimodel LightGBMRegressor --timerange 20210501-20210701 ``` -Backtesting mode requires the user to have the data [pre-downloaded](#downloading-data-for-backtesting) (unlike in dry/live mode where FreqAI automatically downloads the necessary data). The user should be careful to consider that the time range of the downloaded data is more than the backtesting time range. This is because FreqAI needs data prior to the desired backtesting time range in order to train a model to be ready to make predictions on the first candle of the user-set backtesting time range. More details on how to calculate the data to download can be found [here](#deciding-the-sliding-training-window-and-backtesting-duration). +Backtesting mode requires the user to have the data [pre-downloaded](#downloading-data-for-backtesting) (unlike in dry/live mode where FreqAI automatically downloads the necessary data). The user should be careful to consider that the time range of the downloaded data is more than the backtesting time range. This is because FreqAI needs data prior to the desired backtesting time range in order to train a model to be ready to make predictions on the first candle of the user-set backtesting time range. More details on how to calculate the data to download can be found [here](#deciding-the-sliding-training-window-and-backtesting-duration). If this command has never been executed with the existing config file, it will train a new model for each pair, for each backtesting window within the expanded `--timerange`. @@ -538,7 +539,7 @@ Users need to have the data pre-downloaded in the same fashion as if they were d - It's not possible to hyperopt indicators in `populate_any_indicators()` function. This means that the user cannot optimize model parameters using hyperopt. Apart from this exception, it is possible to optimize all other [spaces](hyperopt.md#running-hyperopt-with-smaller-search-space). - The [Backtesting](#backtesting) instructions also apply to Hyperopt. -The best method for combining hyperopt and FreqAI is to focus on hyperopting entry/exit thresholds/criteria. Users need to focus on hyperopting parameters that are not used in their FreqAI features. For example, users should not try to hyperopt rolling window lengths in their feature creation, or any of their FreqAI config which changes predictions. In order to efficiently hyperopt the FreqAI strategy, FreqAI stores predictions as dataframes and reuses them. Hence the requirement to hyperopt entry/exit thresholds/criteria only. +The best method for combining hyperopt and FreqAI is to focus on hyperopting entry/exit thresholds/criteria. Users need to focus on hyperopting parameters that are not used in their FreqAI features. For example, users should not try to hyperopt rolling window lengths in their feature creation, or any of their FreqAI config which changes predictions. In order to efficiently hyperopt the FreqAI strategy, FreqAI stores predictions as dataframes and reuses them. Hence the requirement to hyperopt entry/exit thresholds/criteria only. A good example of a hyperoptable parameter in FreqAI is a value for `DI_values` beyond which we consider outliers and below which we consider inliers: @@ -563,7 +564,7 @@ FreqAI will train have trained 8 separate models at the end of `--timerange` (be Although fractional `backtest_period_days` is allowed, the user should be aware that the `--timerange` is divided by this value to determine the number of models that FreqAI will need to train in order to backtest the full range. For example, if the user wants to set a `--timerange` of 10 days, and asks for a `backtest_period_days` of 0.1, FreqAI will need to train 100 models per pair to complete the full backtest. Because of this, a true backtest of FreqAI adaptive training would take a *very* long time. The best way to fully test a model is to run it dry and let it constantly train. In this case, backtesting would take the exact same amount of time as a dry run. ### Downloading data for backtesting -Live/dry instances will download the data automatically for the user, but users who wish to use backtesting functionality still need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). FreqAI users need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that they have a sufficient amount of training data *before* the start of their backtesting timerange. The amount of additional data can be roughly estimated by moving the start date of the timerange backwards by `train_period_days` and the `startup_candle_count` ([details](#setting-the-startupcandlecount)) from the beginning of the desired backtesting timerange. +Live/dry instances will download the data automatically for the user, but users who wish to use backtesting functionality still need to download the necessary data using `download-data` (details [here](data-download.md#data-downloading)). FreqAI users need to pay careful attention to understanding how much *additional* data needs to be downloaded to ensure that they have a sufficient amount of training data *before* the start of their backtesting timerange. The amount of additional data can be roughly estimated by moving the start date of the timerange backwards by `train_period_days` and the `startup_candle_count` ([details](#setting-the-startupcandlecount)) from the beginning of the desired backtesting timerange. As an example, if we wish to backtest the `--timerange` above of `20210501-20210701`, and we use the example config which sets `train_period_days` to 15. The startup candle count is 40 on a maximum `include_timeframes` of 1h. We would need 20210501 - 15 days - 40 * 1h / 24 hours = 20210414 (16.7 days earlier than the start of the desired training timerange). @@ -662,13 +663,13 @@ The test data is used to evaluate the performance of the model after training. I ### Using the `inlier_metric` -The `inlier_metric` is a metric aimed at quantifying how different a prediction data point is from the most recent historic data points. +The `inlier_metric` is a metric aimed at quantifying how different a prediction data point is from the most recent historic data points. -User can set `inlier_metric_window` to set the look back window. FreqAI will compute the distance between the present prediction point and each of the previous data points (total of `inlier_metric_window` points). +User can set `inlier_metric_window` to set the look back window. FreqAI will compute the distance between the present prediction point and each of the previous data points (total of `inlier_metric_window` points). -This function goes one step further - during training, it computes the `inlier_metric` for all training data points and builds weibull distributions for each each lookback point. The cumulative distribution function for the weibull distribution is used to produce a quantile for each of the data points. The quantiles for each lookback point are averaged to create the `inlier_metric`. +This function goes one step further - during training, it computes the `inlier_metric` for all training data points and builds weibull distributions for each each lookback point. The cumulative distribution function for the weibull distribution is used to produce a quantile for each of the data points. The quantiles for each lookback point are averaged to create the `inlier_metric`. -FreqAI adds this `inlier_metric` score to the training features! In other words, your model is trained to recognize how this temporal inlier metric is related to the user set labels. +FreqAI adds this `inlier_metric` score to the training features! In other words, your model is trained to recognize how this temporal inlier metric is related to the user set labels. This function does **not** remove outliers from the data set. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 077c4ecbf..75825c0d0 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -504,6 +504,7 @@ CONF_SCHEMA = { "weight_factor": {"type": "number", "default": 0}, "principal_component_analysis": {"type": "boolean", "default": False}, "use_SVM_to_remove_outliers": {"type": "boolean", "default": False}, + "plot_feature_importance": {"type": "boolean", "default": False}, "svm_params": {"type": "object", "properties": { "shuffle": {"type": "boolean", "default": False}, diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index be1231a53..19e813e4d 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -563,7 +563,7 @@ class IFreqaiModel(ABC): self.dd.pair_to_end_of_training_queue(pair) self.dd.save_data(model, pair, dk) - if self.freqai_info["feature_parameters"].get("plot_feature_importance", True): + if self.freqai_info["feature_parameters"].get("plot_feature_importance", False): plot_feature_importance(model, pair, dk) if self.freqai_info.get("purge_old_models", False): From ad652817ef6875ac1d1705178f3c2921df758e55 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 19 Sep 2022 11:11:23 +0200 Subject: [PATCH 67/87] Ensure train ordering after restart Ensure lowest timestamps get trained first after restart --- freqtrade/freqai/freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 3f5eff167..fa9d5043b 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -711,7 +711,7 @@ class IFreqaiModel(ABC): key=lambda k: k[1]['trained_timestamp']) for pair in pair_dict_sorted: if pair[0] in current_pairlist: - best_queue.appendleft(pair[0]) + best_queue.appendright(pair[0]) for pair in current_pairlist: if pair not in best_queue: best_queue.appendleft(pair) From 995396c7752657ea29815cf1443d0679a6c0e6ba Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 19 Sep 2022 11:42:56 +0200 Subject: [PATCH 68/87] Add useful log info --- freqtrade/freqai/freqai_interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index fa9d5043b..0dc2baf54 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -703,6 +703,7 @@ class IFreqaiModel(ABC): current_pairlist = self.config.get("exchange", {}).get("pair_whitelist") if not self.dd.pair_dict: logger.info('Set fresh train queue from whitelist.') + logger.info(f'Queue: {current_pairlist}') return deque(current_pairlist) best_queue = deque() @@ -717,6 +718,7 @@ class IFreqaiModel(ABC): best_queue.appendleft(pair) logger.info('Set existing queue from trained timestamps.') + logger.info(f'Best approximation queue: {best_queue}') return best_queue # Following methods which are overridden by user made prediction models. From 9b66297cc0bbb2ae758319a5a4747eeca738451e Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Mon, 19 Sep 2022 12:47:20 +0200 Subject: [PATCH 69/87] Fix append --- freqtrade/freqai/freqai_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 0dc2baf54..9743c35d4 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -712,7 +712,7 @@ class IFreqaiModel(ABC): key=lambda k: k[1]['trained_timestamp']) for pair in pair_dict_sorted: if pair[0] in current_pairlist: - best_queue.appendright(pair[0]) + best_queue.append(pair[0]) for pair in current_pairlist: if pair not in best_queue: best_queue.appendleft(pair) From 42c75b4a7beb71603cad305f72154410209b791e Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 19 Sep 2022 19:16:32 +0200 Subject: [PATCH 70/87] combine log messages --- freqtrade/freqai/freqai_interface.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 9743c35d4..156c6bee5 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -702,8 +702,8 @@ class IFreqaiModel(ABC): """ current_pairlist = self.config.get("exchange", {}).get("pair_whitelist") if not self.dd.pair_dict: - logger.info('Set fresh train queue from whitelist.') - logger.info(f'Queue: {current_pairlist}') + logger.info('Set fresh train queue from whitelist. ' + f'Queue: {current_pairlist}') return deque(current_pairlist) best_queue = deque() @@ -717,8 +717,8 @@ class IFreqaiModel(ABC): if pair not in best_queue: best_queue.appendleft(pair) - logger.info('Set existing queue from trained timestamps.') - logger.info(f'Best approximation queue: {best_queue}') + logger.info('Set existing queue from trained timestamps. ' + f'Best approximation queue: {best_queue}') return best_queue # Following methods which are overridden by user made prediction models. From eb9ac9cbdaf33218e8f93ba3490949899601ca3a Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Sep 2022 20:17:58 +0200 Subject: [PATCH 71/87] add --exchange to convert-trade-data --- freqtrade/commands/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 2835f8582..97d8cc130 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -62,9 +62,9 @@ ARGS_BUILD_CONFIG = ["config"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] -ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] +ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"] -ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "exchange", "trading_mode", +ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"] ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] From 703bcc099a942eedf75bc6fcfa3421ae49c93a25 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Sep 2022 20:32:54 +0200 Subject: [PATCH 72/87] Fix list-pair regex to also support 1INCH/USDT --- freqtrade/data/history/idatahandler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 8abccacdc..8c1823c00 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_\d-]+)\-(\d+[a-zA-Z]{1,2})\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -267,7 +267,7 @@ class IDataHandler(ABC): Rebuild pair name from filename Assumes a asset name of max. 7 length to also support BTC-PERP and BTC-PERP:USD names. """ - res = re.sub(r'^(([A-Za-z]{1,10})|^([A-Za-z\-]{1,6}))(_)', r'\g<1>/', pair, 1) + res = re.sub(r'^(([A-Za-z\d]{1,10})|^([A-Za-z\-]{1,6}))(_)', r'\g<1>/', pair, 1) res = re.sub('_', ':', res, 1) return res From 32d46e8a6bd3dcc26fcdaac2764120691b8d8b58 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 19 Sep 2022 20:59:40 +0200 Subject: [PATCH 73/87] Improve fixture naming --- tests/conftest.py | 2 +- tests/data/test_converter.py | 4 ++-- tests/strategy/test_default_strategy.py | 4 ++-- tests/strategy/test_strategy_loading.py | 22 +++++++++++----------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fffac8e0a..4039f9367 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2282,7 +2282,7 @@ def tickers(): @pytest.fixture -def result(testdatadir): +def dataframe_1m(testdatadir): with (testdatadir / 'UNITTEST_BTC-1m.json').open('r') as data_file: return ohlcv_to_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC", fill_missing=True) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index c6b0059a2..f74383d15 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -18,8 +18,8 @@ from tests.conftest import log_has, log_has_re from tests.data.test_history import _clean_test_file -def test_dataframe_correct_columns(result): - assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume'] +def test_dataframe_correct_columns(dataframe_1m): + assert dataframe_1m.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume'] def test_ohlcv_to_dataframe(ohlcv_history_list, caplog): diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index 5cb8fce16..cb3d61e89 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -21,14 +21,14 @@ def test_strategy_test_v3_structure(): (True, 'short'), (False, 'long'), ]) -def test_strategy_test_v3(result, fee, is_short, side): +def test_strategy_test_v3(dataframe_1m, fee, is_short, side): strategy = StrategyTestV3({}) metadata = {'pair': 'ETH/BTC'} assert type(strategy.minimal_roi) is dict assert type(strategy.stoploss) is float assert type(strategy.timeframe) is str - indicators = strategy.populate_indicators(result, metadata) + indicators = strategy.populate_indicators(dataframe_1m, metadata) assert type(indicators) is DataFrame assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame assert type(strategy.populate_sell_trend(indicators, metadata)) is DataFrame diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index bf81cd068..adffd0875 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -53,7 +53,7 @@ def test_search_all_strategies_with_failed(): assert len(strategies) == 0 -def test_load_strategy(default_conf, result): +def test_load_strategy(default_conf, dataframe_1m): default_conf.update({'strategy': 'SampleStrategy', 'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates') }) @@ -61,22 +61,22 @@ def test_load_strategy(default_conf, result): assert isinstance(strategy.__source__, str) assert 'class SampleStrategy' in strategy.__source__ assert isinstance(strategy.__file__, str) - assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + assert 'rsi' in strategy.advise_indicators(dataframe_1m, {'pair': 'ETH/BTC'}) -def test_load_strategy_base64(result, caplog, default_conf): +def test_load_strategy_base64(dataframe_1m, caplog, default_conf): filepath = Path(__file__).parents[2] / 'freqtrade/templates/sample_strategy.py' encoded_string = urlsafe_b64encode(filepath.read_bytes()).decode("utf-8") default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)}) strategy = StrategyResolver.load_strategy(default_conf) - assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + assert 'rsi' in strategy.advise_indicators(dataframe_1m, {'pair': 'ETH/BTC'}) # Make sure strategy was loaded from base64 (using temp directory)!! assert log_has_re(r"Using resolved strategy SampleStrategy from '" r".*(/|\\).*(/|\\)SampleStrategy\.py'\.\.\.", caplog) -def test_load_strategy_invalid_directory(result, caplog, default_conf): +def test_load_strategy_invalid_directory(caplog, default_conf): default_conf['strategy'] = 'StrategyTestV3' extra_dir = Path.cwd() / 'some/path' with pytest.raises(OperationalException): @@ -104,7 +104,7 @@ def test_load_strategy_noname(default_conf): @pytest.mark.filterwarnings("ignore:deprecated") @pytest.mark.parametrize('strategy_name', ['StrategyTestV2']) -def test_strategy_pre_v3(result, default_conf, strategy_name): +def test_strategy_pre_v3(dataframe_1m, default_conf, strategy_name): default_conf.update({'strategy': strategy_name}) strategy = StrategyResolver.load_strategy(default_conf) @@ -118,7 +118,7 @@ def test_strategy_pre_v3(result, default_conf, strategy_name): assert strategy.timeframe == '5m' assert default_conf['timeframe'] == '5m' - df_indicators = strategy.advise_indicators(result, metadata=metadata) + df_indicators = strategy.advise_indicators(dataframe_1m, metadata=metadata) assert 'adx' in df_indicators dataframe = strategy.advise_entry(df_indicators, metadata=metadata) @@ -417,24 +417,24 @@ def test_call_deprecated_function(default_conf): StrategyResolver.load_strategy(default_conf) -def test_strategy_interface_versioning(result, default_conf): +def test_strategy_interface_versioning(dataframe_1m, default_conf): default_conf.update({'strategy': 'StrategyTestV2'}) strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} assert strategy.INTERFACE_VERSION == 2 - indicator_df = strategy.advise_indicators(result, metadata=metadata) + indicator_df = strategy.advise_indicators(dataframe_1m, metadata=metadata) assert isinstance(indicator_df, DataFrame) assert 'adx' in indicator_df.columns - enterdf = strategy.advise_entry(result, metadata=metadata) + enterdf = strategy.advise_entry(dataframe_1m, metadata=metadata) assert isinstance(enterdf, DataFrame) assert 'buy' not in enterdf.columns assert 'enter_long' in enterdf.columns - exitdf = strategy.advise_exit(result, metadata=metadata) + exitdf = strategy.advise_exit(dataframe_1m, metadata=metadata) assert isinstance(exitdf, DataFrame) assert 'sell' not in exitdf assert 'exit_long' in exitdf From b5fd11f91b265d65fb4c9485794368169037be3b Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 19 Sep 2022 20:39:19 +0200 Subject: [PATCH 74/87] protect against unforeseen issues in scanning thread --- freqtrade/freqai/freqai_interface.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 156c6bee5..5850cdeb3 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -205,9 +205,13 @@ class IFreqaiModel(ABC): if retrain: self.train_timer('start') - self.extract_data_and_train_model( - new_trained_timerange, pair, strategy, dk, data_load_timerange - ) + try: + self.extract_data_and_train_model( + new_trained_timerange, pair, strategy, dk, data_load_timerange + ) + except Exception as msg: + logger.warning(f'Training {pair} raised exception {msg}, skipping.') + self.train_timer('stop') # only rotate the queue after the first has been trained. From 6c18fa0847bccb4b830308cc9cc86784764fc137 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Sep 2022 09:30:41 +0000 Subject: [PATCH 75/87] Update proxy docs --- docs/configuration.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index f8a600e76..bb4f5ce41 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -57,7 +57,7 @@ You can specify additional configuration files in `add_config_files`. Files spec This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands. !!! Tip "Use multiple configuration files to keep secrets secret" - You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself. + You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself. The 2nd file should only specify what you intend to override. If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`). @@ -110,7 +110,7 @@ This is similar to using multiple `--config` parameters, but simpler in usage as "stake_amount": "unlimited" } ``` - + If multiple files are in the `add_config_files` section, then they will be assumed to be at identical levels, having the last occurrence override the earlier config (unless a parent already defined such a key). ## Configuration parameters @@ -659,17 +659,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Using proxy with Freqtrade -To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. - -An example for this can be found in `config_examples/config_full.example.json` - -``` json -"ccxt_async_config": { - "aiohttp_trust_env": true -} -``` - -Then, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values +To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values. ``` bash export HTTP_PROXY="http://addr:port" @@ -677,6 +667,20 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` +#### Proxy just exchange requests + +To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration. + +``` json +"ccxt_config": { + "aiohttp_proxy": "http://addr:port", + "proxies": { + "http": "http://addr:port", + "https": "http://addr:port" + }, +} +``` + ## Next step Now you have configured your config.json, the next step is to [start your bot](bot-usage.md). From 8a91c8e220a1770b08af25f1b3f84f5fba12ed8b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Sep 2022 13:30:59 +0000 Subject: [PATCH 76/87] Sort and dedup pairs before data conversion --- freqtrade/data/converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index bf19c1310..67461973f 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -292,6 +292,7 @@ def convert_ohlcv_format( timeframe, candle_type=candle_type )) + config['pairs'] = sorted(set(config['pairs'])) logger.info(f"Converting candle (OHLCV) data for {config['pairs']}") for timeframe in timeframes: @@ -302,7 +303,7 @@ def convert_ohlcv_format( drop_incomplete=False, startup_candles=0, candle_type=candle_type) - logger.info(f"Converting {len(data)} {candle_type} candles for {pair}") + logger.info(f"Converting {len(data)} {timeframe} {candle_type} candles for {pair}") if len(data) > 0: trg.ohlcv_store( pair=pair, From 0bd6ad55a14a84dcd77cb057453f46ae8acfd3a6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Sep 2022 14:14:54 +0000 Subject: [PATCH 77/87] Always show freqtrade version --- freqtrade/freqtradebot.py | 4 +--- freqtrade/main.py | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 83abc9bc5..d4dc6ef73 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple from schedule import Scheduler -from freqtrade import __version__, constants +from freqtrade import constants from freqtrade.configuration import validate_config_consistency from freqtrade.constants import BuySell, Config, LongShort from freqtrade.data.converter import order_book_to_dataframe @@ -52,8 +52,6 @@ class FreqtradeBot(LoggingMixin): """ self.active_pair_whitelist: List[str] = [] - logger.info('Starting freqtrade %s', __version__) - # Init bot state self.state = State.STOPPED diff --git a/freqtrade/main.py b/freqtrade/main.py index 162b4d029..754c536d0 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -12,6 +12,7 @@ from typing import Any, List if sys.version_info < (3, 8): # pragma: no cover sys.exit("Freqtrade requires Python version >= 3.8") +from freqtrade import __version__ from freqtrade.commands import Arguments from freqtrade.exceptions import FreqtradeException, OperationalException from freqtrade.loggers import setup_logging_pre @@ -34,6 +35,7 @@ def main(sysargv: List[str] = None) -> None: # Call subcommand. if 'func' in args: + logger.info(f'freqtrade {__version__}') return_code = args['func'](args) else: # No subcommand was issued. From 0c01b23cba8564642a2f07ec8a5a817d486821b8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Sep 2022 18:09:14 +0200 Subject: [PATCH 78/87] Capture exceptions in send_msg calls --- freqtrade/rpc/rpc_manager.py | 2 ++ tests/rpc/test_rpc_manager.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 3b60077ad..8390e61aa 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -78,6 +78,8 @@ class RPCManager: mod.send_msg(msg) except NotImplementedError: logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.") + except Exception: + logger.exception('Exception occurred within RPC module %s', mod.name) def process_msg_queue(self, queue: deque) -> None: """ diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index b9ae16a20..d71f38259 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -82,6 +82,21 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: assert telegram_mock.call_count == 0 +def test_send_msg_telegram_error(mocker, default_conf, caplog) -> None: + mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) + mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', side_effect=ValueError()) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc_manager = RPCManager(freqtradebot) + rpc_manager.send_msg({ + 'type': RPCMessageType.STATUS, + 'status': 'test' + }) + + assert log_has("Sending rpc message: {'type': status, 'status': 'test'}", caplog) + assert log_has("Exception occurred within RPC module telegram", caplog) + + def test_process_msg_queue(mocker, default_conf, caplog) -> None: telegram_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg') mocker.patch('freqtrade.rpc.telegram.Telegram._init') From 3b0874eb373f6079d9da204afa28e35a3c79a933 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Sep 2022 20:00:08 +0200 Subject: [PATCH 79/87] Update exit message handling to gracefully handle kucoins "empty" responses closes #7444 --- freqtrade/freqtradebot.py | 17 +++++++++-------- freqtrade/rpc/telegram.py | 7 ++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d4dc6ef73..539bbcb38 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1597,14 +1597,14 @@ class FreqtradeBot(LoggingMixin): # second condition is for mypy only; order will always be passed during sub trade if sub_trade and order is not None: amount = order.safe_filled if fill else order.amount - profit_rate = order.safe_price + order_rate: float = order.safe_price - profit = trade.calc_profit(rate=profit_rate, amount=amount, open_rate=trade.open_rate) - profit_ratio = trade.calc_profit_ratio(profit_rate, amount, trade.open_rate) + profit = trade.calc_profit(rate=order_rate, amount=amount, open_rate=trade.open_rate) + profit_ratio = trade.calc_profit_ratio(order_rate, amount, trade.open_rate) else: - profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested - profit = trade.calc_profit(rate=profit_rate) + (0.0 if fill else trade.realized_profit) - profit_ratio = trade.calc_profit_ratio(profit_rate) + order_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested + profit = trade.calc_profit(rate=order_rate) + (0.0 if fill else trade.realized_profit) + profit_ratio = trade.calc_profit_ratio(order_rate) amount = trade.amount gain = "profit" if profit_ratio > 0 else "loss" @@ -1617,11 +1617,12 @@ class FreqtradeBot(LoggingMixin): 'leverage': trade.leverage, 'direction': 'Short' if trade.is_short else 'Long', 'gain': gain, - 'limit': profit_rate, + 'limit': order_rate, # Deprecated + 'order_rate': order_rate, 'order_type': order_type, 'amount': amount, 'open_rate': trade.open_rate, - 'close_rate': profit_rate, + 'close_rate': order_rate, 'current_rate': current_rate, 'profit_amount': profit, 'profit_ratio': profit_ratio, diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c40bdb963..247373817 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -286,7 +286,7 @@ class Telegram(RPCHandler): if msg['type'] in [RPCMessageType.ENTRY_FILL]: message += f"*Open Rate:* `{msg['open_rate']:.8f}`\n" elif msg['type'] in [RPCMessageType.ENTRY]: - message += f"*Open Rate:* `{msg['limit']:.8f}`\n"\ + message += f"*Open Rate:* `{msg['open_rate']:.8f}`\n"\ f"*Current Rate:* `{msg['current_rate']:.8f}`\n" message += f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}" @@ -353,8 +353,9 @@ class Telegram(RPCHandler): f"*Open Rate:* `{msg['open_rate']:.8f}`\n" ) if msg['type'] == RPCMessageType.EXIT: - message += (f"*Current Rate:* `{msg['current_rate']:.8f}`\n" - f"*Exit Rate:* `{msg['limit']:.8f}`") + message += f"*Current Rate:* `{msg['current_rate']:.8f}`\n" + if msg['order_rate']: + message += f"*Exit Rate:* `{msg['order_rate']:.8f}`" elif msg['type'] == RPCMessageType.EXIT_FILL: message += f"*Exit Rate:* `{msg['close_rate']:.8f}`" From ff36431680758c3a3652486243c0d4409528e999 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Sep 2022 20:31:06 +0200 Subject: [PATCH 80/87] Adjust tests for new messageType handling --- tests/rpc/test_rpc_telegram.py | 15 +++++++++------ tests/test_freqtradebot.py | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f2e490dff..3552d5fe7 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -959,6 +959,7 @@ def test_telegram_forceexit_handle(default_conf, update, ticker, fee, 'gain': 'profit', 'leverage': 1.0, 'limit': 1.173e-05, + 'order_rate': 1.173e-05, 'amount': 91.07468123, 'order_type': 'limit', 'open_rate': 1.098e-05, @@ -1031,6 +1032,7 @@ def test_telegram_force_exit_down_handle(default_conf, update, ticker, fee, 'gain': 'loss', 'leverage': 1.0, 'limit': 1.043e-05, + 'order_rate': 1.043e-05, 'amount': 91.07468123, 'order_type': 'limit', 'open_rate': 1.098e-05, @@ -1092,6 +1094,7 @@ def test_forceexit_all_handle(default_conf, update, ticker, fee, mocker) -> None 'pair': 'ETH/BTC', 'gain': 'loss', 'leverage': 1.0, + 'order_rate': 1.099e-05, 'limit': 1.099e-05, 'amount': 91.07468123, 'order_type': 'limit', @@ -1744,7 +1747,7 @@ def test_send_msg_enter_notification(default_conf, mocker, caplog, message_type, 'exchange': 'Binance', 'pair': 'ETH/BTC', 'leverage': leverage, - 'limit': 1.099e-05, + 'open_rate': 1.099e-05, 'order_type': 'limit', 'direction': enter, 'stake_amount': 0.01465333, @@ -1915,7 +1918,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'leverage': 1.0, 'direction': 'Long', 'gain': 'loss', - 'limit': 3.201e-05, + 'order_rate': 3.201e-05, 'amount': 1333.3333333333335, 'order_type': 'market', 'open_rate': 7.5e-05, @@ -1950,7 +1953,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'pair': 'KEY/ETH', 'direction': 'Long', 'gain': 'loss', - 'limit': 3.201e-05, + 'order_rate': 3.201e-05, 'amount': 1333.3333333333335, 'order_type': 'market', 'open_rate': 7.5e-05, @@ -1989,7 +1992,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'pair': 'KEY/ETH', 'direction': 'Long', 'gain': 'loss', - 'limit': 3.201e-05, + 'order_rate': 3.201e-05, 'amount': 1333.3333333333335, 'order_type': 'market', 'open_rate': 7.5e-05, @@ -2162,7 +2165,7 @@ def test_send_msg_buy_notification_no_fiat( 'exchange': 'Binance', 'pair': 'ETH/BTC', 'leverage': leverage, - 'limit': 1.099e-05, + 'open_rate': 1.099e-05, 'order_type': 'limit', 'direction': enter, 'stake_amount': 0.01465333, @@ -2205,7 +2208,7 @@ def test_send_msg_sell_notification_no_fiat( 'gain': 'loss', 'leverage': leverage, 'direction': direction, - 'limit': 3.201e-05, + 'order_rate': 3.201e-05, 'amount': 1333.3333333333335, 'order_type': 'limit', 'open_rate': 7.5e-05, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index c1152ac09..7851a73f8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3256,6 +3256,7 @@ def test_execute_trade_exit_up(default_conf_usdt, ticker_usdt, fee, ticker_usdt_ 'pair': 'ETH/USDT', 'gain': 'profit', 'limit': 2.0 if is_short else 2.2, + 'order_rate': 2.0 if is_short else 2.2, 'amount': pytest.approx(amt), 'order_type': 'limit', 'buy_tag': None, @@ -3321,6 +3322,7 @@ def test_execute_trade_exit_down(default_conf_usdt, ticker_usdt, fee, ticker_usd 'leverage': 1.0, 'gain': 'loss', 'limit': 2.2 if is_short else 2.01, + 'order_rate': 2.2 if is_short else 2.01, 'amount': pytest.approx(29.70297029) if is_short else 30.0, 'order_type': 'limit', 'buy_tag': None, @@ -3405,6 +3407,7 @@ def test_execute_trade_exit_custom_exit_price( 'leverage': 1.0, 'gain': profit_or_loss, 'limit': limit, + 'order_rate': limit, 'amount': pytest.approx(amount), 'order_type': 'limit', 'buy_tag': None, @@ -3476,6 +3479,7 @@ def test_execute_trade_exit_down_stoploss_on_exchange_dry_run( 'leverage': 1.0, 'gain': 'loss', 'limit': 2.02 if is_short else 1.98, + 'order_rate': 2.02 if is_short else 1.98, 'amount': pytest.approx(29.70297029 if is_short else 30.0), 'order_type': 'limit', 'buy_tag': None, @@ -3741,6 +3745,7 @@ def test_execute_trade_exit_market_order( 'leverage': 1.0, 'gain': profit_or_loss, 'limit': limit, + 'order_rate': limit, 'amount': pytest.approx(amount), 'order_type': 'market', 'buy_tag': None, From 8f41f943b44dbda75e38546ee5c7a5d0ab88d553 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Sep 2022 06:42:55 +0200 Subject: [PATCH 81/87] Fix 0.0 amount message wording --- freqtrade/freqtradebot.py | 2 +- tests/test_integration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 539bbcb38..a4bbf291c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -594,7 +594,7 @@ class FreqtradeBot(LoggingMixin): amount = trade.amount if amount == 0.0: - logger.info("Amount to sell is 0.0 due to exchange limits - not selling.") + logger.info("Amount to exit is 0.0 due to exchange limits - not exiting.") return remaining = (trade.amount - amount) * current_exit_rate diff --git a/tests/test_integration.py b/tests/test_integration.py index 77ed822d1..a7b4fbdd3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -521,4 +521,4 @@ def test_dca_exiting(default_conf_usdt, ticker_usdt, fee, mocker, caplog) -> Non assert trade.orders[-1].ft_order_side == 'sell' assert pytest.approx(trade.stake_amount) == 40.198 assert trade.is_open - assert log_has_re('Amount to sell is 0.0 due to exchange limits - not selling.', caplog) + assert log_has_re('Amount to exit is 0.0 due to exchange limits - not exiting.', caplog) From 02f2096fc35d12859c8921c51d2857168d23ee05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Sep 2022 06:51:14 +0200 Subject: [PATCH 82/87] Reverse and fix rangestability conditions closes #7447 --- freqtrade/plugins/pairlist/rangestabilityfilter.py | 10 +++------- tests/plugins/test_pairlist.py | 4 ++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 0bc2cdb47..1bc7ad48f 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -100,23 +100,19 @@ class RangeStabilityFilter(IPairList): if cached_res is not None: return cached_res - result = False + result = True if daily_candles is not None and not daily_candles.empty: highest_high = daily_candles['high'].max() lowest_low = daily_candles['low'].min() pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 - if pct_change >= self._min_rate_of_change: - result = True - else: + if pct_change < self._min_rate_of_change: self.log_once(f"Removed {pair} from whitelist, because rate of change " f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " f"which is below the threshold of {self._min_rate_of_change}.", logger.info) result = False if self._max_rate_of_change: - if pct_change <= self._max_rate_of_change: - result = True - else: + if pct_change > self._max_rate_of_change: self.log_once( f"Removed {pair} from whitelist, because rate of change " f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 48a0f81cb..538751251 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -467,6 +467,10 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "RangeStabilityFilter", "lookback_days": 10, "max_rate_of_change": 0.01, "refresh_period": 1440}], "BTC", []), # All removed because of max_rate_of_change being 0.017 + ([{"method": "StaticPairList"}, + {"method": "RangeStabilityFilter", "lookback_days": 10, + "min_rate_of_change": 0.018, "max_rate_of_change": 0.02, "refresh_period": 1440}], + "BTC", []), # All removed - limits are above the highest change_rate ([{"method": "StaticPairList"}, {"method": "VolatilityFilter", "lookback_days": 3, "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}], From f7b8c5a767970eb84d4071beca8d7d2b8fc5fa5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Sep 2022 12:46:43 +0000 Subject: [PATCH 83/87] Reorder telegram noise sample --- docs/telegram-usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index b9324def4..055512f26 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -82,6 +82,8 @@ Example configuration showing the different settings: "warning": "on", "startup": "off", "entry": "silent", + "entry_fill": "on", + "entry_cancel": "silent", "exit": { "roi": "silent", "emergency_exit": "on", @@ -93,9 +95,7 @@ Example configuration showing the different settings: "custom_exit": "silent", "partial_exit": "on" }, - "entry_cancel": "silent", "exit_cancel": "on", - "entry_fill": "off", "exit_fill": "off", "protection_trigger": "off", "protection_trigger_global": "on", From 923182680e4a246d0f2fdcbe31092b9a89eeb10e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Sep 2022 12:43:17 +0000 Subject: [PATCH 84/87] Explicitly define notification defaults --- freqtrade/constants.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 75825c0d0..1b3edddef 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -289,11 +289,12 @@ CONF_SCHEMA = { 'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'entry': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'entry_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'entry_fill': {'type': 'string', - 'enum': TELEGRAM_SETTING_OPTIONS, - 'default': 'off' - }, + 'entry_fill': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, + 'entry_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, }, 'exit': { 'type': ['string', 'object'], 'additionalProperties': { @@ -301,12 +302,12 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS } }, - 'exit_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'exit_fill': { 'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'on' }, + 'exit_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'protection_trigger': { 'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, @@ -315,14 +316,17 @@ CONF_SCHEMA = { 'protection_trigger_global': { 'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'on' }, 'show_candle': { 'type': 'string', 'enum': ['off', 'ohlc'], + 'default': 'off' }, 'strategy_msg': { 'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'on' }, } }, From 08e183fb556d3af099f72ef2c78d2e555cdd8510 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Sep 2022 20:59:12 +0200 Subject: [PATCH 85/87] Add note about okx trading mode --- docs/exchanges.md | 2 +- freqtrade/exchange/okx.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index dc2003f9c..a9ba16c64 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -233,7 +233,7 @@ OKX requires a passphrase for each api key, you will therefore need to add this !!! Warning "Futures" OKX Futures has the concept of "position mode" - which can be Net or long/short (hedge mode). - Freqtrade supports both modes - but changing the mode mid-trading is not supported and will lead to exceptions and failures to place trades. + Freqtrade supports both modes (we recommend to use net mode) - but changing the mode mid-trading is not supported and will lead to exceptions and failures to place trades. OKX also only provides MARK candles for the past ~3 months. Backtesting futures prior to that date will therefore lead to slight deviations, as funding-fees cannot be calculated correctly without this data. ## Gate.io diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index 49f8ea107..2db5fb6a9 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -71,6 +71,7 @@ class Okx(Exchange): try: if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']: accounts = self._api.fetch_accounts() + self._log_exchange_response('fetch_accounts', accounts) if len(accounts) > 0: self.net_only = accounts[0].get('info', {}).get('posMode') == 'net_mode' except ccxt.DDoSProtection as e: From 91dc5e7aa671ac9e42ec1fc72329089280aacabb Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 21 Sep 2022 21:12:08 +0200 Subject: [PATCH 86/87] Be sure to provide an amount in entry notifications --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a4bbf291c..eb5705c34 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -921,7 +921,7 @@ class FreqtradeBot(LoggingMixin): 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), - 'amount': order.safe_amount_after_fee if fill else order.amount, + 'amount': order.safe_amount_after_fee if fill else (order.amount or trade.amount), 'open_date': trade.open_date or datetime.utcnow(), 'current_rate': current_rate, 'sub_trade': sub_trade, From 71e6c54ea452a75e0cc1580fc275b19de076a6a7 Mon Sep 17 00:00:00 2001 From: th0rntwig <107926911+th0rntwig@users.noreply.github.com> Date: Thu, 22 Sep 2022 18:11:50 +0200 Subject: [PATCH 87/87] Normalise distances before Weibull fit (#7432) * Normalise distances before Weibull * Track inlier-metric params --- freqtrade/freqai/data_kitchen.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index d2abd0ad2..005005368 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -775,12 +775,22 @@ class FreqaiDataKitchen: def compute_inlier_metric(self, set_='train') -> None: """ - Compute inlier metric from backwards distance distributions. This metric defines how well features from a timepoint fit into previous timepoints. """ + def normalise(dataframe: DataFrame, key: str) -> DataFrame: + if set_ == 'train': + min_value = dataframe.min() + max_value = dataframe.max() + self.data[f'{key}_min'] = min_value + self.data[f'{key}_max'] = max_value + else: + min_value = self.data[f'{key}_min'] + max_value = self.data[f'{key}_max'] + return (dataframe - min_value) / (max_value - min_value) + no_prev_pts = self.freqai_config["feature_parameters"]["inlier_metric_window"] if set_ == 'train': @@ -825,7 +835,12 @@ class FreqaiDataKitchen: inliers = pd.DataFrame(index=distances.index) for key in distances.keys(): current_distances = distances[key].dropna() - fit_params = stats.weibull_min.fit(current_distances) + current_distances = normalise(current_distances, key) + if set_ == 'train': + fit_params = stats.weibull_min.fit(current_distances) + self.data[f'{key}_fit_params'] = fit_params + else: + fit_params = self.data[f'{key}_fit_params'] quantiles = stats.weibull_min.cdf(current_distances, *fit_params) df_inlier = pd.DataFrame(