From f1e831a7b8acfcdab524b8325a961fe5491c862c Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 26 Mar 2023 13:43:59 +0200 Subject: [PATCH 01/21] fix bug in backtest target setting --- freqtrade/freqai/data_kitchen.py | 8 ++++---- freqtrade/freqai/freqai_interface.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 52d487b08..21b41db2d 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -1291,7 +1291,7 @@ class FreqaiDataKitchen: return dataframe - def use_strategy_to_populate_indicators( + def use_strategy_to_populate_indicators( # noqa: C901 self, strategy: IStrategy, corr_dataframes: dict = {}, @@ -1362,12 +1362,12 @@ class FreqaiDataKitchen: dataframe = self.populate_features(dataframe.copy(), corr_pair, strategy, corr_dataframes, base_dataframes, True) - dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata) + if self.live: + dataframe = strategy.set_freqai_targets(dataframe.copy(), metadata=metadata) + dataframe = self.remove_special_chars_from_feature_names(dataframe) self.get_unique_classes_from_labels(dataframe) - dataframe = self.remove_special_chars_from_feature_names(dataframe) - if self.config.get('reduce_df_footprint', False): dataframe = reduce_dataframe_footprint(dataframe) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index b657bd811..fe62adea9 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -306,7 +306,7 @@ class IFreqaiModel(ABC): if check_features: self.dd.load_metadata(dk) dataframe_dummy_features = self.dk.use_strategy_to_populate_indicators( - strategy, prediction_dataframe=dataframe.tail(1), pair=metadata["pair"] + strategy, prediction_dataframe=dataframe.tail(1), pair=pair ) dk.find_features(dataframe_dummy_features) self.check_if_feature_list_matches_strategy(dk) @@ -316,7 +316,7 @@ class IFreqaiModel(ABC): else: if populate_indicators: dataframe = self.dk.use_strategy_to_populate_indicators( - strategy, prediction_dataframe=dataframe, pair=metadata["pair"] + strategy, prediction_dataframe=dataframe, pair=pair ) populate_indicators = False @@ -332,6 +332,9 @@ class IFreqaiModel(ABC): dataframe_train = dk.slice_dataframe(tr_train, dataframe_base_train) dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe_base_backtest) + dataframe_train = dk.remove_special_chars_from_feature_names(dataframe_train) + dataframe_backtest = dk.remove_special_chars_from_feature_names(dataframe_backtest) + if not self.model_exists(dk): dk.find_features(dataframe_train) dk.find_labels(dataframe_train) From 55781e7f10fc66aaae4a0039766a8bde94a82355 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sun, 26 Mar 2023 19:22:52 +0200 Subject: [PATCH 02/21] fix tests --- tests/freqai/conftest.py | 2 ++ tests/freqai/test_freqai_datadrawer.py | 7 ++++++- tests/freqai/test_freqai_datakitchen.py | 1 + tests/freqai/test_freqai_interface.py | 6 ++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index e140ee80b..75034767d 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -119,6 +119,7 @@ def make_unfiltered_dataframe(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True freqai.dk.pair = "ADA/BTC" data_load_timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(data_load_timerange, freqai.dk) @@ -152,6 +153,7 @@ def make_data_dictionary(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True freqai.dk.pair = "ADA/BTC" data_load_timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(data_load_timerange, freqai.dk) diff --git a/tests/freqai/test_freqai_datadrawer.py b/tests/freqai/test_freqai_datadrawer.py index da3b8f9c1..8ab2c75da 100644 --- a/tests/freqai/test_freqai_datadrawer.py +++ b/tests/freqai/test_freqai_datadrawer.py @@ -19,6 +19,7 @@ def test_update_historic_data(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -41,6 +42,7 @@ def test_load_all_pairs_histories(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -60,6 +62,7 @@ def test_get_base_and_corr_dataframes(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dd.load_all_pair_histories(timerange, freqai.dk) sub_timerange = TimeRange.parse_timerange("20180111-20180114") @@ -87,6 +90,7 @@ def test_use_strategy_to_populate_indicators(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180114") freqai.dd.load_all_pair_histories(timerange, freqai.dk) sub_timerange = TimeRange.parse_timerange("20180111-20180114") @@ -103,8 +107,9 @@ def test_get_timerange_from_live_historic_predictions(mocker, freqai_conf): exchange = get_patched_exchange(mocker, freqai_conf) strategy.dp = DataProvider(freqai_conf, exchange) freqai = strategy.freqai - freqai.live = True + freqai.live = False freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = False timerange = TimeRange.parse_timerange("20180126-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) sub_timerange = TimeRange.parse_timerange("20180128-20180130") diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 95665a775..3f0fc697d 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -180,6 +180,7 @@ def test_get_full_model_path(mocker, freqai_conf, model): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 3b370aea4..e149ca517 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -87,6 +87,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, freqai.live = True freqai.can_short = can_short freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True freqai.dk.set_paths('ADA/BTC', 10000) timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -135,6 +136,7 @@ def test_extract_data_and_train_model_MultiTargets(mocker, freqai_conf, model, s freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -178,6 +180,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -238,6 +241,7 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) freqai = strategy.freqai freqai.live = False freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) sub_timerange = TimeRange.parse_timerange("20180110-20180130") @@ -394,6 +398,7 @@ def test_principal_component_analysis(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) @@ -425,6 +430,7 @@ def test_plot_feature_importance(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) From 3cabcabcbd5230de3a9c8cb45759d9e540ea4c4f Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 27 Mar 2023 15:23:01 +0200 Subject: [PATCH 03/21] ensure labels are properly defined in backtesting --- freqtrade/freqai/freqai_interface.py | 1 + tests/freqai/test_freqai_interface.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index fe62adea9..7444bf404 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -334,6 +334,7 @@ class IFreqaiModel(ABC): dataframe_train = dk.remove_special_chars_from_feature_names(dataframe_train) dataframe_backtest = dk.remove_special_chars_from_feature_names(dataframe_backtest) + dk.get_unique_classes_from_labels(dataframe_train) if not self.model_exists(dk): dk.find_features(dataframe_train) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index e149ca517..5f8071446 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -241,7 +241,6 @@ def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog) freqai = strategy.freqai freqai.live = False freqai.dk = FreqaiDataKitchen(freqai_conf) - freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) sub_timerange = TimeRange.parse_timerange("20180110-20180130") @@ -375,6 +374,9 @@ def test_backtesting_fit_live_predictions(mocker, freqai_conf, caplog): sub_timerange = TimeRange.parse_timerange("20180129-20180130") corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(sub_timerange, "LTC/BTC", freqai.dk) df = freqai.dk.use_strategy_to_populate_indicators(strategy, corr_df, base_df, "LTC/BTC") + df = strategy.set_freqai_targets(df.copy(), metadata={"pair": "LTC/BTC"}) + df = freqai.dk.remove_special_chars_from_feature_names(df) + freqai.dk.get_unique_classes_from_labels(df) freqai.dk.pair = "ADA/BTC" freqai.dk.full_df = df.fillna(0) freqai.dk.full_df From 355fde3bca6f668da1fef36e187f85d09cf0b3f7 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 29 Mar 2023 22:01:54 +0200 Subject: [PATCH 04/21] revert setting dk to live in test_plot_feature_importances --- tests/freqai/test_freqai_interface.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 5f8071446..1122d9e9c 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -432,7 +432,6 @@ def test_plot_feature_importance(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) - freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) From 92a060c5b45f683009186c6a7169e5f8bfae9b55 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 3 Apr 2023 20:18:57 +0200 Subject: [PATCH 05/21] Make stop_price_parameter configurable by exchange --- freqtrade/exchange/exchange.py | 13 +++++++------ freqtrade/exchange/okx.py | 20 ++------------------ 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6c236106f..4d7be8fdf 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -60,6 +60,7 @@ class Exchange: # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, + "stop_price_param": "stopPrice", "order_time_in_force": ["GTC"], "ohlcv_params": {}, "ohlcv_candle_limit": 500, @@ -1115,11 +1116,11 @@ class Exchange: """ if not self._ft_has.get('stoploss_on_exchange'): raise OperationalException(f"stoploss is not implemented for {self.name}.") - + price_param = self._ft_has['stop_price_param'] return ( - order.get('stopPrice', None) is None - or ((side == "sell" and stop_loss > float(order['stopPrice'])) or - (side == "buy" and stop_loss < float(order['stopPrice']))) + order.get(price_param, None) is None + or ((side == "sell" and stop_loss > float(order[price_param])) or + (side == "buy" and stop_loss < float(order[price_param]))) ) def _get_stop_order_type(self, user_order_type) -> Tuple[str, str]: @@ -1159,8 +1160,8 @@ class Exchange: def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict: params = self._params.copy() - # Verify if stopPrice works for your exchange! - params.update({'stopPrice': stop_price}) + # Verify if stopPrice works for your exchange, else configure stop_price_param + params.update({self._ft_has['stop_price_param']: stop_price}) return params @retrier(retries=0) diff --git a/freqtrade/exchange/okx.py b/freqtrade/exchange/okx.py index a4fcaeca0..84b7deb7a 100644 --- a/freqtrade/exchange/okx.py +++ b/freqtrade/exchange/okx.py @@ -28,6 +28,7 @@ class Okx(Exchange): "funding_fee_timeframe": "8h", "stoploss_order_types": {"limit": "limit"}, "stoploss_on_exchange": True, + "stop_price_param": "stopLossPrice", } _ft_has_futures: Dict = { "tickers_have_quoteVolume": False, @@ -162,29 +163,12 @@ class Okx(Exchange): return pair_tiers[-1]['maxNotional'] / leverage def _get_stop_params(self, side: BuySell, ordertype: str, stop_price: float) -> Dict: - - params = self._params.copy() - # Verify if stopPrice works for your exchange! - params.update({'stopLossPrice': stop_price}) - + params = super()._get_stop_params(side, ordertype, stop_price) if self.trading_mode == TradingMode.FUTURES and self.margin_mode: params['tdMode'] = self.margin_mode.value params['posSide'] = self._get_posSide(side, True) return params - def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool: - """ - OKX uses non-default stoploss price naming. - """ - if not self._ft_has.get('stoploss_on_exchange'): - raise OperationalException(f"stoploss is not implemented for {self.name}.") - - return ( - order.get('stopLossPrice', None) is None - or ((side == "sell" and stop_loss > float(order['stopLossPrice'])) or - (side == "buy" and stop_loss < float(order['stopLossPrice']))) - ) - def fetch_stoploss_order(self, order_id: str, pair: str, params: Dict = {}) -> Dict: if self._config['dry_run']: return self.fetch_dry_run_order(order_id) From fe02f611fbc82edcde6ac743b243cd189f0075dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Apr 2023 06:46:35 +0200 Subject: [PATCH 06/21] Fix typo in reinforcement learning closes #8431 --- docs/freqai-reinforcement-learning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md index f5679a4ba..f298dbf4d 100644 --- a/docs/freqai-reinforcement-learning.md +++ b/docs/freqai-reinforcement-learning.md @@ -180,7 +180,7 @@ As you begin to modify the strategy and the prediction model, you will quickly r # you can use feature values from dataframe # Assumes the shifted RSI indicator has been generated in the strategy. - rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{pair}_" + rsi_now = self.raw_features[f"%-rsi-period_10_shift-1_{pair}_" f"{self.config['timeframe']}"].iloc[self._current_tick] # reward agent for entering trades From f03a99918a422ea5a513bb3ad65a71a9b3b10eb8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 4 Apr 2023 20:04:28 +0200 Subject: [PATCH 07/21] Ensure hyper param file can be loaded closes #8452 --- freqtrade/optimize/hyperopt_tools.py | 15 +++++++++++++-- freqtrade/strategy/hyper.py | 5 ++--- tests/strategy/test_interface.py | 9 ++++++--- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index e2133a956..1e7befdf6 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) NON_OPT_PARAM_APPENDIX = " # value loaded from strategy" +HYPER_PARAMS_FILE_FORMAT = rapidjson.NM_NATIVE | rapidjson.NM_NAN + def hyperopt_serializer(x): if isinstance(x, np.integer): @@ -76,9 +78,18 @@ class HyperoptTools(): with filename.open('w') as f: rapidjson.dump(final_params, f, indent=2, default=hyperopt_serializer, - number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN + number_mode=HYPER_PARAMS_FILE_FORMAT ) + @staticmethod + def load_params(filename: Path) -> Dict: + """ + Load parameters from file + """ + with filename.open('r') as f: + params = rapidjson.load(f, number_mode=HYPER_PARAMS_FILE_FORMAT) + return params + @staticmethod def try_export_params(config: Config, strategy_name: str, params: Dict): if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): @@ -189,7 +200,7 @@ class HyperoptTools(): for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing', 'max_open_trades']: HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s) - print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) + print(rapidjson.dumps(result_dict, default=str, number_mode=HYPER_PARAMS_FILE_FORMAT)) else: HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:", diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 52ba22951..d38110a2a 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union from freqtrade.constants import Config from freqtrade.exceptions import OperationalException -from freqtrade.misc import deep_merge_dicts, json_load +from freqtrade.misc import deep_merge_dicts from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.strategy.parameters import BaseParameter @@ -124,8 +124,7 @@ class HyperStrategyMixin: if filename.is_file(): logger.info(f"Loading parameters from file {filename}") try: - with filename.open('r') as f: - params = json_load(f) + params = HyperoptTools.load_params(filename) if params.get('strategy_name') != self.__class__.__name__: raise OperationalException('Invalid parameter file provided.') return params diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 7b1399507..713ffa5cb 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -986,7 +986,8 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): } } } - mocker.patch('freqtrade.strategy.hyper.json_load', return_value=expected_result) + mocker.patch('freqtrade.strategy.hyper.HyperoptTools.load_params', + return_value=expected_result) PairLocks.timeframe = default_conf['timeframe'] strategy = StrategyResolver.load_strategy(default_conf) assert strategy.stoploss == -0.05 @@ -1005,11 +1006,13 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): } } - mocker.patch('freqtrade.strategy.hyper.json_load', return_value=expected_result) + mocker.patch('freqtrade.strategy.hyper.HyperoptTools.load_params', + return_value=expected_result) with pytest.raises(OperationalException, match="Invalid parameter file provided."): StrategyResolver.load_strategy(default_conf) - mocker.patch('freqtrade.strategy.hyper.json_load', MagicMock(side_effect=ValueError())) + mocker.patch('freqtrade.strategy.hyper.HyperoptTools.load_params', + MagicMock(side_effect=ValueError())) StrategyResolver.load_strategy(default_conf) assert log_has("Invalid parameter file format.", caplog) From dae3f72be73a77a4ef3f0585a54ea41013b5f762 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Apr 2023 14:11:31 +0200 Subject: [PATCH 08/21] Bump Dockerfile to latest 3.10 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a4a168c1..655f9ee94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.10-slim-bullseye as base +FROM python:3.10.11-slim-bullseye as base # Setup env ENV LANG C.UTF-8 From a75d8910076990fbf8ff6245170c8f4b8a30a7f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Apr 2023 14:45:06 +0200 Subject: [PATCH 09/21] Ensure minimum sqlalchemy version is respected --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index edd7b243b..131e8a8a7 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ setup( install_requires=[ # from requirements.txt 'ccxt>=2.6.26', - 'SQLAlchemy', + 'SQLAlchemy>=2.0.6', 'python-telegram-bot>=13.4', 'arrow>=0.17.0', 'cachetools', From 77985fa59144372eec901fa727730d11f64bab5e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Apr 2023 14:49:53 +0200 Subject: [PATCH 10/21] Update thread name for uvicorn worker --- freqtrade/rpc/api_server/uvicorn_threaded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py index a79c1a5fc..48786bec2 100644 --- a/freqtrade/rpc/api_server/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server/uvicorn_threaded.py @@ -55,7 +55,7 @@ class UvicornServer(uvicorn.Server): @contextlib.contextmanager def run_in_thread(self): - self.thread = threading.Thread(target=self.run) + self.thread = threading.Thread(target=self.run, name='FTUvicorn') self.thread.start() while not self.started: time.sleep(1e-3) From 1952e453bbdb7ea2076a0cc3285adb428b3fdb62 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Apr 2023 17:35:11 +0200 Subject: [PATCH 11/21] Improved formatting for fetch order_or_stop calls --- freqtrade/freqtradebot.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index af4f42feb..26e918f18 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1483,8 +1483,8 @@ class FreqtradeBot(LoggingMixin): return False try: - order = self.exchange.cancel_order_with_result(order['id'], trade.pair, - trade.amount) + order = self.exchange.cancel_order_with_result( + order['id'], trade.pair, trade.amount) except InvalidOrderException: logger.exception( f"Could not cancel {trade.exit_side} order {trade.open_order_id}") @@ -1786,9 +1786,8 @@ class FreqtradeBot(LoggingMixin): if not stoploss_order: logger.info(f'Found open order for {trade}') try: - order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id, - trade.pair, - stoploss_order) + order = action_order or self.exchange.fetch_order_or_stoploss_order( + order_id, trade.pair, stoploss_order) except InvalidOrderException as exception: logger.warning('Unable to fetch order %s: %s', order_id, exception) return False From f8d89c46e55d733393554e349b0980e0cc8ad2b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 7 Apr 2023 19:48:44 +0200 Subject: [PATCH 12/21] Don't reset open_order_id if the order didn't cancel --- freqtrade/freqtradebot.py | 3 ++- tests/test_freqtradebot.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 26e918f18..48e3ec209 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1496,17 +1496,18 @@ class FreqtradeBot(LoggingMixin): # Order might be filled above in odd timing issues. if order.get('status') in ('canceled', 'cancelled'): trade.exit_reason = None + trade.open_order_id = None else: trade.exit_reason = exit_reason_prev cancelled = True else: reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE'] trade.exit_reason = None + trade.open_order_id = None self.update_trade_state(trade, trade.open_order_id, order) logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.') - trade.open_order_id = None trade.close_rate = None trade.close_rate_requested = None diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index ab5dd4af5..7bded0f82 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2955,6 +2955,9 @@ def test_manage_open_orders_exit_usercustom( assert rpc_mock.call_count == 2 assert freqtrade.strategy.check_exit_timeout.call_count == 1 assert freqtrade.strategy.check_entry_timeout.call_count == 0 + trade = Trade.session.scalars(select(Trade)).first() + # cancelling didn't succeed - order-id remains open. + assert trade.open_order_id is not None # 2nd canceled trade - Fail execute exit caplog.clear() @@ -3465,6 +3468,7 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: # TODO: should not be magicmock trade = MagicMock() + trade.open_order_id = '125' reason = CANCEL_REASON['TIMEOUT'] order = {'remaining': 1, 'id': '125', @@ -3472,6 +3476,10 @@ def test_handle_cancel_exit_cancel_exception(mocker, default_conf_usdt) -> None: 'status': "open"} assert not freqtrade.handle_cancel_exit(trade, order, reason) + # mocker.patch(f'{EXMS}.cancel_order_with_result', return_value=order) + # assert not freqtrade.handle_cancel_exit(trade, order, reason) + # assert trade.open_order_id == '125' + @pytest.mark.parametrize("is_short, open_rate, amt", [ (False, 2.0, 30.0), From c2c97d9f78801ede95191e799d480a3b943a14cd Mon Sep 17 00:00:00 2001 From: robcaulk Date: Sat, 8 Apr 2023 13:20:29 +0200 Subject: [PATCH 13/21] make a fake pair_dict instead of MagicMocking it --- tests/freqai/test_freqai_interface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index 1122d9e9c..5c2d39e53 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -432,10 +432,12 @@ def test_plot_feature_importance(mocker, freqai_conf): freqai = strategy.freqai freqai.live = True freqai.dk = FreqaiDataKitchen(freqai_conf) + freqai.dk.live = True timerange = TimeRange.parse_timerange("20180110-20180130") freqai.dd.load_all_pair_histories(timerange, freqai.dk) - freqai.dd.pair_dict = MagicMock() + freqai.dd.pair_dict = {"ADA/BTC": {"model_filename": "fake_name", + "trained_timestamp": 1, "data_path": "", "extras": {}}} data_load_timerange = TimeRange.parse_timerange("20180110-20180130") new_timerange = TimeRange.parse_timerange("20180120-20180130") From dd8900a1c67c6d9c6ef4361472ed6dec27e148bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 08:37:44 +0200 Subject: [PATCH 14/21] Improve ordering of backtest output --- docs/backtesting.md | 15 ++++++++------- freqtrade/optimize/optimize_reports.py | 10 +++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 0227df3f6..166c2b28b 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -274,19 +274,20 @@ A backtesting result will look like that: | XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 0 23 34.3 | | ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 0 15 31.8 | | TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 0 243 43.4 | -========================================================= EXIT REASON STATS ========================================================== -| Exit Reason | Exits | Wins | Draws | Losses | -|:-------------------|--------:|------:|-------:|--------:| -| trailing_stop_loss | 205 | 150 | 0 | 55 | -| stop_loss | 166 | 0 | 0 | 166 | -| exit_signal | 56 | 36 | 0 | 20 | -| force_exit | 2 | 0 | 0 | 2 | ====================================================== LEFT OPEN TRADES REPORT ====================================================== | Pair | Entries | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Win Draw Loss Win% | |:---------|---------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:| | ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 0 0 100 | | LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 0 0 100 | | TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 0 0 100 | +==================== EXIT REASON STATS ==================== +| Exit Reason | Exits | Wins | Draws | Losses | +|:-------------------|--------:|------:|-------:|--------:| +| trailing_stop_loss | 205 | 150 | 0 | 55 | +| stop_loss | 166 | 0 | 0 | 166 | +| exit_signal | 56 | 36 | 0 | 20 | +| force_exit | 2 | 0 | 0 | 2 | + ================== SUMMARY METRICS ================== | Metric | Value | |-----------------------------+---------------------| diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 83f698fbe..5cccd76a2 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -865,6 +865,11 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) print(table) + table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + if (results.get('results_per_enter_tag') is not None or results.get('results_per_buy_tag') is not None): # results_per_buy_tag is deprecated and should be removed 2 versions after short golive. @@ -884,11 +889,6 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '=')) print(table) - table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) - if isinstance(table, str) and len(table) > 0: - print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) - print(table) - for period in backtest_breakdown: days_breakdown_stats = generate_periodic_breakdown_stats( trade_list=results['trades'], period=period) From df51111c339796a7e9c2c7740b98c1fa3c683090 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 08:38:33 +0200 Subject: [PATCH 15/21] Always show strategy summary --- freqtrade/optimize/optimize_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 5cccd76a2..b4925770d 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -917,11 +917,11 @@ def show_backtest_results(config: Config, backtest_stats: Dict): strategy, results, stake_currency, config.get('backtest_breakdown', [])) - if len(backtest_stats['strategy']) > 1: + if len(backtest_stats['strategy']) > 0: # Print Strategy summary table table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency) - print(f"{results['backtest_start']} -> {results['backtest_end']} |" + print(f"Backtested {results['backtest_start']} -> {results['backtest_end']} |" f" Max open trades : {results['max_open_trades']}") print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) print(table) From 526943f29e6fc8ed46300c80cfdc43fce6df03c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 9 Apr 2023 19:44:38 +0200 Subject: [PATCH 16/21] Remove freqUI alpha warning --- docs/rest-api.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index 5f604ef43..860a44499 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -9,9 +9,6 @@ This same command can also be used to update freqUI, should there be a new relea Once the bot is started in trade / dry-run mode (with `freqtrade trade`) - the UI will be available under the configured port below (usually `http://127.0.0.1:8080`). -!!! info "Alpha release" - FreqUI is still considered an alpha release - if you encounter bugs or inconsistencies please open a [FreqUI issue](https://github.com/freqtrade/frequi/issues/new/choose). - !!! Note "developers" Developers should not use this method, but instead use the method described in the [freqUI repository](https://github.com/freqtrade/frequi) to get the source-code of freqUI. From 8854ef8cba46571eb98a22a2cba6352ebfcf6ced Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 03:56:33 +0000 Subject: [PATCH 17/21] Bump ccxt from 3.0.50 to 3.0.58 Bumps [ccxt](https://github.com/ccxt/ccxt) from 3.0.50 to 3.0.58. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/3.0.50...3.0.58) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 34c7da0fa..7944a38f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.2 pandas==1.5.3 pandas-ta==0.3.14b -ccxt==3.0.50 +ccxt==3.0.58 cryptography==40.0.1 aiohttp==3.8.4 SQLAlchemy==2.0.8 From a449f7c78c9702402b1ebd8730c0e26ef3d72b99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 03:56:38 +0000 Subject: [PATCH 18/21] Bump pytest from 7.2.2 to 7.3.0 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.2 to 7.3.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.2...7.3.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f36ef6def..e44e9357b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ coveralls==3.3.1 ruff==0.0.260 mypy==1.1.1 pre-commit==3.2.1 -pytest==7.2.2 +pytest==7.3.0 pytest-asyncio==0.21.0 pytest-cov==4.0.0 pytest-mock==3.10.0 From 7e1f3aa545cd2c06faeb8329640daba003485af8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 03:56:51 +0000 Subject: [PATCH 19/21] Bump filelock from 3.10.6 to 3.11.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.10.6 to 3.11.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.10.6...3.11.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 2c7c27d98..d617eeb0f 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,5 +5,5 @@ scipy==1.10.1 scikit-learn==1.1.3 scikit-optimize==0.9.0 -filelock==3.10.6 +filelock==3.11.0 progressbar2==4.2.0 From 26eb4f7fe69b0a6f829b840c9567e0c62d6f87d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 03:56:57 +0000 Subject: [PATCH 20/21] Bump pymdown-extensions from 9.10 to 9.11 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 9.10 to 9.11. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/9.10...9.11) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index c70415c85..2c8d455df 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,5 +2,5 @@ markdown==3.3.7 mkdocs==1.4.2 mkdocs-material==9.1.5 mdx_truly_sane_lists==1.3 -pymdown-extensions==9.10 +pymdown-extensions==9.11 jinja2==3.1.2 From 2ea01571970539f0e605cafe838dbe9c94df9265 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 03:57:51 +0000 Subject: [PATCH 21/21] Bump pypa/gh-action-pypi-publish from 1.8.4 to 1.8.5 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.4 to 1.8.5. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.4...v1.8.5) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e856607fc..52c772bd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -425,7 +425,7 @@ jobs: python setup.py sdist bdist_wheel - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.8.5 if: (github.event_name == 'release') with: user: __token__ @@ -433,7 +433,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.4 + uses: pypa/gh-action-pypi-publish@v1.8.5 if: (github.event_name == 'release') with: user: __token__