diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a23181c37..759ac0a6a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,9 +15,9 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.1 + - types-requests==2.28.3 - types-tabulate==0.8.11 - - types-python-dateutil==2.8.18 + - types-python-dateutil==2.8.19 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/docs/includes/protections.md b/docs/includes/protections.md index d67924cfe..e0ad8189f 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -50,6 +50,8 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only consider one side, and will then only lock this one side, allowing for example shorts to continue after a series of long stoplosses. +`required_profit` will determine the required relative profit (or loss) for stoplosses to consider. This should normally not be set and defaults to 0.0 - which means all losing stoplosses will be triggering a block. + The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. ``` python @@ -61,6 +63,7 @@ def protections(self): "lookback_period_candles": 24, "trade_limit": 4, "stop_duration_candles": 4, + "required_profit": 0.0, "only_per_pair": False, "only_per_side": False } diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a07f4f944..205516d6d 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ -markdown==3.4.1 -mkdocs==1.3.0 +markdown==3.3.7 +mkdocs==1.3.1 mkdocs-material==8.3.9 mdx_truly_sane_lists==1.3 pymdown-extensions==9.5 diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 11e37b953..79bc769e6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1264,7 +1264,7 @@ class Exchange: return False required = ('fee', 'status', 'amount') - return all(k in corder for k in required) + return all(corder.get(k, None) is not None for k in required) def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: """ diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 5562ffba0..4d37ef8c1 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -8,10 +8,10 @@ from pathlib import Path from typing import Any, Dict, Tuple import numpy as np -import numpy.typing as npt import pandas as pd from joblib import dump, load from joblib.externals import cloudpickle +from numpy.typing import ArrayLike from pandas import DataFrame from freqtrade.configuration import TimeRange @@ -81,8 +81,7 @@ class FreqaiDataDrawer: """ Locate and load a previously saved data drawer full of all pair model metadata in present model folder. - :returns: - exists: bool = whether or not the drawer was located + :return: bool - whether or not the drawer was located """ exists = self.pair_dictionary_path.is_file() if exists: @@ -101,8 +100,7 @@ class FreqaiDataDrawer: def load_historic_predictions_from_disk(self): """ Locate and load a previously saved historic predictions. - :returns: - exists: bool = whether or not the drawer was located + :return: bool - whether or not the drawer was located """ exists = self.historic_predictions_path.is_file() if exists: @@ -221,7 +219,7 @@ class FreqaiDataDrawer: self.pair_dict[pair]["priority"] = len(self.pair_dict) def set_initial_return_values(self, pair: str, dk: FreqaiDataKitchen, - pred_df: DataFrame, do_preds: npt.ArrayLike) -> None: + pred_df: DataFrame, do_preds: ArrayLike) -> None: """ Set the initial return values to a persistent dataframe. This avoids needing to repredict on historical candles, and also stores historical predictions despite retrainings (so stored @@ -240,7 +238,8 @@ class FreqaiDataDrawer: mrv_df["do_predict"] = do_preds - def append_model_predictions(self, pair: str, predictions, do_preds, dk, len_df) -> None: + def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: ArrayLike, + dk: FreqaiDataKitchen, len_df: int) -> None: # strat seems to feed us variable sized dataframes - and since we are trying to build our # own return array in the same shape, we need to figure out how the size has changed @@ -295,7 +294,7 @@ class FreqaiDataDrawer: dataframe = pd.concat([dataframe[to_keep], df], axis=1) return dataframe - def return_null_values_to_strategy(self, dataframe: DataFrame, dk) -> None: + def return_null_values_to_strategy(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> None: """ Build 0 filled dataframe to return to strategy """ @@ -422,7 +421,7 @@ class FreqaiDataDrawer: dk.model_filename = self.pair_dict[coin]["model_filename"] dk.data_path = Path(self.pair_dict[coin]["data_path"]) if self.freqai_info.get("follow_mode", False): - # follower can be on a different system which is rsynced to the leader: + # follower can be on a different system which is rsynced from the leader: dk.data_path = Path( self.config["user_data_dir"] / "models" diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index 4bee3fefd..ec69a78c4 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -11,8 +11,8 @@ from pathlib import Path from typing import Any, Dict, Tuple import numpy as np -import numpy.typing as npt import pandas as pd +from numpy.typing import ArrayLike from pandas import DataFrame from freqtrade.configuration import TimeRange @@ -61,19 +61,21 @@ class IFreqaiModel(ABC): self.config = config self.assert_config(self.config) - self.freqai_info = config["freqai"] - self.data_split_parameters = config.get("freqai", {}).get("data_split_parameters") - self.model_training_parameters = config.get("freqai", {}).get("model_training_parameters") + self.freqai_info: Dict[str, Any] = config["freqai"] + self.data_split_parameters: Dict[str, Any] = config.get("freqai", {}).get( + "data_split_parameters", {}) + self.model_training_parameters: Dict[str, Any] = config.get("freqai", {}).get( + "model_training_parameters", {}) self.feature_parameters = config.get("freqai", {}).get("feature_parameters") self.retrain = False self.first = True self.set_full_path() - self.follow_mode = self.freqai_info.get("follow_mode", False) + self.follow_mode: bool = self.freqai_info.get("follow_mode", False) self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode) self.lock = threading.Lock() - self.identifier = self.freqai_info.get("identifier", "no_id_provided") + self.identifier: str = self.freqai_info.get("identifier", "no_id_provided") self.scanning = False - self.keras = self.freqai_info.get("keras", False) + self.keras: bool = self.freqai_info.get("keras", False) if self.keras and self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0): self.freqai_info["feature_parameters"]["DI_threshold"] = 0 logger.warning("DI threshold is not configured for Keras models yet. Deactivating.") @@ -253,7 +255,7 @@ class IFreqaiModel(ABC): # get the model metadata associated with the current pair (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"]) - # if the metadata doesnt exist, the follower returns null arrays to strategy + # if the metadata doesn't exist, the follower returns null arrays to strategy if self.follow_mode and return_null_array: logger.info("Returning null array from follower to strategy") self.dd.return_null_values_to_strategy(dataframe, dk) @@ -364,7 +366,7 @@ class IFreqaiModel(ABC): raise OperationalException( "Trying to access pretrained model with `identifier` " "but found different features furnished by current strategy." - "Change `identifer` to train from scratch, or ensure the" + "Change `identifier` to train from scratch, or ensure the" "strategy is furnishing the same features as the pretrained" "model" ) @@ -457,7 +459,7 @@ class IFreqaiModel(ABC): data_load_timerange: TimeRange, ): """ - Retreive data and train model in single threaded mode (only used if model directory is empty + Retrieve data and train model in single threaded mode (only used if model directory is empty upon startup for dry/live ) :param new_trained_timerange: TimeRange = the timerange to train the model on :param metadata: dict = strategy provided metadata @@ -548,7 +550,7 @@ class IFreqaiModel(ABC): @abstractmethod def predict( self, dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = True - ) -> Tuple[DataFrame, npt.ArrayLike]: + ) -> Tuple[DataFrame, ArrayLike]: """ Filter the prediction features data and predict with it. :param unfiltered_dataframe: Full dataframe for the current backtest period. diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 713a2da07..abc90a685 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -23,13 +23,14 @@ class StoplossGuard(IProtection): self._trade_limit = protection_config.get('trade_limit', 10) self._disable_global_stop = protection_config.get('only_per_pair', False) self._only_per_side = protection_config.get('only_per_side', False) + self._profit_limit = protection_config.get('required_profit', 0.0) def short_desc(self) -> str: """ Short method description - used for startup-messages """ return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses " - f"within {self.lookback_period_str}.") + f"with profit < {self._profit_limit:.2%} within {self.lookback_period_str}.") def _reason(self) -> str: """ @@ -49,7 +50,7 @@ class StoplossGuard(IProtection): trades = [trade for trade in trades1 if (str(trade.exit_reason) in ( ExitType.TRAILING_STOP_LOSS.value, ExitType.STOP_LOSS.value, ExitType.STOPLOSS_ON_EXCHANGE.value) - and trade.close_profit and trade.close_profit < 0)] + and trade.close_profit and trade.close_profit < self._profit_limit)] if self._only_per_side: # Long or short trades only diff --git a/requirements-dev.txt b/requirements-dev.txt index 5de839f0a..f054718a1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 flake8==4.0.1 flake8-tidy-imports==4.8.0 -mypy==0.961 +mypy==0.971 pre-commit==2.20.0 pytest==7.1.2 pytest-asyncio==0.19.0 @@ -25,6 +25,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.1 +types-requests==2.28.3 types-tabulate==0.8.11 -types-python-dateutil==2.8.18 +types-python-dateutil==2.8.19 diff --git a/requirements.txt b/requirements.txt index b27c8f559..b9e87749d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.1 pandas==1.4.3 pandas-ta==0.3.14b -ccxt==1.90.89 +ccxt==1.91.29 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.4 aiohttp==3.8.1 @@ -28,7 +28,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.8 # Properly format api responses -orjson==3.7.7 +orjson==3.7.8 # Notify systemd sdnotify==0.3.2 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ff8b4b40c..9252040ea 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -2910,6 +2910,9 @@ def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order, ({'amount': 10.0, 'fee': {}}, False), ({'result': 'testest123'}, False), ('hello_world', False), + ({'status': 'canceled', 'amount': None, 'fee': None}, False), + ({'status': 'canceled', 'filled': None, 'amount': None, 'fee': None}, False), + ]) def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, order, result): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 3c333200c..4cebb6492 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -424,7 +424,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " - "2 stoplosses within 60 minutes.'}]", + "2 stoplosses with profit < 0.00% within 60 minutes.'}]", None ), ({"method": "CooldownPeriod", "stop_duration": 60}, @@ -442,9 +442,9 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): None ), ({"method": "StoplossGuard", "lookback_period_candles": 12, "trade_limit": 2, - "stop_duration": 60}, + "required_profit": -0.05, "stop_duration": 60}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " - "2 stoplosses within 12 candles.'}]", + "2 stoplosses with profit < -5.00% within 12 candles.'}]", None ), ({"method": "CooldownPeriod", "stop_duration_candles": 5}, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 3afdd5245..e6b7c4dd2 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1402,7 +1402,6 @@ def test_api_strategies(botclient): 'InformativeDecoratorTest', 'StrategyTestV2', 'StrategyTestV3', - 'StrategyTestV3Analysis', 'StrategyTestV3Futures', 'freqai_test_multimodel_strat', 'freqai_test_strat' diff --git a/tests/strategy/strats/strategy_test_v3_analysis.py b/tests/strategy/strats/strategy_test_v3_analysis.py deleted file mode 100644 index 290fef156..000000000 --- a/tests/strategy/strats/strategy_test_v3_analysis.py +++ /dev/null @@ -1,175 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -import talib.abstract as ta -from pandas import DataFrame - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy, - RealParameter) - - -class StrategyTestV3Analysis(IStrategy): - """ - Strategy used by tests freqtrade bot. - Please do not modify this strategy, it's intended for internal use only. - Please look at the SampleStrategy in the user_data/strategy directory - or strategy repository https://github.com/freqtrade/freqtrade-strategies - for samples and inspiration. - """ - INTERFACE_VERSION = 3 - - # Minimal ROI designed for the strategy - minimal_roi = { - "40": 0.0, - "30": 0.01, - "20": 0.02, - "0": 0.04 - } - - # Optimal stoploss designed for the strategy - stoploss = -0.10 - - # Optimal timeframe for the strategy - timeframe = '5m' - - # Optional order type mapping - order_types = { - 'entry': 'limit', - 'exit': 'limit', - 'stoploss': 'limit', - 'stoploss_on_exchange': False - } - - # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 20 - - # Optional time in force for orders - order_time_in_force = { - 'entry': 'gtc', - 'exit': 'gtc', - } - - buy_params = { - 'buy_rsi': 35, - # Intentionally not specified, so "default" is tested - # 'buy_plusdi': 0.4 - } - - sell_params = { - 'sell_rsi': 74, - 'sell_minusdi': 0.4 - } - - buy_rsi = IntParameter([0, 50], default=30, space='buy') - buy_plusdi = RealParameter(low=0, high=1, default=0.5, space='buy') - sell_rsi = IntParameter(low=50, high=100, default=70, space='sell') - sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell', - load=False) - protection_enabled = BooleanParameter(default=True) - protection_cooldown_lookback = IntParameter([0, 50], default=30) - - # TODO: Can this work with protection tests? (replace HyperoptableStrategy implicitly ... ) - # @property - # def protections(self): - # prot = [] - # if self.protection_enabled.value: - # prot.append({ - # "method": "CooldownPeriod", - # "stop_duration_candles": self.protection_cooldown_lookback.value - # }) - # return prot - - bot_started = False - - def bot_start(self): - self.bot_started = True - - def informative_pairs(self): - - return [] - - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - - # Minus Directional Indicator / Movement - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # Plus Directional Indicator / Movement - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - - # EMA - Exponential Moving Average - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - - return dataframe - - def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - - dataframe.loc[ - ( - (dataframe['rsi'] < self.buy_rsi.value) & - (dataframe['fastd'] < 35) & - (dataframe['adx'] > 30) & - (dataframe['plus_di'] > self.buy_plusdi.value) - ) | - ( - (dataframe['adx'] > 65) & - (dataframe['plus_di'] > self.buy_plusdi.value) - ), - ['enter_long', 'enter_tag']] = 1, 'enter_tag_long' - - dataframe.loc[ - ( - qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) - ), - ['enter_short', 'enter_tag']] = 1, 'enter_tag_short' - - return dataframe - - def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - ( - ( - (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) | - (qtpylib.crossed_above(dataframe['fastd'], 70)) - ) & - (dataframe['adx'] > 10) & - (dataframe['minus_di'] > 0) - ) | - ( - (dataframe['adx'] > 70) & - (dataframe['minus_di'] > self.sell_minusdi.value) - ), - ['exit_long', 'exit_tag']] = 1, 'exit_tag_long' - - dataframe.loc[ - ( - qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) - ), - ['exit_long', 'exit_tag']] = 1, 'exit_tag_short' - - return dataframe diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 5b6f15d11..aaad26e5b 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -34,7 +34,7 @@ def test_search_all_strategies_no_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) assert isinstance(strategies, list) - assert len(strategies) == 9 + assert len(strategies) == 8 assert isinstance(strategies[0], dict) @@ -42,10 +42,10 @@ def test_search_all_strategies_with_failed(): directory = Path(__file__).parent / "strats" strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) assert isinstance(strategies, list) - assert len(strategies) == 10 + assert len(strategies) == 9 # with enum_failed=True search_all_objects() shall find 2 good strategies # and 1 which fails to load - assert len([x for x in strategies if x['class'] is not None]) == 9 + assert len([x for x in strategies if x['class'] is not None]) == 8 assert len([x for x in strategies if x['class'] is None]) == 1