Merge branch 'feat/freqai' of github.com:lolongcovas/freqtrade into feat/freqai

.pre-commit-config.yaml

@@ -15,9 +15,9 @@ repos:
         additional_dependencies:
           - types-cachetools==5.2.1
           - types-filelock==3.2.7
-          - types-requests==2.28.1
+          - types-requests==2.28.3
           - types-tabulate==0.8.11
-          - types-python-dateutil==2.8.18
+          - types-python-dateutil==2.8.19
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

docs/includes/protections.md

@@ -50,6 +50,8 @@ This applies across all pairs, unless `only_per_pair` is set to true, which will
 
 Similarly, this protection will by default look at all trades (long and short). For futures bots, setting `only_per_side` will make the bot only consider one side, and will then only lock this one side, allowing for example shorts to continue after a series of long stoplosses.
 
+`required_profit` will determine the required relative profit (or loss) for stoplosses to consider. This should normally not be set and defaults to 0.0 - which means all losing stoplosses will be triggering a block.
+
 The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles.
 
 ``` python
@@ -61,6 +63,7 @@ def protections(self):
             "lookback_period_candles": 24,
             "trade_limit": 4,
             "stop_duration_candles": 4,
+            "required_profit": 0.0,
             "only_per_pair": False,
             "only_per_side": False
         }

docs/requirements-docs.txt

@@ -1,5 +1,5 @@
-markdown==3.4.1
+markdown==3.3.7
-mkdocs==1.3.0
+mkdocs==1.3.1
 mkdocs-material==8.3.9
 mdx_truly_sane_lists==1.3
 pymdown-extensions==9.5

freqtrade/exchange/exchange.py

@@ -1264,7 +1264,7 @@ class Exchange:
             return False
 
         required = ('fee', 'status', 'amount')
-        return all(k in corder for k in required)
+        return all(corder.get(k, None) is not None for k in required)
 
     def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
         """

freqtrade/freqai/data_drawer.py

@@ -8,10 +8,10 @@ from pathlib import Path
 from typing import Any, Dict, Tuple
 
 import numpy as np
-import numpy.typing as npt
 import pandas as pd
 from joblib import dump, load
 from joblib.externals import cloudpickle
+from numpy.typing import ArrayLike
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
@@ -81,8 +81,7 @@ class FreqaiDataDrawer:
         """
         Locate and load a previously saved data drawer full of all pair model metadata in
         present model folder.
-        :returns:
+        :return: bool - whether or not the drawer was located
-        exists: bool = whether or not the drawer was located
         """
         exists = self.pair_dictionary_path.is_file()
         if exists:
@@ -101,8 +100,7 @@ class FreqaiDataDrawer:
     def load_historic_predictions_from_disk(self):
         """
         Locate and load a previously saved historic predictions.
-        :returns:
+        :return: bool - whether or not the drawer was located
-        exists: bool = whether or not the drawer was located
         """
         exists = self.historic_predictions_path.is_file()
         if exists:
@@ -221,7 +219,7 @@ class FreqaiDataDrawer:
         self.pair_dict[pair]["priority"] = len(self.pair_dict)
 
     def set_initial_return_values(self, pair: str, dk: FreqaiDataKitchen,
-                                  pred_df: DataFrame, do_preds: npt.ArrayLike) -> None:
+                                  pred_df: DataFrame, do_preds: ArrayLike) -> None:
         """
         Set the initial return values to a persistent dataframe. This avoids needing to repredict on
         historical candles, and also stores historical predictions despite retrainings (so stored
@@ -240,7 +238,8 @@ class FreqaiDataDrawer:
 
         mrv_df["do_predict"] = do_preds
 
-    def append_model_predictions(self, pair: str, predictions, do_preds, dk, len_df) -> None:
+    def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: ArrayLike,
+                                 dk: FreqaiDataKitchen, len_df: int) -> None:
 
         # strat seems to feed us variable sized dataframes - and since we are trying to build our
         # own return array in the same shape, we need to figure out how the size has changed
@@ -295,7 +294,7 @@ class FreqaiDataDrawer:
         dataframe = pd.concat([dataframe[to_keep], df], axis=1)
         return dataframe
 
-    def return_null_values_to_strategy(self, dataframe: DataFrame, dk) -> None:
+    def return_null_values_to_strategy(self, dataframe: DataFrame, dk: FreqaiDataKitchen) -> None:
         """
         Build 0 filled dataframe to return to strategy
         """
@@ -422,7 +421,7 @@ class FreqaiDataDrawer:
             dk.model_filename = self.pair_dict[coin]["model_filename"]
             dk.data_path = Path(self.pair_dict[coin]["data_path"])
             if self.freqai_info.get("follow_mode", False):
-                # follower can be on a different system which is rsynced to the leader:
+                # follower can be on a different system which is rsynced from the leader:
                 dk.data_path = Path(
                     self.config["user_data_dir"]
                     / "models"

freqtrade/freqai/freqai_interface.py

@@ -11,8 +11,8 @@ from pathlib import Path
 from typing import Any, Dict, Tuple
 
 import numpy as np
-import numpy.typing as npt
 import pandas as pd
+from numpy.typing import ArrayLike
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
@@ -61,19 +61,21 @@ class IFreqaiModel(ABC):
 
         self.config = config
         self.assert_config(self.config)
-        self.freqai_info = config["freqai"]
+        self.freqai_info: Dict[str, Any] = config["freqai"]
-        self.data_split_parameters = config.get("freqai", {}).get("data_split_parameters")
+        self.data_split_parameters: Dict[str, Any] = config.get("freqai", {}).get(
-        self.model_training_parameters = config.get("freqai", {}).get("model_training_parameters")
+            "data_split_parameters", {})
+        self.model_training_parameters: Dict[str, Any] = config.get("freqai", {}).get(
+            "model_training_parameters", {})
         self.feature_parameters = config.get("freqai", {}).get("feature_parameters")
         self.retrain = False
         self.first = True
         self.set_full_path()
-        self.follow_mode = self.freqai_info.get("follow_mode", False)
+        self.follow_mode: bool = self.freqai_info.get("follow_mode", False)
         self.dd = FreqaiDataDrawer(Path(self.full_path), self.config, self.follow_mode)
         self.lock = threading.Lock()
-        self.identifier = self.freqai_info.get("identifier", "no_id_provided")
+        self.identifier: str = self.freqai_info.get("identifier", "no_id_provided")
         self.scanning = False
-        self.keras = self.freqai_info.get("keras", False)
+        self.keras: bool = self.freqai_info.get("keras", False)
         if self.keras and self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0):
             self.freqai_info["feature_parameters"]["DI_threshold"] = 0
             logger.warning("DI threshold is not configured for Keras models yet. Deactivating.")
@@ -253,7 +255,7 @@ class IFreqaiModel(ABC):
         # get the model metadata associated with the current pair
         (_, trained_timestamp, return_null_array) = self.dd.get_pair_dict_info(metadata["pair"])
 
-        # if the metadata doesnt exist, the follower returns null arrays to strategy
+        # if the metadata doesn't exist, the follower returns null arrays to strategy
         if self.follow_mode and return_null_array:
             logger.info("Returning null array from follower to strategy")
             self.dd.return_null_values_to_strategy(dataframe, dk)
@@ -364,7 +366,7 @@ class IFreqaiModel(ABC):
             raise OperationalException(
                 "Trying to access pretrained model with `identifier` "
                 "but found different features furnished by current strategy."
-                "Change `identifer` to train from scratch, or ensure the"
+                "Change `identifier` to train from scratch, or ensure the"
                 "strategy is furnishing the same features as the pretrained"
                 "model"
             )
@@ -457,7 +459,7 @@ class IFreqaiModel(ABC):
         data_load_timerange: TimeRange,
     ):
         """
-        Retreive data and train model in single threaded mode (only used if model directory is empty
+        Retrieve data and train model in single threaded mode (only used if model directory is empty
         upon startup for dry/live )
         :param new_trained_timerange: TimeRange = the timerange to train the model on
         :param metadata: dict = strategy provided metadata
@@ -548,7 +550,7 @@ class IFreqaiModel(ABC):
     @abstractmethod
     def predict(
         self, dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = True
-    ) -> Tuple[DataFrame, npt.ArrayLike]:
+    ) -> Tuple[DataFrame, ArrayLike]:
         """
         Filter the prediction features data and predict with it.
         :param unfiltered_dataframe: Full dataframe for the current backtest period.

freqtrade/plugins/protections/stoploss_guard.py

@@ -23,13 +23,14 @@ class StoplossGuard(IProtection):
         self._trade_limit = protection_config.get('trade_limit', 10)
         self._disable_global_stop = protection_config.get('only_per_pair', False)
         self._only_per_side = protection_config.get('only_per_side', False)
+        self._profit_limit = protection_config.get('required_profit', 0.0)
 
     def short_desc(self) -> str:
         """
         Short method description - used for startup-messages
         """
         return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses "
-                f"within {self.lookback_period_str}.")
+                f"with profit < {self._profit_limit:.2%} within {self.lookback_period_str}.")
 
     def _reason(self) -> str:
         """
@@ -49,7 +50,7 @@ class StoplossGuard(IProtection):
         trades = [trade for trade in trades1 if (str(trade.exit_reason) in (
             ExitType.TRAILING_STOP_LOSS.value, ExitType.STOP_LOSS.value,
             ExitType.STOPLOSS_ON_EXCHANGE.value)
-            and trade.close_profit and trade.close_profit < 0)]
+            and trade.close_profit and trade.close_profit < self._profit_limit)]
 
         if self._only_per_side:
             # Long or short trades only

requirements-dev.txt

@@ -8,7 +8,7 @@
 coveralls==3.3.1
 flake8==4.0.1
 flake8-tidy-imports==4.8.0
-mypy==0.961
+mypy==0.971
 pre-commit==2.20.0
 pytest==7.1.2
 pytest-asyncio==0.19.0
@@ -25,6 +25,6 @@ nbconvert==6.5.0
 # mypy types
 types-cachetools==5.2.1
 types-filelock==3.2.7
-types-requests==2.28.1
+types-requests==2.28.3
 types-tabulate==0.8.11
-types-python-dateutil==2.8.18
+types-python-dateutil==2.8.19

requirements.txt

@@ -2,7 +2,7 @@ numpy==1.23.1
 pandas==1.4.3
 pandas-ta==0.3.14b
 
-ccxt==1.90.89
+ccxt==1.91.29
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==37.0.4
 aiohttp==3.8.1
@@ -28,7 +28,7 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.8
 # Properly format api responses
-orjson==3.7.7
+orjson==3.7.8
 
 # Notify systemd
 sdnotify==0.3.2

tests/exchange/test_exchange.py

@@ -2910,6 +2910,9 @@ def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order,
     ({'amount': 10.0, 'fee': {}}, False),
     ({'result': 'testest123'}, False),
     ('hello_world', False),
+    ({'status': 'canceled', 'amount': None, 'fee': None}, False),
+    ({'status': 'canceled', 'filled': None, 'amount': None, 'fee': None}, False),
+
 ])
 def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, order, result):
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)

tests/plugins/test_protections.py

@@ -424,7 +424,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
 @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [
     ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60},
      "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, "
-     "2 stoplosses within 60 minutes.'}]",
+     "2 stoplosses with profit < 0.00% within 60 minutes.'}]",
      None
      ),
     ({"method": "CooldownPeriod", "stop_duration": 60},
@@ -442,9 +442,9 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog):
      None
      ),
     ({"method": "StoplossGuard", "lookback_period_candles": 12, "trade_limit": 2,
-      "stop_duration": 60},
+      "required_profit": -0.05, "stop_duration": 60},
      "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, "
-     "2 stoplosses within 12 candles.'}]",
+     "2 stoplosses with profit < -5.00% within 12 candles.'}]",
      None
      ),
     ({"method": "CooldownPeriod", "stop_duration_candles": 5},

tests/rpc/test_rpc_apiserver.py

@@ -1402,7 +1402,6 @@ def test_api_strategies(botclient):
         'InformativeDecoratorTest',
         'StrategyTestV2',
         'StrategyTestV3',
-        'StrategyTestV3Analysis',
         'StrategyTestV3Futures',
         'freqai_test_multimodel_strat',
         'freqai_test_strat'

tests/strategy/strats/strategy_test_v3_analysis.py

@@ -1,175 +0,0 @@
-# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
-
-import talib.abstract as ta
-from pandas import DataFrame
-
-import freqtrade.vendor.qtpylib.indicators as qtpylib
-from freqtrade.strategy import (BooleanParameter, DecimalParameter, IntParameter, IStrategy,
-                                RealParameter)
-
-
-class StrategyTestV3Analysis(IStrategy):
-    """
-    Strategy used by tests freqtrade bot.
-    Please do not modify this strategy, it's  intended for internal use only.
-    Please look at the SampleStrategy in the user_data/strategy directory
-    or strategy repository https://github.com/freqtrade/freqtrade-strategies
-    for samples and inspiration.
-    """
-    INTERFACE_VERSION = 3
-
-    # Minimal ROI designed for the strategy
-    minimal_roi = {
-        "40": 0.0,
-        "30": 0.01,
-        "20": 0.02,
-        "0": 0.04
-    }
-
-    # Optimal stoploss designed for the strategy
-    stoploss = -0.10
-
-    # Optimal timeframe for the strategy
-    timeframe = '5m'
-
-    # Optional order type mapping
-    order_types = {
-        'entry': 'limit',
-        'exit': 'limit',
-        'stoploss': 'limit',
-        'stoploss_on_exchange': False
-    }
-
-    # Number of candles the strategy requires before producing valid signals
-    startup_candle_count: int = 20
-
-    # Optional time in force for orders
-    order_time_in_force = {
-        'entry': 'gtc',
-        'exit': 'gtc',
-    }
-
-    buy_params = {
-        'buy_rsi': 35,
-        # Intentionally not specified, so "default" is tested
-        # 'buy_plusdi': 0.4
-    }
-
-    sell_params = {
-        'sell_rsi': 74,
-        'sell_minusdi': 0.4
-    }
-
-    buy_rsi = IntParameter([0, 50], default=30, space='buy')
-    buy_plusdi = RealParameter(low=0, high=1, default=0.5, space='buy')
-    sell_rsi = IntParameter(low=50, high=100, default=70, space='sell')
-    sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell',
-                                    load=False)
-    protection_enabled = BooleanParameter(default=True)
-    protection_cooldown_lookback = IntParameter([0, 50], default=30)
-
-    # TODO: Can this work with protection tests? (replace HyperoptableStrategy implicitly ... )
-    # @property
-    # def protections(self):
-    #     prot = []
-    #     if self.protection_enabled.value:
-    #         prot.append({
-    #             "method": "CooldownPeriod",
-    #             "stop_duration_candles": self.protection_cooldown_lookback.value
-    #         })
-    #     return prot
-
-    bot_started = False
-
-    def bot_start(self):
-        self.bot_started = True
-
-    def informative_pairs(self):
-
-        return []
-
-    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-
-        # Momentum Indicator
-        # ------------------------------------
-
-        # ADX
-        dataframe['adx'] = ta.ADX(dataframe)
-
-        # MACD
-        macd = ta.MACD(dataframe)
-        dataframe['macd'] = macd['macd']
-        dataframe['macdsignal'] = macd['macdsignal']
-        dataframe['macdhist'] = macd['macdhist']
-
-        # Minus Directional Indicator / Movement
-        dataframe['minus_di'] = ta.MINUS_DI(dataframe)
-
-        # Plus Directional Indicator / Movement
-        dataframe['plus_di'] = ta.PLUS_DI(dataframe)
-
-        # RSI
-        dataframe['rsi'] = ta.RSI(dataframe)
-
-        # Stoch fast
-        stoch_fast = ta.STOCHF(dataframe)
-        dataframe['fastd'] = stoch_fast['fastd']
-        dataframe['fastk'] = stoch_fast['fastk']
-
-        # Bollinger bands
-        bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
-        dataframe['bb_lowerband'] = bollinger['lower']
-        dataframe['bb_middleband'] = bollinger['mid']
-        dataframe['bb_upperband'] = bollinger['upper']
-
-        # EMA - Exponential Moving Average
-        dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
-
-        return dataframe
-
-    def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-
-        dataframe.loc[
-            (
-                (dataframe['rsi'] < self.buy_rsi.value) &
-                (dataframe['fastd'] < 35) &
-                (dataframe['adx'] > 30) &
-                (dataframe['plus_di'] > self.buy_plusdi.value)
-            ) |
-            (
-                (dataframe['adx'] > 65) &
-                (dataframe['plus_di'] > self.buy_plusdi.value)
-            ),
-            ['enter_long', 'enter_tag']] = 1, 'enter_tag_long'
-
-        dataframe.loc[
-            (
-                qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value)
-            ),
-            ['enter_short', 'enter_tag']] = 1, 'enter_tag_short'
-
-        return dataframe
-
-    def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        dataframe.loc[
-            (
-                (
-                    (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) |
-                    (qtpylib.crossed_above(dataframe['fastd'], 70))
-                ) &
-                (dataframe['adx'] > 10) &
-                (dataframe['minus_di'] > 0)
-            ) |
-            (
-                (dataframe['adx'] > 70) &
-                (dataframe['minus_di'] > self.sell_minusdi.value)
-            ),
-            ['exit_long', 'exit_tag']] = 1, 'exit_tag_long'
-
-        dataframe.loc[
-            (
-                qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value)
-            ),
-            ['exit_long', 'exit_tag']] = 1, 'exit_tag_short'
-
-        return dataframe

tests/strategy/test_strategy_loading.py

@@ -34,7 +34,7 @@ def test_search_all_strategies_no_failed():
     directory = Path(__file__).parent / "strats"
     strategies = StrategyResolver.search_all_objects(directory, enum_failed=False)
     assert isinstance(strategies, list)
-    assert len(strategies) == 9
+    assert len(strategies) == 8
     assert isinstance(strategies[0], dict)
 
 
@@ -42,10 +42,10 @@ def test_search_all_strategies_with_failed():
     directory = Path(__file__).parent / "strats"
     strategies = StrategyResolver.search_all_objects(directory, enum_failed=True)
     assert isinstance(strategies, list)
-    assert len(strategies) == 10
+    assert len(strategies) == 9
     # with enum_failed=True search_all_objects() shall find 2 good strategies
     # and 1 which fails to load
-    assert len([x for x in strategies if x['class'] is not None]) == 9
+    assert len([x for x in strategies if x['class'] is not None]) == 8
     assert len([x for x in strategies if x['class'] is None]) == 1