Merge pull request #3394 from freqtrade/disable_dataframechecks

Allow changing severity of strategy-validations to log only.

config_full.json.example

@@ -132,6 +132,7 @@
         "process_throttle_secs": 5,
         "heartbeat_interval": 60
     },
+    "disable_dataframe_checks": false,
     "strategy": "DefaultStrategy",
     "strategy_path": "user_data/strategies/",
     "dataformat_ohlcv": "json",

docs/configuration.md

@@ -107,6 +107,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances. <br> **Datatype:** String, SQLAlchemy connect string
 | `initial_state` | Defines the initial application state. More information below. <br>*Defaults to `stopped`.* <br> **Datatype:** Enum, either `stopped` or `running`
 | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below. <br> **Datatype:** Boolean
+| `disable_dataframe_checks` | Disable checking the OHLCV dataframe returned from the strategy methods for correctness. Only use when intentionally changing the dataframe and understand what you are doing. [Strategy Override](#parameters-in-the-strategy).[Strategy Override](#parameters-in-the-strategy).<br> *Defaults to `False`*. <br> **Datatype:** Boolean
 | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`. <br> **Datatype:** ClassName
 | `strategy_path` | Adds an additional strategy lookup path (must be a directory). <br> **Datatype:** String
 | `internals.process_throttle_secs` | Set the process throttle. Value in second. <br>*Defaults to `5` seconds.* <br> **Datatype:** Positive Integer
@@ -135,6 +136,7 @@ Values set in the configuration file always overwrite values set in the strategy
 * `stake_currency`
 * `stake_amount`
 * `unfilledtimeout`
+* `disable_dataframe_checks`
 * `use_sell_signal` (ask_strategy)
 * `sell_profit_only` (ask_strategy)
 * `ignore_roi_if_buy_signal` (ask_strategy)

freqtrade/constants.py

@@ -227,6 +227,7 @@ CONF_SCHEMA = {
         'db_url': {'type': 'string'},
         'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
         'forcebuy_enable': {'type': 'boolean'},
+        'disable_dataframe_checks': {'type': 'boolean'},
         'internals': {
             'type': 'object',
             'default': {},

freqtrade/resolvers/strategy_resolver.py

@@ -52,37 +52,38 @@ class StrategyResolver(IResolver):
 
         # Set attributes
         # Check if we need to override configuration
-        #             (Attribute name,                    default,     ask_strategy)
+        #             (Attribute name,                    default,     subkey)
-        attributes = [("minimal_roi",                     {"0": 10.0}, False),
+        attributes = [("minimal_roi",                     {"0": 10.0}, None),
-                      ("ticker_interval",                 None,        False),
+                      ("ticker_interval",                 None,        None),
-                      ("stoploss",                        None,        False),
+                      ("stoploss",                        None,        None),
-                      ("trailing_stop",                   None,        False),
+                      ("trailing_stop",                   None,        None),
-                      ("trailing_stop_positive",          None,        False),
+                      ("trailing_stop_positive",          None,        None),
-                      ("trailing_stop_positive_offset",   0.0,         False),
+                      ("trailing_stop_positive_offset",   0.0,         None),
-                      ("trailing_only_offset_is_reached", None,        False),
+                      ("trailing_only_offset_is_reached", None,        None),
-                      ("process_only_new_candles",        None,        False),
+                      ("process_only_new_candles",        None,        None),
-                      ("order_types",                     None,        False),
+                      ("order_types",                     None,        None),
-                      ("order_time_in_force",             None,        False),
+                      ("order_time_in_force",             None,        None),
-                      ("stake_currency",                  None,        False),
+                      ("stake_currency",                  None,        None),
-                      ("stake_amount",                    None,        False),
+                      ("stake_amount",                    None,        None),
-                      ("startup_candle_count",            None,        False),
+                      ("startup_candle_count",            None,        None),
-                      ("unfilledtimeout",                 None,        False),
+                      ("unfilledtimeout",                 None,        None),
-                      ("use_sell_signal",                 True,        True),
+                      ("use_sell_signal",                 True,        'ask_strategy'),
-                      ("sell_profit_only",                False,       True),
+                      ("sell_profit_only",                False,       'ask_strategy'),
-                      ("ignore_roi_if_buy_signal",        False,       True),
+                      ("ignore_roi_if_buy_signal",        False,       'ask_strategy'),
+                      ("disable_dataframe_checks",        False,       'internals')
                       ]
-        for attribute, default, ask_strategy in attributes:
+        for attribute, default, subkey in attributes:
-            if ask_strategy:
+            if subkey:
-                StrategyResolver._override_attribute_helper(strategy, config['ask_strategy'],
+                StrategyResolver._override_attribute_helper(strategy, config.get(subkey, {}),
                                                             attribute, default)
             else:
                 StrategyResolver._override_attribute_helper(strategy, config,
                                                             attribute, default)
 
         # Loop this list again to have output combined
-        for attribute, _, exp in attributes:
+        for attribute, _, subkey in attributes:
-            if exp and attribute in config['ask_strategy']:
+            if subkey and attribute in config[subkey]:
-                logger.info("Strategy using %s: %s", attribute, config['ask_strategy'][attribute])
+                logger.info("Strategy using %s: %s", attribute, config[subkey][attribute])
             elif attribute in config:
                 logger.info("Strategy using %s: %s", attribute, config[attribute])
 

freqtrade/strategy/interface.py

@@ -106,6 +106,9 @@ class IStrategy(ABC):
     # run "populate_indicators" only for new candle
     process_only_new_candles: bool = False
 
+    # Disable checking the dataframe (converts the error into a warning message)
+    disable_dataframe_checks: bool = False
+
     # Count of candles the strategy requires before producing valid signals
     startup_candle_count: int = 0
 
@@ -285,8 +288,7 @@ class IStrategy(ABC):
         """ keep some data for dataframes """
         return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1]
 
-    @staticmethod
+    def assert_df(self, dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime):
-    def assert_df(dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime):
         """ make sure data is unmodified """
         message = ""
         if df_len != len(dataframe):
@@ -296,7 +298,10 @@ class IStrategy(ABC):
         elif df_date != dataframe["date"].iloc[-1]:
             message = "last date"
         if message:
-            raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.")
+            if self.disable_dataframe_checks:
+                logger.warning(f"Dataframe returned from strategy has mismatching {message}.")
+            else:
+                raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.")
 
     def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
         """

tests/strategy/test_interface.py

@@ -130,7 +130,7 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history):
                    caplog)
 
 
-def test_assert_df(default_conf, mocker, ohlcv_history):
+def test_assert_df(default_conf, mocker, ohlcv_history, caplog):
     # Ensure it's running when passed correctly
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
                         ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
@@ -148,6 +148,14 @@ def test_assert_df(default_conf, mocker, ohlcv_history):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
                             ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
 
+    _STRATEGY.disable_dataframe_checks = True
+    caplog.clear()
+    _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
+                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+    assert log_has_re(r"Dataframe returned from strategy.*last date\.", caplog)
+    # reset to avoid problems in other tests due to test leakage
+    _STRATEGY.disable_dataframe_checks = False
+
 
 def test_get_signal_handles_exceptions(mocker, default_conf):
     exchange = get_patched_exchange(mocker, default_conf)