From ea5daee5054621daf1577b2e1cde7336dea0b1fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 May 2020 19:37:18 +0200 Subject: [PATCH 1/6] Allow changing severity of strategy-validations to log only. --- config_full.json.example | 1 + docs/configuration.md | 1 + freqtrade/constants.py | 1 + freqtrade/resolvers/strategy_resolver.py | 49 ++++++++++++------------ freqtrade/strategy/interface.py | 11 ++++-- tests/strategy/test_interface.py | 10 ++++- 6 files changed, 45 insertions(+), 28 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 0cd265cbe..481742817 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -132,6 +132,7 @@ "process_throttle_secs": 5, "heartbeat_interval": 60 }, + "disable_dataframe_checks": false, "strategy": "DefaultStrategy", "strategy_path": "user_data/strategies/", "dataformat_ohlcv": "json", diff --git a/docs/configuration.md b/docs/configuration.md index 93e53de6f..97e6b7911 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -107,6 +107,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string | `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean +| `disable_dataframe_checks` | Disable checking the dataframe for correctness. Only use when intentionally changing the dataframe. [Strategy Override](#parameters-in-the-strategy).[Strategy Override](#parameters-in-the-strategy).
*Defaults to `False`*.
**Datatype:** Boolean | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName | `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String | `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer diff --git a/freqtrade/constants.py b/freqtrade/constants.py index c1bf30f17..1984d4866 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -227,6 +227,7 @@ CONF_SCHEMA = { 'db_url': {'type': 'string'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, + 'disable_dataframe_checks': {'type': 'boolean'}, 'internals': { 'type': 'object', 'default': {}, diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index cddc7c9cd..51cb8fc05 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -52,37 +52,38 @@ class StrategyResolver(IResolver): # Set attributes # Check if we need to override configuration - # (Attribute name, default, ask_strategy) - attributes = [("minimal_roi", {"0": 10.0}, False), - ("ticker_interval", None, False), - ("stoploss", None, False), - ("trailing_stop", None, False), - ("trailing_stop_positive", None, False), - ("trailing_stop_positive_offset", 0.0, False), - ("trailing_only_offset_is_reached", None, False), - ("process_only_new_candles", None, False), - ("order_types", None, False), - ("order_time_in_force", None, False), - ("stake_currency", None, False), - ("stake_amount", None, False), - ("startup_candle_count", None, False), - ("unfilledtimeout", None, False), - ("use_sell_signal", True, True), - ("sell_profit_only", False, True), - ("ignore_roi_if_buy_signal", False, True), + # (Attribute name, default, subkey) + attributes = [("minimal_roi", {"0": 10.0}, None), + ("ticker_interval", None, None), + ("stoploss", None, None), + ("trailing_stop", None, None), + ("trailing_stop_positive", None, None), + ("trailing_stop_positive_offset", 0.0, None), + ("trailing_only_offset_is_reached", None, None), + ("process_only_new_candles", None, None), + ("order_types", None, None), + ("order_time_in_force", None, None), + ("stake_currency", None, None), + ("stake_amount", None, None), + ("startup_candle_count", None, None), + ("unfilledtimeout", None, None), + ("use_sell_signal", True, 'ask_strategy'), + ("sell_profit_only", False, 'ask_strategy'), + ("ignore_roi_if_buy_signal", False, 'ask_strategy'), + ("disable_dataframe_checks", False, 'internals') ] - for attribute, default, ask_strategy in attributes: - if ask_strategy: - StrategyResolver._override_attribute_helper(strategy, config['ask_strategy'], + for attribute, default, subkey in attributes: + if subkey: + StrategyResolver._override_attribute_helper(strategy, config.get(subkey, {}), attribute, default) else: StrategyResolver._override_attribute_helper(strategy, config, attribute, default) # Loop this list again to have output combined - for attribute, _, exp in attributes: - if exp and attribute in config['ask_strategy']: - logger.info("Strategy using %s: %s", attribute, config['ask_strategy'][attribute]) + for attribute, _, subkey in attributes: + if subkey and attribute in config[subkey]: + logger.info("Strategy using %s: %s", attribute, config[subkey][attribute]) elif attribute in config: logger.info("Strategy using %s: %s", attribute, config[attribute]) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 400997baf..ed2344a53 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -106,6 +106,9 @@ class IStrategy(ABC): # run "populate_indicators" only for new candle process_only_new_candles: bool = False + # Disable checking the dataframe (converts the error into a warning message) + disable_dataframe_checks: bool = False + # Count of candles the strategy requires before producing valid signals startup_candle_count: int = 0 @@ -285,8 +288,7 @@ class IStrategy(ABC): """ keep some data for dataframes """ return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1] - @staticmethod - def assert_df(dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime): + def assert_df(self, dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime): """ make sure data is unmodified """ message = "" if df_len != len(dataframe): @@ -296,7 +298,10 @@ class IStrategy(ABC): elif df_date != dataframe["date"].iloc[-1]: message = "last date" if message: - raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.") + if self.disable_dataframe_checks: + logger.warning(f"Dataframe returned from strategy has mismatching {message}.") + else: + raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.") def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]: """ diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index dd6b11a06..55dd07e9e 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -130,7 +130,7 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history): caplog) -def test_assert_df(default_conf, mocker, ohlcv_history): +def test_assert_df(default_conf, mocker, ohlcv_history, caplog): # Ensure it's running when passed correctly _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date']) @@ -148,6 +148,14 @@ def test_assert_df(default_conf, mocker, ohlcv_history): _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date']) + _STRATEGY.disable_dataframe_checks = True + caplog.clear() + _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), + ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date']) + assert log_has_re(r"Dataframe returned from strategy.*last date\.", caplog) + # reset to avoid problems in other tests + _STRATEGY.disable_dataframe_checks = False + def test_get_signal_handles_exceptions(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) From 2ed10aeb9be90fa5989455ae000db507032e24f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 May 2020 19:39:13 +0200 Subject: [PATCH 2/6] Add to missing point in documentation --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index 97e6b7911..0bd913e88 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -136,6 +136,7 @@ Values set in the configuration file always overwrite values set in the strategy * `stake_currency` * `stake_amount` * `unfilledtimeout` +* `disable_dataframe_checks` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) From 7ea59b6d8e5214ad27d2beb8a0970507acc755a2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 May 2020 09:43:50 +0200 Subject: [PATCH 3/6] Update comment (to trigger CI) --- tests/strategy/test_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 55dd07e9e..e5539099b 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -153,7 +153,7 @@ def test_assert_df(default_conf, mocker, ohlcv_history, caplog): _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date']) assert log_has_re(r"Dataframe returned from strategy.*last date\.", caplog) - # reset to avoid problems in other tests + # reset to avoid problems in other tests due to test leakage _STRATEGY.disable_dataframe_checks = False From 28b35178e99182d5381e1941ebc5520f524301a4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 May 2020 11:24:29 +0200 Subject: [PATCH 4/6] Update doc wording Co-authored-by: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0bd913e88..8853b6e05 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -107,7 +107,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string | `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` | `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean -| `disable_dataframe_checks` | Disable checking the dataframe for correctness. Only use when intentionally changing the dataframe. [Strategy Override](#parameters-in-the-strategy).[Strategy Override](#parameters-in-the-strategy).
*Defaults to `False`*.
**Datatype:** Boolean +| `disable_dataframe_checks` | Disable checking the OHLCV dataframe returned from the strategy methods for correctness. Only use when intentionally changing the dataframe and understand what you are doing. [Strategy Override](#parameters-in-the-strategy).[Strategy Override](#parameters-in-the-strategy).
*Defaults to `False`*.
**Datatype:** Boolean | `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName | `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String | `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer From a9c57e51473d2cd6e9a29a84bdcadf23beba90fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 May 2020 11:46:48 +0200 Subject: [PATCH 5/6] Add disable_dataframe_checks to strategy templates --- freqtrade/templates/base_strategy.py.j2 | 4 ++++ freqtrade/templates/sample_strategy.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index c37164568..3ff490d6c 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -57,6 +57,10 @@ class {{ strategy }}(IStrategy): # Run "populate_indicators()" only for new candle. process_only_new_candles = False + # Disable checking the dataframe (converts the error into a warning message) + # Only use if you understand the implications! + disable_dataframe_checks: bool = False + # These values can be overridden in the "ask_strategy" section in the config. use_sell_signal = True sell_profit_only = False diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index f78489173..a70643aee 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -58,6 +58,10 @@ class SampleStrategy(IStrategy): # Run "populate_indicators()" only for new candle. process_only_new_candles = False + # Disable checking the dataframe (converts the error into a warning message) + # Only use if you understand the implications! + disable_dataframe_checks: bool = False + # These values can be overridden in the "ask_strategy" section in the config. use_sell_signal = True sell_profit_only = False From 376c536dd1d1081d5d88092ad63bbbe99e73612f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 30 May 2020 16:23:33 +0200 Subject: [PATCH 6/6] Revert "Add disable_dataframe_checks to strategy templates" This reverts commit a9c57e51473d2cd6e9a29a84bdcadf23beba90fb. --- freqtrade/templates/base_strategy.py.j2 | 4 ---- freqtrade/templates/sample_strategy.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 3ff490d6c..c37164568 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -57,10 +57,6 @@ class {{ strategy }}(IStrategy): # Run "populate_indicators()" only for new candle. process_only_new_candles = False - # Disable checking the dataframe (converts the error into a warning message) - # Only use if you understand the implications! - disable_dataframe_checks: bool = False - # These values can be overridden in the "ask_strategy" section in the config. use_sell_signal = True sell_profit_only = False diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index a70643aee..f78489173 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -58,10 +58,6 @@ class SampleStrategy(IStrategy): # Run "populate_indicators()" only for new candle. process_only_new_candles = False - # Disable checking the dataframe (converts the error into a warning message) - # Only use if you understand the implications! - disable_dataframe_checks: bool = False - # These values can be overridden in the "ask_strategy" section in the config. use_sell_signal = True sell_profit_only = False