From b6b9c8e5cc401bb5f876d74515b652cb5a5e6537 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Oct 2020 07:11:08 +0200 Subject: [PATCH 01/73] Move "slow-log" to it's own mixin --- freqtrade/mixins/__init__.py | 2 ++ freqtrade/mixins/logging_mixin.py | 34 +++++++++++++++++++++++++++++++ freqtrade/pairlist/IPairList.py | 25 +++-------------------- 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 freqtrade/mixins/__init__.py create mode 100644 freqtrade/mixins/logging_mixin.py diff --git a/freqtrade/mixins/__init__.py b/freqtrade/mixins/__init__.py new file mode 100644 index 000000000..f4a640fa3 --- /dev/null +++ b/freqtrade/mixins/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.mixins.logging_mixin import LoggingMixin diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py new file mode 100644 index 000000000..4e19e45a4 --- /dev/null +++ b/freqtrade/mixins/logging_mixin.py @@ -0,0 +1,34 @@ + + +from cachetools import TTLCache, cached + + +class LoggingMixin(): + """ + Logging Mixin + Shows similar messages only once every `refresh_period`. + """ + def __init__(self, logger, refresh_period: int = 3600): + """ + :param refresh_period: in seconds - Show identical messages in this intervals + """ + self.logger = logger + self.refresh_period = refresh_period + self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) + + def log_on_refresh(self, logmethod, message: str) -> None: + """ + Logs message - not more often than "refresh_period" to avoid log spamming + Logs the log-message as debug as well to simplify debugging. + :param logmethod: Function that'll be called. Most likely `logger.info`. + :param message: String containing the message to be sent to the function. + :return: None. + """ + @cached(cache=self._log_cache) + def _log_on_refresh(message: str): + logmethod(message) + + # Log as debug first + self.logger.debug(message) + # Call hidden function. + _log_on_refresh(message) diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index c869e499b..5f29241ce 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -6,16 +6,15 @@ from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, Dict, List -from cachetools import TTLCache, cached - from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active +from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) -class IPairList(ABC): +class IPairList(LoggingMixin, ABC): def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], @@ -36,7 +35,7 @@ class IPairList(ABC): self._pairlist_pos = pairlist_pos self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._last_refresh = 0 - self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) + LoggingMixin.__init__(self, logger, self.refresh_period) @property def name(self) -> str: @@ -46,24 +45,6 @@ class IPairList(ABC): """ return self.__class__.__name__ - def log_on_refresh(self, logmethod, message: str) -> None: - """ - Logs message - not more often than "refresh_period" to avoid log spamming - Logs the log-message as debug as well to simplify debugging. - :param logmethod: Function that'll be called. Most likely `logger.info`. - :param message: String containing the message to be sent to the function. - :return: None. - """ - - @cached(cache=self._log_cache) - def _log_on_refresh(message: str): - logmethod(message) - - # Log as debug first - logger.debug(message) - # Call hidden function. - _log_on_refresh(message) - @abstractproperty def needstickers(self) -> bool: """ From a0bd2ce837bb61a4e335a1980239f536101e3a70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Oct 2020 08:06:29 +0200 Subject: [PATCH 02/73] Add first version of protection manager --- freqtrade/plugins/__init__.py | 0 freqtrade/plugins/protectionmanager.py | 51 ++++++++++++++++++++ freqtrade/plugins/protections/__init__.py | 2 + freqtrade/plugins/protections/iprotection.py | 24 +++++++++ freqtrade/resolvers/__init__.py | 1 + freqtrade/resolvers/protection_resolver.py | 44 +++++++++++++++++ 6 files changed, 122 insertions(+) create mode 100644 freqtrade/plugins/__init__.py create mode 100644 freqtrade/plugins/protectionmanager.py create mode 100644 freqtrade/plugins/protections/__init__.py create mode 100644 freqtrade/plugins/protections/iprotection.py create mode 100644 freqtrade/resolvers/protection_resolver.py diff --git a/freqtrade/plugins/__init__.py b/freqtrade/plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py new file mode 100644 index 000000000..ff64ca789 --- /dev/null +++ b/freqtrade/plugins/protectionmanager.py @@ -0,0 +1,51 @@ +""" +Protection manager class +""" +import logging +from typing import Dict, List + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.protections import IProtection +from freqtrade.resolvers import ProtectionResolver + + +logger = logging.getLogger(__name__) + + +class ProtectionManager(): + + def __init__(self, exchange, config: dict) -> None: + self._exchange = exchange + self._config = config + + self._protection_handlers: List[IProtection] = [] + self._tickers_needed = False + for protection_handler_config in self._config.get('protections', None): + if 'method' not in protection_handler_config: + logger.warning(f"No method found in {protection_handler_config}, ignoring.") + continue + protection_handler = ProtectionResolver.load_protection( + protection_handler_config['method'], + exchange=exchange, + protectionmanager=self, + config=config, + protection_config=protection_handler_config, + ) + self._tickers_needed |= protection_handler.needstickers + self._protection_handlers.append(protection_handler) + + if not self._protection_handlers: + raise OperationalException("No protection Handlers defined") + + @property + def name_list(self) -> List[str]: + """ + Get list of loaded Protection Handler names + """ + return [p.name for p in self._protection_handlers] + + def short_desc(self) -> List[Dict]: + """ + List of short_desc for each Pairlist Handler + """ + return [{p.name: p.short_desc()} for p in self._pairlist_handlers] diff --git a/freqtrade/plugins/protections/__init__.py b/freqtrade/plugins/protections/__init__.py new file mode 100644 index 000000000..5ecae7888 --- /dev/null +++ b/freqtrade/plugins/protections/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.plugins.protections.iprotection import IProtection diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py new file mode 100644 index 000000000..b10856f70 --- /dev/null +++ b/freqtrade/plugins/protections/iprotection.py @@ -0,0 +1,24 @@ + +import logging +from abc import ABC, abstractmethod +from typing import Any, Dict + + +logger = logging.getLogger(__name__) + + +class IProtection(ABC): + + def __init__(self, config: Dict[str, Any]) -> None: + self._config = config + + @property + def name(self) -> str: + return self.__class__.__name__ + + @abstractmethod + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + -> Please overwrite in subclasses + """ diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index b42ec4931..ef24bf481 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -6,6 +6,7 @@ from freqtrade.resolvers.exchange_resolver import ExchangeResolver # Don't import HyperoptResolver to avoid loading the whole Optimize tree # from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.resolvers.pairlist_resolver import PairListResolver +from freqtrade.resolvers.protection_resolver import ProtectionResolver from freqtrade.resolvers.strategy_resolver import StrategyResolver diff --git a/freqtrade/resolvers/protection_resolver.py b/freqtrade/resolvers/protection_resolver.py new file mode 100644 index 000000000..9a85104c3 --- /dev/null +++ b/freqtrade/resolvers/protection_resolver.py @@ -0,0 +1,44 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom pairlists +""" +import logging +from pathlib import Path +from typing import Dict + +from freqtrade.plugins.protections import IProtection +from freqtrade.resolvers import IResolver + + +logger = logging.getLogger(__name__) + + +class ProtectionResolver(IResolver): + """ + This class contains all the logic to load custom PairList class + """ + object_type = IProtection + object_type_str = "Protection" + user_subdir = None + initial_search_path = Path(__file__).parent.parent.joinpath('plugins/protections').resolve() + + @staticmethod + def load_protection(protection_name: str, exchange, protectionmanager, + config: Dict, protection_config: Dict) -> IProtection: + """ + Load the protection with protection_name + :param protection_name: Classname of the pairlist + :param exchange: Initialized exchange class + :param protectionmanager: Initialized protection manager + :param config: configuration dictionary + :param protection_config: Configuration dedicated to this pairlist + :return: initialized Protection class + """ + return ProtectionResolver.load_object(protection_name, config, + kwargs={'exchange': exchange, + 'pairlistmanager': protectionmanager, + 'config': config, + 'pairlistconfig': protection_config, + }, + ) From 3447f1ae531733eabd620b15b849fb48e204ae6d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Oct 2020 07:40:44 +0200 Subject: [PATCH 03/73] Implement first stop method --- config_full.json.example | 7 +++ docs/includes/protections.md | 36 ++++++++++++ freqtrade/plugins/protectionmanager.py | 5 +- freqtrade/plugins/protections/iprotection.py | 11 +++- .../plugins/protections/stoploss_guard.py | 55 +++++++++++++++++++ freqtrade/resolvers/protection_resolver.py | 11 +--- 6 files changed, 112 insertions(+), 13 deletions(-) create mode 100644 docs/includes/protections.md create mode 100644 freqtrade/plugins/protections/stoploss_guard.py diff --git a/config_full.json.example b/config_full.json.example index 5ee2a1faf..96aa82d5f 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -75,6 +75,13 @@ "refresh_period": 1440 } ], + "protections": [ + { + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 4 + } + ], "exchange": { "name": "bittrex", "sandbox": false, diff --git a/docs/includes/protections.md b/docs/includes/protections.md new file mode 100644 index 000000000..078ba0c2b --- /dev/null +++ b/docs/includes/protections.md @@ -0,0 +1,36 @@ +## Protections + +Protections will protect your strategy from unexpected events and market conditions. + +### Available Protection Handlers + +* [`StoplossGuard`](#stoploss-guard) (default, if not configured differently) + +#### Stoploss Guard + +`StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case it will stop trading until this condition is no longer true. + +```json +"protections": [{ + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 4 +}], +``` + +!!! Note + `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the result was negative. + +### Full example of Protections + +The below example stops trading if more than 4 stoploss occur within a 1 hour (60 minute) limit. + +```json +"protections": [ + { + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 4 + } + ], +``` diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index ff64ca789..5185c93f0 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -14,8 +14,7 @@ logger = logging.getLogger(__name__) class ProtectionManager(): - def __init__(self, exchange, config: dict) -> None: - self._exchange = exchange + def __init__(self, config: dict) -> None: self._config = config self._protection_handlers: List[IProtection] = [] @@ -26,8 +25,6 @@ class ProtectionManager(): continue protection_handler = ProtectionResolver.load_protection( protection_handler_config['method'], - exchange=exchange, - protectionmanager=self, config=config, protection_config=protection_handler_config, ) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index b10856f70..75d1fb3ad 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -1,6 +1,7 @@ import logging from abc import ABC, abstractmethod +from datetime import datetime from typing import Any, Dict @@ -9,8 +10,9 @@ logger = logging.getLogger(__name__) class IProtection(ABC): - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: self._config = config + self._protection_config = protection_config @property def name(self) -> str: @@ -22,3 +24,10 @@ class IProtection(ABC): Short method description - used for startup-messages -> Please overwrite in subclasses """ + + @abstractmethod + def stop_trade_enters_global(self, date_now: datetime) -> bool: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + """ diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py new file mode 100644 index 000000000..3418dd1da --- /dev/null +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -0,0 +1,55 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from sqlalchemy import or_, and_ + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection +from freqtrade.strategy.interface import SellType + + +logger = logging.getLogger(__name__) + + +class StoplossGuard(IProtection): + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + self._lookback_period = protection_config.get('lookback_period', 60) + self._trade_limit = protection_config.get('trade_limit', 10) + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return f"{self.name} - Frequent Stoploss Guard" + + def _stoploss_guard(self, date_now: datetime, pair: str = None) -> bool: + """ + Evaluate recent trades + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + filters = [ + Trade.is_open.is_(False), + Trade.close_date > look_back_until, + or_(Trade.sell_reason == SellType.STOP_LOSS.value, + and_(Trade.sell_reason == SellType.TRAILING_STOP_LOSS.value, + Trade.close_profit < 0)) + ] + if pair: + filters.append(Trade.pair == pair) + trades = Trade.get_trades(filters).all() + + if len(trades) > self.trade_limit: + return True + + return False + + def stop_trade_enters_global(self, date_now: datetime) -> bool: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + """ + return self._stoploss_guard(date_now, pair=None) diff --git a/freqtrade/resolvers/protection_resolver.py b/freqtrade/resolvers/protection_resolver.py index 9a85104c3..928bd4633 100644 --- a/freqtrade/resolvers/protection_resolver.py +++ b/freqtrade/resolvers/protection_resolver.py @@ -24,21 +24,16 @@ class ProtectionResolver(IResolver): initial_search_path = Path(__file__).parent.parent.joinpath('plugins/protections').resolve() @staticmethod - def load_protection(protection_name: str, exchange, protectionmanager, - config: Dict, protection_config: Dict) -> IProtection: + def load_protection(protection_name: str, config: Dict, protection_config: Dict) -> IProtection: """ Load the protection with protection_name :param protection_name: Classname of the pairlist - :param exchange: Initialized exchange class - :param protectionmanager: Initialized protection manager :param config: configuration dictionary :param protection_config: Configuration dedicated to this pairlist :return: initialized Protection class """ return ProtectionResolver.load_object(protection_name, config, - kwargs={'exchange': exchange, - 'pairlistmanager': protectionmanager, - 'config': config, - 'pairlistconfig': protection_config, + kwargs={'config': config, + 'protection_config': protection_config, }, ) From 04878c3ce1806536dcb46d9bbdc1dd38b32f88fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Oct 2020 07:41:41 +0200 Subject: [PATCH 04/73] Rename test directory for pairlist --- tests/{pairlist => plugins}/__init__.py | 0 tests/{pairlist => plugins}/test_pairlist.py | 0 tests/{pairlist => plugins}/test_pairlocks.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/{pairlist => plugins}/__init__.py (100%) rename tests/{pairlist => plugins}/test_pairlist.py (100%) rename tests/{pairlist => plugins}/test_pairlocks.py (100%) diff --git a/tests/pairlist/__init__.py b/tests/plugins/__init__.py similarity index 100% rename from tests/pairlist/__init__.py rename to tests/plugins/__init__.py diff --git a/tests/pairlist/test_pairlist.py b/tests/plugins/test_pairlist.py similarity index 100% rename from tests/pairlist/test_pairlist.py rename to tests/plugins/test_pairlist.py diff --git a/tests/pairlist/test_pairlocks.py b/tests/plugins/test_pairlocks.py similarity index 100% rename from tests/pairlist/test_pairlocks.py rename to tests/plugins/test_pairlocks.py From 246b4a57a40f25750840e515d6f8119c2a5be291 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Oct 2020 19:24:09 +0200 Subject: [PATCH 05/73] add small note to pairlist dev docs --- docs/developer.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/developer.md b/docs/developer.md index c253f4460..662905d65 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -119,6 +119,9 @@ The base-class provides an instance of the exchange (`self._exchange`) the pairl self._pairlist_pos = pairlist_pos ``` +!!! Note + You'll need to register your pairlist in `constants.py` under the variable `AVAILABLE_PAIRLISTS` - otherwise it will not be selectable. + Now, let's step through the methods which require actions: #### Pairlist configuration From f39a534fc039795af1eb45761d998b221e9a1867 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Oct 2020 20:03:56 +0200 Subject: [PATCH 06/73] Implement global stop (First try) --- freqtrade/constants.py | 1 + freqtrade/freqtradebot.py | 5 ++++- freqtrade/plugins/__init__.py | 2 ++ freqtrade/plugins/protectionmanager.py | 20 +++++++++++++------ freqtrade/plugins/protections/iprotection.py | 2 +- .../plugins/protections/stoploss_guard.py | 2 +- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 601e525c1..d070386d0 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -27,6 +27,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] +AVAILABLE_PROTECTIONS = ['StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c8d281852..2dbd7f099 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -23,6 +23,7 @@ from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db +from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -78,6 +79,8 @@ class FreqtradeBot: self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) + self.protections = ProtectionManager(self.config) + # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass @@ -178,7 +181,7 @@ class FreqtradeBot: self.exit_positions(trades) # Then looking for buy opportunities - if self.get_free_open_trades(): + if self.get_free_open_trades() and not self.protections.global_stop(): self.enter_positions() Trade.session.flush() diff --git a/freqtrade/plugins/__init__.py b/freqtrade/plugins/__init__.py index e69de29bb..96943268b 100644 --- a/freqtrade/plugins/__init__.py +++ b/freqtrade/plugins/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +# from freqtrade.plugins.protectionmanager import ProtectionManager diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 5185c93f0..31b0ca300 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -7,7 +7,7 @@ from typing import Dict, List from freqtrade.exceptions import OperationalException from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver - +from datetime import datetime logger = logging.getLogger(__name__) @@ -18,8 +18,7 @@ class ProtectionManager(): self._config = config self._protection_handlers: List[IProtection] = [] - self._tickers_needed = False - for protection_handler_config in self._config.get('protections', None): + for protection_handler_config in self._config.get('protections', []): if 'method' not in protection_handler_config: logger.warning(f"No method found in {protection_handler_config}, ignoring.") continue @@ -28,11 +27,10 @@ class ProtectionManager(): config=config, protection_config=protection_handler_config, ) - self._tickers_needed |= protection_handler.needstickers self._protection_handlers.append(protection_handler) if not self._protection_handlers: - raise OperationalException("No protection Handlers defined") + logger.info("No protection Handlers defined.") @property def name_list(self) -> List[str]: @@ -45,4 +43,14 @@ class ProtectionManager(): """ List of short_desc for each Pairlist Handler """ - return [{p.name: p.short_desc()} for p in self._pairlist_handlers] + return [{p.name: p.short_desc()} for p in self._protection_handlers] + + def global_stop(self) -> bool: + now = datetime.utcnow() + for protection_handler in self._protection_handlers: + result = protection_handler.global_stop(now) + + # Early stopping - first positive result stops the application + if result: + return True + return False diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 75d1fb3ad..25bcee923 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -26,7 +26,7 @@ class IProtection(ABC): """ @abstractmethod - def stop_trade_enters_global(self, date_now: datetime) -> bool: + def global_stop(self, date_now: datetime) -> bool: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 3418dd1da..c6cddb01e 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -47,7 +47,7 @@ class StoplossGuard(IProtection): return False - def stop_trade_enters_global(self, date_now: datetime) -> bool: + def global_stop(self, date_now: datetime) -> bool: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". From 816703b8e112d161727367664b7bbfdf2159b5d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Oct 2020 07:38:00 +0200 Subject: [PATCH 07/73] Improve protections work --- freqtrade/constants.py | 11 ++++++++++- freqtrade/plugins/protectionmanager.py | 4 +++- freqtrade/plugins/protections/iprotection.py | 5 ++++- freqtrade/plugins/protections/stoploss_guard.py | 6 ++++-- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d070386d0..9a93bfae3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -193,7 +193,16 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, - 'config': {'type': 'object'} + }, + 'required': ['method'], + } + }, + 'protections': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, }, 'required': ['method'], } diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 31b0ca300..dd6076ec1 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -2,12 +2,13 @@ Protection manager class """ import logging +from datetime import datetime from typing import Dict, List from freqtrade.exceptions import OperationalException from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver -from datetime import datetime + logger = logging.getLogger(__name__) @@ -47,6 +48,7 @@ class ProtectionManager(): def global_stop(self) -> bool: now = datetime.utcnow() + for protection_handler in self._protection_handlers: result = protection_handler.global_stop(now) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 25bcee923..ecb4cad09 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -4,15 +4,18 @@ from abc import ABC, abstractmethod from datetime import datetime from typing import Any, Dict +from freqtrade.mixins import LoggingMixin + logger = logging.getLogger(__name__) -class IProtection(ABC): +class IProtection(LoggingMixin, ABC): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: self._config = config self._protection_config = protection_config + LoggingMixin.__init__(self, logger) @property def name(self) -> str: diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index c6cddb01e..3b0b8c773 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,7 +3,7 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict -from sqlalchemy import or_, and_ +from sqlalchemy import and_, or_ from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection @@ -42,7 +42,9 @@ class StoplossGuard(IProtection): filters.append(Trade.pair == pair) trades = Trade.get_trades(filters).all() - if len(trades) > self.trade_limit: + if len(trades) > self._trade_limit: + self.log_on_refresh(logger.info, f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.") return True return False From 2b85e7eac3b8936d777eb751f15899afd4aa214f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Oct 2020 07:59:32 +0200 Subject: [PATCH 08/73] Add initial tests for StoplossGuard protection --- tests/plugins/test_protections.py | 79 +++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/plugins/test_protections.py diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py new file mode 100644 index 000000000..2bb0886ee --- /dev/null +++ b/tests/plugins/test_protections.py @@ -0,0 +1,79 @@ +from freqtrade.strategy.interface import SellType +from unittest.mock import MagicMock, PropertyMock +import random +import pytest +from datetime import datetime, timedelta + +from freqtrade.constants import AVAILABLE_PROTECTIONS +from freqtrade.persistence import Trade +from tests.conftest import get_patched_freqtradebot, log_has_re + + +def generate_mock_trade(pair: str, fee: float, is_open: bool, + sell_reason: str = SellType.SELL_SIGNAL, + min_ago_open: int = None, min_ago_close: int = None, + ): + open_rate = random.random() + + trade = Trade( + pair=pair, + stake_amount=0.01, + fee_open=fee, + fee_close=fee, + open_date=datetime.utcnow() - timedelta(minutes=min_ago_open or 200), + close_date=datetime.utcnow() - timedelta(minutes=min_ago_close or 30), + open_rate=open_rate, + is_open=is_open, + amount=0.01 / open_rate, + exchange='bittrex', + ) + trade.recalc_open_trade_price() + if not is_open: + trade.close(open_rate * (1 - 0.9)) + trade.sell_reason = sell_reason + return trade + + +@pytest.mark.usefixtures("init_persistence") +def test_stoploss_guard(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 2 + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, + )) + + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + # This trade does not count, as it's closed too long ago + Trade.session.add(generate_mock_trade( + 'BCH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, + )) + + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, + )) + # 3 Trades closed - but the 2nd has been closed too long ago. + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'LTC/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, + )) + + assert freqtrade.protections.global_stop() + assert log_has_re(message, caplog) From 56975db2ed5387e8a14b5a17e290c0d45a0cdba6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Oct 2020 08:07:09 +0200 Subject: [PATCH 09/73] Add more tests --- config_full.json.example | 3 ++- freqtrade/plugins/protectionmanager.py | 13 ++++----- freqtrade/plugins/protections/__init__.py | 2 +- freqtrade/plugins/protections/iprotection.py | 6 +++-- .../plugins/protections/stoploss_guard.py | 27 ++++++++++++++----- tests/plugins/test_protections.py | 24 ++++++++++++++--- 6 files changed, 54 insertions(+), 21 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 96aa82d5f..eb20065ce 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -79,7 +79,8 @@ { "method": "StoplossGuard", "lookback_period": 60, - "trade_limit": 4 + "trade_limit": 4, + "stopduration": 60 } ], "exchange": { diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index dd6076ec1..c4822a323 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -2,10 +2,10 @@ Protection manager class """ import logging -from datetime import datetime +from datetime import datetime, timezone from typing import Dict, List -from freqtrade.exceptions import OperationalException +from freqtrade.persistence import PairLocks from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver @@ -47,12 +47,13 @@ class ProtectionManager(): return [{p.name: p.short_desc()} for p in self._protection_handlers] def global_stop(self) -> bool: - now = datetime.utcnow() + now = datetime.now(timezone.utc) for protection_handler in self._protection_handlers: - result = protection_handler.global_stop(now) + result, until, reason = protection_handler.global_stop(now) - # Early stopping - first positive result stops the application - if result: + # Early stopping - first positive result blocks further trades + if result and until: + PairLocks.lock_pair('*', until, reason) return True return False diff --git a/freqtrade/plugins/protections/__init__.py b/freqtrade/plugins/protections/__init__.py index 5ecae7888..936355052 100644 --- a/freqtrade/plugins/protections/__init__.py +++ b/freqtrade/plugins/protections/__init__.py @@ -1,2 +1,2 @@ # flake8: noqa: F401 -from freqtrade.plugins.protections.iprotection import IProtection +from freqtrade.plugins.protections.iprotection import IProtection, ProtectionReturn diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index ecb4cad09..cadf01184 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -2,13 +2,15 @@ import logging from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, Dict +from typing import Any, Dict, Optional, Tuple from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) +ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]] + class IProtection(LoggingMixin, ABC): @@ -29,7 +31,7 @@ class IProtection(LoggingMixin, ABC): """ @abstractmethod - def global_stop(self, date_now: datetime) -> bool: + def global_stop(self, date_now: datetime) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 3b0b8c773..db3655a38 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -1,12 +1,12 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Tuple from sqlalchemy import and_, or_ from freqtrade.persistence import Trade -from freqtrade.plugins.protections import IProtection +from freqtrade.plugins.protections import IProtection, ProtectionReturn from freqtrade.strategy.interface import SellType @@ -17,16 +17,26 @@ class StoplossGuard(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) + self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 10) + self._stopduration = protection_config.get('stopduration', 60) + + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' + f'locking for {self._stopduration} min.') def short_desc(self) -> str: """ Short method description - used for startup-messages """ - return f"{self.name} - Frequent Stoploss Guard" + return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses " + f"within {self._lookback_period} minutes.") - def _stoploss_guard(self, date_now: datetime, pair: str = None) -> bool: + def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn: """ Evaluate recent trades """ @@ -45,13 +55,16 @@ class StoplossGuard(IProtection): if len(trades) > self._trade_limit: self.log_on_refresh(logger.info, f"Trading stopped due to {self._trade_limit} " f"stoplosses within {self._lookback_period} minutes.") - return True + until = date_now + timedelta(minutes=self._stopduration) + return True, until, self._reason() - return False + return False, None, None - def global_stop(self, date_now: datetime) -> bool: + def global_stop(self, date_now: datetime) -> ProtectionReturn: """ Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until """ return self._stoploss_guard(date_now, pair=None) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 2bb0886ee..d2815338e 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -1,11 +1,10 @@ -from freqtrade.strategy.interface import SellType -from unittest.mock import MagicMock, PropertyMock import random -import pytest from datetime import datetime, timedelta -from freqtrade.constants import AVAILABLE_PROTECTIONS +import pytest + from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has_re @@ -77,3 +76,20 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert freqtrade.protections.global_stop() assert log_has_re(message, caplog) + + +@pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ + ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, + "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " + "2 stoplosses within 60 minutes.'}]", + None + ), +]) +def test_protection_manager_desc(mocker, default_conf, protectionconf, + desc_expected, exception_expected): + + default_conf['protections'] = [protectionconf] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + short_desc = str(freqtrade.protections.short_desc()) + assert short_desc == desc_expected From 05be33ccd4240ccf39a92f122637003cea530854 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Oct 2020 13:55:54 +0200 Subject: [PATCH 10/73] Simplify is_pair_locked --- freqtrade/persistence/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 6027908da..04d5a7695 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -688,7 +688,7 @@ class PairLock(_DECL_BASE): @staticmethod def query_pair_locks(pair: Optional[str], now: datetime) -> Query: """ - Get all locks for this pair + Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty :param now: Datetime object (generated via datetime.now(timezone.utc)). """ From ff7ba23477d819ec3e3e1b91edc87149ee68efbc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Oct 2020 14:46:13 +0200 Subject: [PATCH 11/73] Simplify enter_positions and add global pairlock check --- freqtrade/freqtradebot.py | 10 +++++++--- tests/test_freqtradebot.py | 28 +++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2dbd7f099..75ff07b17 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -180,8 +180,10 @@ class FreqtradeBot: # First process current opened trades (positions) self.exit_positions(trades) + # Evaluate if protections should apply + self.protections.global_stop() # Then looking for buy opportunities - if self.get_free_open_trades() and not self.protections.global_stop(): + if self.get_free_open_trades(): self.enter_positions() Trade.session.flush() @@ -361,6 +363,9 @@ class FreqtradeBot: logger.info("No currency pair in active pair whitelist, " "but checking to sell open trades.") return trades_created + if PairLocks.is_global_lock(): + logger.info("Global pairlock active. Not creating new trades.") + return trades_created # Create entity and execute trade for each pair from whitelist for pair in whitelist: try: @@ -369,8 +374,7 @@ class FreqtradeBot: logger.warning('Unable to create trade for %s: %s', pair, exception) if not trades_created: - logger.debug("Found no buy signals for whitelisted currencies. " - "Trying again...") + logger.debug("Found no buy signals for whitelisted currencies. Trying again...") return trades_created diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6adef510f..94ed06cd9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -15,7 +15,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.persistence import Order, Trade +from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.rpc import RPCMessageType from freqtrade.state import RunMode, State @@ -678,6 +678,32 @@ def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_o assert log_has("Active pair whitelist is empty.", caplog) +@pytest.mark.usefixtures("init_persistence") +def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, fee, + mocker, caplog) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + n = freqtrade.enter_positions() + message = "Global pairlock active. Not creating new trades." + n = freqtrade.enter_positions() + # 0 trades, but it's not because of pairlock. + assert n == 0 + assert not log_has(message, caplog) + + PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') + n = freqtrade.enter_positions() + assert n == 0 + assert log_has(message, caplog) + + def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True From 2a66c33a4e2c6ae4d116321a2fd8b46638f34354 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Oct 2020 16:52:26 +0200 Subject: [PATCH 12/73] Add locks per pair --- config_full.json.example | 4 ++ freqtrade/constants.py | 2 +- freqtrade/plugins/protectionmanager.py | 9 +++ .../plugins/protections/cooldown_period.py | 68 +++++++++++++++++++ freqtrade/plugins/protections/iprotection.py | 9 +++ .../plugins/protections/stoploss_guard.py | 11 ++- 6 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 freqtrade/plugins/protections/cooldown_period.py diff --git a/config_full.json.example b/config_full.json.example index eb20065ce..839f99dbd 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -81,6 +81,10 @@ "lookback_period": 60, "trade_limit": 4, "stopduration": 60 + }, + { + "method": "CooldownPeriod", + "stopduration": 20 } ], "exchange": { diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9a93bfae3..d06047f4c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -27,7 +27,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] -AVAILABLE_PROTECTIONS = ['StoplossGuard'] +AVAILABLE_PROTECTIONS = ['StoplossGuard', 'CooldownPeriod'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index c4822a323..b0929af88 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -57,3 +57,12 @@ class ProtectionManager(): PairLocks.lock_pair('*', until, reason) return True return False + + def stop_per_pair(self, pair) -> bool: + now = datetime.now(timezone.utc) + for protection_handler in self._protection_handlers: + result, until, reason = protection_handler.stop_per_pair(pair, now) + if result and until: + PairLocks.lock_pair(pair, until, reason) + return True + return False diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py new file mode 100644 index 000000000..c6b6685b2 --- /dev/null +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -0,0 +1,68 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class CooldownPeriod(IProtection): + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._stopduration = protection_config.get('stopduration', 60) + + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'Cooldown period for {self._stopduration} min.') + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Cooldown period.") + + def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: + """ + Get last trade for this pair + """ + look_back_until = date_now - timedelta(minutes=self._stopduration) + filters = [ + Trade.is_open.is_(False), + Trade.close_date > look_back_until, + Trade.pair == pair, + ] + trade = Trade.get_trades(filters).first() + if trade: + self.log_on_refresh(logger.info, f"Cooldown for {pair} for {self._stopduration}.") + until = trade.close_date + timedelta(minutes=self._stopduration) + return True, until, self._reason() + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + # Not implemented for cooldown period. + return False, None, None + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._cooldown_period(pair, date_now) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index cadf01184..5dbcf72f6 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -36,3 +36,12 @@ class IProtection(LoggingMixin, ABC): Stops trading (position entering) for all pairs This must evaluate to true for the whole period of the "cooldown period". """ + + @abstractmethod + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index db3655a38..18888b854 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict, Tuple +from typing import Any, Dict from sqlalchemy import and_, or_ @@ -68,3 +68,12 @@ class StoplossGuard(IProtection): If true, all pairs will be locked with until """ return self._stoploss_guard(date_now, pair=None) + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return False, None, None From fe0afb98832e662fbec06decc951143e8e5c113b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Oct 2020 17:09:30 +0200 Subject: [PATCH 13/73] Implement calling of per-pair protection --- freqtrade/freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 75ff07b17..7bfd64c2d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1415,6 +1415,7 @@ class FreqtradeBot: # Updating wallets when order is closed if not trade.is_open: + self.protections.stop_per_pair(trade.pair) self.wallets.update() return False From 8dbef6bbeab0662a5014082f7ab65e2abb63d1ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 24 Oct 2020 20:25:40 +0200 Subject: [PATCH 14/73] Add test for cooldown period --- .../plugins/protections/cooldown_period.py | 8 ++-- tests/plugins/test_protections.py | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index c6b6685b2..24f55419b 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -1,9 +1,8 @@ import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict - from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -28,7 +27,7 @@ class CooldownPeriod(IProtection): """ Short method description - used for startup-messages """ - return (f"{self.name} - Cooldown period.") + return (f"{self.name} - Cooldown period of {self._stopduration} min.") def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: """ @@ -43,7 +42,8 @@ class CooldownPeriod(IProtection): trade = Trade.get_trades(filters).first() if trade: self.log_on_refresh(logger.info, f"Cooldown for {pair} for {self._stopduration}.") - until = trade.close_date + timedelta(minutes=self._stopduration) + until = trade.close_date.replace( + tzinfo=timezone.utc) + timedelta(minutes=self._stopduration) return True, until, self._reason() return False, None, None diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index d2815338e..59ada7c1e 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta import pytest -from freqtrade.persistence import Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has_re @@ -76,6 +76,43 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert freqtrade.protections.global_stop() assert log_has_re(message, caplog) + assert PairLocks.is_global_lock() + + +@pytest.mark.usefixtures("init_persistence") +def test_CooldownPeriod(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "CooldownPeriod", + "stopduration": 60, + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, + )) + + assert not freqtrade.protections.global_stop() + assert freqtrade.protections.stop_per_pair('XRP/BTC') + assert PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=205, min_ago_close=35, + )) + + assert not freqtrade.protections.global_stop() + assert not PairLocks.is_pair_locked('ETH/BTC') + assert freqtrade.protections.stop_per_pair('ETH/BTC') + assert PairLocks.is_pair_locked('ETH/BTC') + assert not PairLocks.is_global_lock() @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ @@ -84,6 +121,10 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): "2 stoplosses within 60 minutes.'}]", None ), + ({"method": "CooldownPeriod", "stopduration": 60}, + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 min.'}]", + None + ), ]) def test_protection_manager_desc(mocker, default_conf, protectionconf, desc_expected, exception_expected): From 9f6c2a583fff165fd56d935cdd93f47aba13cbb2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Nov 2020 07:48:27 +0100 Subject: [PATCH 15/73] Better wording for config options --- freqtrade/constants.py | 5 ++++- .../plugins/protections/cooldown_period.py | 12 ++++++------ .../plugins/protections/stoploss_guard.py | 18 +++++++++--------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d06047f4c..6319d1f62 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -203,8 +203,11 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, + 'stop_duration': {'type': 'number', 'minimum': 0.0}, + 'trade_limit': {'type': 'number', 'integer': 1}, + 'lookback_period': {'type': 'number', 'integer': 1}, }, - 'required': ['method'], + 'required': ['method', 'trade_limit'], } }, 'telegram': { diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 24f55419b..ed618f6d4 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -15,25 +15,25 @@ class CooldownPeriod(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) - self._stopduration = protection_config.get('stopduration', 60) + self._stop_duration = protection_config.get('stop_duration', 60) def _reason(self) -> str: """ LockReason to use """ - return (f'Cooldown period for {self._stopduration} min.') + return (f'Cooldown period for {self._stop_duration} min.') def short_desc(self) -> str: """ Short method description - used for startup-messages """ - return (f"{self.name} - Cooldown period of {self._stopduration} min.") + return (f"{self.name} - Cooldown period of {self._stop_duration} min.") def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: """ Get last trade for this pair """ - look_back_until = date_now - timedelta(minutes=self._stopduration) + look_back_until = date_now - timedelta(minutes=self._stop_duration) filters = [ Trade.is_open.is_(False), Trade.close_date > look_back_until, @@ -41,9 +41,9 @@ class CooldownPeriod(IProtection): ] trade = Trade.get_trades(filters).first() if trade: - self.log_on_refresh(logger.info, f"Cooldown for {pair} for {self._stopduration}.") + self.log_on_refresh(logger.info, f"Cooldown for {pair} for {self._stop_duration}.") until = trade.close_date.replace( - tzinfo=timezone.utc) + timedelta(minutes=self._stopduration) + tzinfo=timezone.utc) + timedelta(minutes=self._stop_duration) return True, until, self._reason() return False, None, None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 18888b854..408492063 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -20,14 +20,7 @@ class StoplossGuard(IProtection): self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 10) - self._stopduration = protection_config.get('stopduration', 60) - - def _reason(self) -> str: - """ - LockReason to use - """ - return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' - f'locking for {self._stopduration} min.') + self._stop_duration = protection_config.get('stop_duration', 60) def short_desc(self) -> str: """ @@ -36,6 +29,13 @@ class StoplossGuard(IProtection): return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses " f"within {self._lookback_period} minutes.") + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' + f'locking for {self._stop_duration} min.') + def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn: """ Evaluate recent trades @@ -55,7 +55,7 @@ class StoplossGuard(IProtection): if len(trades) > self._trade_limit: self.log_on_refresh(logger.info, f"Trading stopped due to {self._trade_limit} " f"stoplosses within {self._lookback_period} minutes.") - until = date_now + timedelta(minutes=self._stopduration) + until = date_now + timedelta(minutes=self._stop_duration) return True, until, self._reason() return False, None, None From 00d4820bc108976b759170efa0127e1e7960b5fc Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Nov 2020 07:49:30 +0100 Subject: [PATCH 16/73] Add low_profit_pairs --- docs/includes/protections.md | 15 ++++ freqtrade/constants.py | 2 +- .../plugins/protections/low_profit_pairs.py | 81 +++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 freqtrade/plugins/protections/low_profit_pairs.py diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 078ba0c2b..aa0ca0f97 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -21,6 +21,21 @@ Protections will protect your strategy from unexpected events and market conditi !!! Note `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the result was negative. +#### Low Profit Pairs + +`LowProfitpairs` uses all trades for a pair within a `lookback_period` (in minutes) to determine the overall profit ratio. +If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). + +```json +"protections": [{ + "method": "LowProfitpairs", + "lookback_period": 60, + "trade_limit": 4, + "stop_duration": 60, + "required_profit": 0.02 +}], +``` + ### Full example of Protections The below example stops trading if more than 4 stoploss occur within a 1 hour (60 minute) limit. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 6319d1f62..812883da0 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -27,7 +27,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] -AVAILABLE_PROTECTIONS = ['StoplossGuard', 'CooldownPeriod'] +AVAILABLE_PROTECTIONS = ['StoplossGuard', 'CooldownPeriod', 'LowProfitpairs'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py new file mode 100644 index 000000000..739642de7 --- /dev/null +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -0,0 +1,81 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class LowProfitpairs(IProtection): + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._lookback_period = protection_config.get('lookback_period', 60) + self._trade_limit = protection_config.get('trade_limit', 1) + self._stop_duration = protection_config.get('stop_duration', 60) + self._required_profit = protection_config.get('required_profit', 0.0) + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Low Profit Protection, locks pairs with " + f"profit < {self._required_profit} within {self._lookback_period} minutes.") + + def _reason(self, profit: float) -> str: + """ + LockReason to use + """ + return (f'{profit} < {self._required_profit} in {self._lookback_period} min, ' + f'locking for {self._stop_duration} min.') + + def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn: + """ + Evaluate recent trades for pair + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + filters = [ + Trade.is_open.is_(False), + Trade.close_date > look_back_until, + ] + if pair: + filters.append(Trade.pair == pair) + trades = Trade.get_trades(filters).all() + if len(trades) < self._trade_limit: + # Not enough trades in the relevant period + return False, None, None + + profit = sum(trade.close_profit for trade in trades) + if profit < self._required_profit: + self.log_on_refresh( + logger.info, + f"Trading for {pair} stopped due to {profit} < {self._required_profit} " + f"within {self._lookback_period} minutes.") + until = date_now + timedelta(minutes=self._stop_duration) + return True, until, self._reason(profit) + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + return False, None, None + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._low_profit(date_now, pair=None) From 1f703dc3419e8a6179363248f6443177b3f86942 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Nov 2020 08:00:10 +0100 Subject: [PATCH 17/73] Improve protection documentation --- docs/includes/protections.md | 62 ++++++++++++++++++++++++++++++++---- freqtrade/constants.py | 2 +- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index aa0ca0f97..8efb02b95 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -8,13 +8,14 @@ Protections will protect your strategy from unexpected events and market conditi #### Stoploss Guard -`StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case it will stop trading until this condition is no longer true. +`StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. ```json "protections": [{ "method": "StoplossGuard", "lookback_period": 60, - "trade_limit": 4 + "trade_limit": 4, + "stop_duration": 60 }], ``` @@ -27,25 +28,72 @@ Protections will protect your strategy from unexpected events and market conditi If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). ```json -"protections": [{ +"protections": [ + { "method": "LowProfitpairs", "lookback_period": 60, "trade_limit": 4, "stop_duration": 60, "required_profit": 0.02 -}], + } +], ``` -### Full example of Protections +#### Cooldown Period + +`CooldownPeriod` locks a pair for `stop_duration` (in minutes) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. + -The below example stops trading if more than 4 stoploss occur within a 1 hour (60 minute) limit. ```json "protections": [ + { + "method": "CooldownPeriod", + "stop_duration": 60 + } +], +``` + +!!! Note: + This Protection applies only at pair-level, and will never lock all pairs globally. + +### Full example of Protections + +All protections can be combined at will, also with different parameters, creating a increasing wall for under-performing pairs. +All protections are evaluated in the sequence they are defined. + +The below example: + +* stops trading if more than 4 stoploss occur for all pairs within a 1 hour (60 minute) limit (`StoplossGuard`). +* Locks each pair after selling for an additional 10 minutes (`CooldownPeriod`), giving other pairs a chance to get filled. +* Locks all pairs that had 4 Trades within the last 6 hours with a combined profit ratio of below 0.02 (<2%). (`LowProfitpairs`) +* Locks all pairs for 120 minutes that had a profit of below 0.01 (<1%) within the last 24h (`60 * 24 = 1440`), a minimum of 7 trades + +```json +"protections": [ + { + "method": "CooldownPeriod", + "stop_duration": 10 + }, { "method": "StoplossGuard", "lookback_period": 60, - "trade_limit": 4 + "trade_limit": 4, + "stop_duration": 60 + }, + { + "method": "LowProfitpairs", + "lookback_period": 360, + "trade_limit": 4, + "stop_duration": 60, + "required_profit": 0.02 + }, + { + "method": "LowProfitpairs", + "lookback_period": 1440, + "trade_limit": 7, + "stop_duration": 120, + "required_profit": 0.01 } ], ``` diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 812883da0..3f6b6f440 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -207,7 +207,7 @@ CONF_SCHEMA = { 'trade_limit': {'type': 'number', 'integer': 1}, 'lookback_period': {'type': 'number', 'integer': 1}, }, - 'required': ['method', 'trade_limit'], + 'required': ['method'], } }, 'telegram': { From bb06365c503c6c9a9cf5c90f994b588d4568431f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Nov 2020 20:54:24 +0100 Subject: [PATCH 18/73] Improve protection documentation --- docs/includes/protections.md | 29 ++++++++++++------- freqtrade/constants.py | 2 +- .../plugins/protections/low_profit_pairs.py | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 8efb02b95..91b10cf65 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -2,35 +2,46 @@ Protections will protect your strategy from unexpected events and market conditions. +!!! Note + Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy. + +!!! Tip + Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). + ### Available Protection Handlers -* [`StoplossGuard`](#stoploss-guard) (default, if not configured differently) +* [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. +* [`LowProfitPairs`](#low-profit-pairs) Lock pairs with low profits +* [`CooldownPeriod`](#cooldown-period) Don't enter a trade right after selling a trade. #### Stoploss Guard `StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. ```json -"protections": [{ +"protections": [ + { "method": "StoplossGuard", "lookback_period": 60, "trade_limit": 4, "stop_duration": 60 -}], + } +], ``` !!! Note `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the result was negative. + `trade_limit` and `lookback_period` will need to be tuned for your strategy. #### Low Profit Pairs -`LowProfitpairs` uses all trades for a pair within a `lookback_period` (in minutes) to determine the overall profit ratio. +`LowProfitPairs` uses all trades for a pair within a `lookback_period` (in minutes) to determine the overall profit ratio. If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). ```json "protections": [ { - "method": "LowProfitpairs", + "method": "LowProfitPairs", "lookback_period": 60, "trade_limit": 4, "stop_duration": 60, @@ -43,8 +54,6 @@ If that ratio is below `required_profit`, that pair will be locked for `stop_dur `CooldownPeriod` locks a pair for `stop_duration` (in minutes) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. - - ```json "protections": [ { @@ -66,7 +75,7 @@ The below example: * stops trading if more than 4 stoploss occur for all pairs within a 1 hour (60 minute) limit (`StoplossGuard`). * Locks each pair after selling for an additional 10 minutes (`CooldownPeriod`), giving other pairs a chance to get filled. -* Locks all pairs that had 4 Trades within the last 6 hours with a combined profit ratio of below 0.02 (<2%). (`LowProfitpairs`) +* Locks all pairs that had 4 Trades within the last 6 hours with a combined profit ratio of below 0.02 (<2%). (`LowProfitPairs`) * Locks all pairs for 120 minutes that had a profit of below 0.01 (<1%) within the last 24h (`60 * 24 = 1440`), a minimum of 7 trades ```json @@ -82,14 +91,14 @@ The below example: "stop_duration": 60 }, { - "method": "LowProfitpairs", + "method": "LowProfitPairs", "lookback_period": 360, "trade_limit": 4, "stop_duration": 60, "required_profit": 0.02 }, { - "method": "LowProfitpairs", + "method": "LowProfitPairs", "lookback_period": 1440, "trade_limit": 7, "stop_duration": 120, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 3f6b6f440..bc8acc8b3 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -27,7 +27,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] -AVAILABLE_PROTECTIONS = ['StoplossGuard', 'CooldownPeriod', 'LowProfitpairs'] +AVAILABLE_PROTECTIONS = ['StoplossGuard', 'CooldownPeriod', 'LowProfitPairs'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 739642de7..cbc0052ef 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -11,7 +11,7 @@ from freqtrade.plugins.protections import IProtection, ProtectionReturn logger = logging.getLogger(__name__) -class LowProfitpairs(IProtection): +class LowProfitPairs(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) From 9484ee6690ae15bd8a7e769042db1337d8e2d710 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 11 Nov 2020 21:11:22 +0100 Subject: [PATCH 19/73] Test for low_profit_pairs --- .../plugins/protections/low_profit_pairs.py | 2 +- tests/plugins/test_protections.py | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index cbc0052ef..dc5e1ba24 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -78,4 +78,4 @@ class LowProfitPairs(IProtection): :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return self._low_profit(date_now, pair=None) + return self._low_profit(date_now, pair=pair) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 59ada7c1e..3417b1a56 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -115,6 +115,56 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() +@pytest.mark.usefixtures("init_persistence") +def test_LowProfitPairs(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "LowProfitPairs", + "lookback_period": 400, + "stopduration": 60, + "trade_limit": 2, + "required_profit": 0.0, + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=800, min_ago_close=450, + )) + + # Not locked with 1 trade + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=120, + )) + + # Not locked with 1 trade (first trade is outside of lookback_period) + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=110, min_ago_close=20, + )) + + # Locks due to 2nd trade + assert not freqtrade.protections.global_stop() + assert freqtrade.protections.stop_per_pair('XRP/BTC') + assert PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " @@ -125,6 +175,11 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 min.'}]", None ), + ({"method": "LowProfitPairs", "stopduration": 60}, + "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " + "profit < 0.0 within 60 minutes.'}]", + None + ), ]) def test_protection_manager_desc(mocker, default_conf, protectionconf, desc_expected, exception_expected): From 5133675988e3f8e609d0828606af1910f7e264f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Nov 2020 11:41:48 +0100 Subject: [PATCH 20/73] Apply all stops in the list, even if the first would apply already --- freqtrade/plugins/protectionmanager.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index b0929af88..64c7208ce 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -48,21 +48,22 @@ class ProtectionManager(): def global_stop(self) -> bool: now = datetime.now(timezone.utc) - + result = False for protection_handler in self._protection_handlers: result, until, reason = protection_handler.global_stop(now) # Early stopping - first positive result blocks further trades if result and until: PairLocks.lock_pair('*', until, reason) - return True - return False + result = True + return result def stop_per_pair(self, pair) -> bool: now = datetime.now(timezone.utc) + result = False for protection_handler in self._protection_handlers: result, until, reason = protection_handler.stop_per_pair(pair, now) if result and until: PairLocks.lock_pair(pair, until, reason) - return True - return False + result = True + return result From 47cd856fea54fbe12f46d25494430d7cf432b2f2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 15 Nov 2020 16:18:45 +0100 Subject: [PATCH 21/73] Include protection documentation --- docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 2e8f6555f..b70a85c04 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,6 +91,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information below](#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts +| `protections` | Define one or more protections to be used. [More information below](#protections).
**Datatype:** List of Dicts | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -575,6 +576,7 @@ Assuming both buy and sell are using market orders, a configuration similar to t Obviously, if only one side is using limit orders, different pricing combinations can be used. --8<-- "includes/pairlists.md" +--8<-- "includes/protections.md" ## Switch to Dry-run mode From 59091ef2b774d5e9cc481e0047dbd8db34967156 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 17 Nov 2020 19:43:12 +0100 Subject: [PATCH 22/73] Add helper method to calculate protection until --- freqtrade/freqtradebot.py | 3 +++ freqtrade/plugins/protectionmanager.py | 3 ++- .../plugins/protections/cooldown_period.py | 4 ++-- freqtrade/plugins/protections/iprotection.py | 19 +++++++++++++++++-- .../plugins/protections/low_profit_pairs.py | 4 ++-- .../plugins/protections/stoploss_guard.py | 2 +- 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7bfd64c2d..f2ee4d7f0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -182,6 +182,7 @@ class FreqtradeBot: # Evaluate if protections should apply self.protections.global_stop() + # Then looking for buy opportunities if self.get_free_open_trades(): self.enter_positions() @@ -1416,6 +1417,8 @@ class FreqtradeBot: # Updating wallets when order is closed if not trade.is_open: self.protections.stop_per_pair(trade.pair) + # Evaluate if protections should apply + # self.protections.global_stop() self.wallets.update() return False diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 64c7208ce..a79447f02 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -54,7 +54,8 @@ class ProtectionManager(): # Early stopping - first positive result blocks further trades if result and until: - PairLocks.lock_pair('*', until, reason) + if not PairLocks.is_global_lock(until): + PairLocks.lock_pair('*', until, reason) result = True return result diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index ed618f6d4..56635984b 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -42,8 +42,8 @@ class CooldownPeriod(IProtection): trade = Trade.get_trades(filters).first() if trade: self.log_on_refresh(logger.info, f"Cooldown for {pair} for {self._stop_duration}.") - until = trade.close_date.replace( - tzinfo=timezone.utc) + timedelta(minutes=self._stop_duration) + until = self.calculate_lock_end([trade], self._stop_duration) + return True, until, self._reason() return False, None, None diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 5dbcf72f6..8048fccf0 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -1,10 +1,11 @@ import logging from abc import ABC, abstractmethod -from datetime import datetime -from typing import Any, Dict, Optional, Tuple +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple from freqtrade.mixins import LoggingMixin +from freqtrade.persistence import Trade logger = logging.getLogger(__name__) @@ -45,3 +46,17 @@ class IProtection(LoggingMixin, ABC): :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ + + @staticmethod + def calculate_lock_end(trades: List[Trade], stop_minutes: int) -> datetime: + """ + Get lock end time + """ + max_date: datetime = max([trade.close_date for trade in trades]) + # comming from Database, tzinfo is not set. + if max_date.tzinfo is None: + max_date = max_date.replace(tzinfo=timezone.utc) + + until = max_date + timedelta(minutes=stop_minutes) + + return until diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index dc5e1ba24..38d0886bb 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -3,7 +3,6 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict - from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -57,7 +56,8 @@ class LowProfitPairs(IProtection): logger.info, f"Trading for {pair} stopped due to {profit} < {self._required_profit} " f"within {self._lookback_period} minutes.") - until = date_now + timedelta(minutes=self._stop_duration) + until = self.calculate_lock_end(trades, self._stop_duration) + return True, until, self._reason(profit) return False, None, None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 408492063..6335172f8 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -55,7 +55,7 @@ class StoplossGuard(IProtection): if len(trades) > self._trade_limit: self.log_on_refresh(logger.info, f"Trading stopped due to {self._trade_limit} " f"stoplosses within {self._lookback_period} minutes.") - until = date_now + timedelta(minutes=self._stop_duration) + until = self.calculate_lock_end(trades, self._stop_duration) return True, until, self._reason() return False, None, None From fc97266dd47011aa49c85d35bb7f194711fd57d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Nov 2020 07:21:59 +0100 Subject: [PATCH 23/73] Add "now" to lock_pair method --- freqtrade/persistence/pairlock_middleware.py | 13 +++++++++++-- freqtrade/plugins/protectionmanager.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 44fc228f6..38b5a5d63 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -22,10 +22,19 @@ class PairLocks(): timeframe: str = '' @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None) -> None: + def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + """ + Create PairLock from now to "until". + Uses database by default, unless PairLocks.use_db is set to False, + in which case a list is maintained. + :param pair: pair to lock. use '*' to lock all pairs + :param until: End time of the lock. Will be rounded up to the next candle. + :param reason: Reason string that will be shown as reason for the lock + :param now: Current timestamp. Used to determine lock start time. + """ lock = PairLock( pair=pair, - lock_time=datetime.now(timezone.utc), + lock_time=now or datetime.now(timezone.utc), lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until), reason=reason, active=True diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index a79447f02..33a51970c 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -55,7 +55,7 @@ class ProtectionManager(): # Early stopping - first positive result blocks further trades if result and until: if not PairLocks.is_global_lock(until): - PairLocks.lock_pair('*', until, reason) + PairLocks.lock_pair('*', until, reason, now=now) result = True return result From e29d918ea54af338131b518cbe8ffad012c506a6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Nov 2020 08:01:12 +0100 Subject: [PATCH 24/73] Avoid double-locks also in per pair locks --- freqtrade/plugins/protectionmanager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 33a51970c..e58a50c80 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -65,6 +65,7 @@ class ProtectionManager(): for protection_handler in self._protection_handlers: result, until, reason = protection_handler.stop_per_pair(pair, now) if result and until: - PairLocks.lock_pair(pair, until, reason) + if not PairLocks.is_pair_locked(pair, until): + PairLocks.lock_pair(pair, until, reason, now=now) result = True return result From 2e5b9fd4b27bfbbb4c027f8e2e28d88e5677a9b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 18 Nov 2020 08:04:19 +0100 Subject: [PATCH 25/73] format profit in low_profit_pairs --- freqtrade/plugins/protections/low_profit_pairs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 38d0886bb..cc827529f 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -54,7 +54,7 @@ class LowProfitPairs(IProtection): if profit < self._required_profit: self.log_on_refresh( logger.info, - f"Trading for {pair} stopped due to {profit} < {self._required_profit} " + f"Trading for {pair} stopped due to {profit:.2f} < {self._required_profit} " f"within {self._lookback_period} minutes.") until = self.calculate_lock_end(trades, self._stop_duration) From 8ebd6ad2003ca29ba5dac192ce0258cef9e6894d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Nov 2020 19:45:22 +0100 Subject: [PATCH 26/73] Rename login-mixin log method --- freqtrade/mixins/logging_mixin.py | 6 +++--- freqtrade/pairlist/AgeFilter.py | 8 ++++---- freqtrade/pairlist/PrecisionFilter.py | 5 ++--- freqtrade/pairlist/PriceFilter.py | 18 +++++++++--------- freqtrade/pairlist/SpreadFilter.py | 6 +++--- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/pairlist/rangestabilityfilter.py | 9 ++++----- .../plugins/protections/cooldown_period.py | 2 +- .../plugins/protections/low_profit_pairs.py | 2 +- .../plugins/protections/stoploss_guard.py | 4 ++-- tests/plugins/test_pairlist.py | 8 ++++---- 11 files changed, 34 insertions(+), 36 deletions(-) diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py index 4e19e45a4..db2307ad3 100644 --- a/freqtrade/mixins/logging_mixin.py +++ b/freqtrade/mixins/logging_mixin.py @@ -16,7 +16,7 @@ class LoggingMixin(): self.refresh_period = refresh_period self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) - def log_on_refresh(self, logmethod, message: str) -> None: + def log_once(self, logmethod, message: str) -> None: """ Logs message - not more often than "refresh_period" to avoid log spamming Logs the log-message as debug as well to simplify debugging. @@ -25,10 +25,10 @@ class LoggingMixin(): :return: None. """ @cached(cache=self._log_cache) - def _log_on_refresh(message: str): + def _log_once(message: str): logmethod(message) # Log as debug first self.logger.debug(message) # Call hidden function. - _log_on_refresh(message) + _log_once(message) diff --git a/freqtrade/pairlist/AgeFilter.py b/freqtrade/pairlist/AgeFilter.py index e2a13c20a..dd63c1147 100644 --- a/freqtrade/pairlist/AgeFilter.py +++ b/freqtrade/pairlist/AgeFilter.py @@ -76,9 +76,9 @@ class AgeFilter(IPairList): self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000 return True else: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because age {len(daily_candles)} is less than " - f"{self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}") + self.log_once(logger.info, + f"Removed {ticker['symbol']} from whitelist, because age " + f"{len(daily_candles)} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}") return False return False diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index 29e32fd44..a28d54205 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -59,9 +59,8 @@ class PrecisionFilter(IPairList): logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: - self.log_on_refresh(logger.info, - f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") + self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, because " + f"stop price {sp} would be <= stop limit {stop_gap_price}") return False return True diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py index bef1c0a15..a5d73b728 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -64,9 +64,9 @@ class PriceFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ if ticker['last'] is None or ticker['last'] == 0: - self.log_on_refresh(logger.info, - f"Removed {ticker['symbol']} from whitelist, because " - "ticker['last'] is empty (Usually no trade in the last 24h).") + self.log_once(logger.info, + f"Removed {ticker['symbol']} from whitelist, because " + "ticker['last'] is empty (Usually no trade in the last 24h).") return False # Perform low_price_ratio check. @@ -74,22 +74,22 @@ class PriceFilter(IPairList): compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last']) changeperc = compare / ticker['last'] if changeperc > self._low_price_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") + self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") return False # Perform min_price check. if self._min_price != 0: if ticker['last'] < self._min_price: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because last price < {self._min_price:.8f}") + self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because last price < {self._min_price:.8f}") return False # Perform max_price check. if self._max_price != 0: if ticker['last'] > self._max_price: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because last price > {self._max_price:.8f}") + self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because last price > {self._max_price:.8f}") return False return True diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py index a636b90bd..963ecb82a 100644 --- a/freqtrade/pairlist/SpreadFilter.py +++ b/freqtrade/pairlist/SpreadFilter.py @@ -45,9 +45,9 @@ class SpreadFilter(IPairList): if 'bid' in ticker and 'ask' in ticker: spread = 1 - ticker['bid'] / ticker['ask'] if spread > self._max_spread_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because spread {spread * 100:.3f}% >" - f"{self._max_spread_ratio * 100}%") + self.log_once(logger.info, + f"Removed {ticker['symbol']} from whitelist, because spread " + f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%") return False else: return True diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 7d3c2c653..24e1674fd 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -111,6 +111,6 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] - self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + self.log_once(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") return pairs diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py index b460ff477..7a1b69a1a 100644 --- a/freqtrade/pairlist/rangestabilityfilter.py +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -78,11 +78,10 @@ class RangeStabilityFilter(IPairList): if pct_change >= self._min_rate_of_change: result = True else: - self.log_on_refresh(logger.info, - f"Removed {pair} from whitelist, " - f"because rate of change over {plural(self._days, 'day')} is " - f"{pct_change:.3f}, which is below the " - f"threshold of {self._min_rate_of_change}.") + self.log_once(logger.info, + f"Removed {pair} from whitelist, because rate of change " + f"over {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is below the threshold of {self._min_rate_of_change}.") result = False self._pair_cache[pair] = result diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 56635984b..447ca4363 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -41,7 +41,7 @@ class CooldownPeriod(IProtection): ] trade = Trade.get_trades(filters).first() if trade: - self.log_on_refresh(logger.info, f"Cooldown for {pair} for {self._stop_duration}.") + self.log_once(logger.info, f"Cooldown for {pair} for {self._stop_duration}.") until = self.calculate_lock_end([trade], self._stop_duration) return True, until, self._reason() diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index cc827529f..96fb2b08e 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -52,7 +52,7 @@ class LowProfitPairs(IProtection): profit = sum(trade.close_profit for trade in trades) if profit < self._required_profit: - self.log_on_refresh( + self.log_once( logger.info, f"Trading for {pair} stopped due to {profit:.2f} < {self._required_profit} " f"within {self._lookback_period} minutes.") diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 6335172f8..8b6871915 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -53,8 +53,8 @@ class StoplossGuard(IProtection): trades = Trade.get_trades(filters).all() if len(trades) > self._trade_limit: - self.log_on_refresh(logger.info, f"Trading stopped due to {self._trade_limit} " - f"stoplosses within {self._lookback_period} minutes.") + self.log_once(logger.info, f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.") until = self.calculate_lock_end(trades, self._stop_duration) return True, until, self._reason() diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 1d2f16b45..2f1617f6c 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -92,7 +92,7 @@ def static_pl_conf(whitelist_conf): return whitelist_conf -def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): +def test_log_cached(mocker, static_pl_conf, markets, tickers): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), @@ -102,14 +102,14 @@ def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): logmock = MagicMock() # Assign starting whitelist pl = freqtrade.pairlists._pairlist_handlers[0] - pl.log_on_refresh(logmock, 'Hello world') + pl.log_once(logmock, 'Hello world') assert logmock.call_count == 1 - pl.log_on_refresh(logmock, 'Hello world') + pl.log_once(logmock, 'Hello world') assert logmock.call_count == 1 assert pl._log_cache.currsize == 1 assert ('Hello world',) in pl._log_cache._Cache__data - pl.log_on_refresh(logmock, 'Hello world2') + pl.log_once(logmock, 'Hello world2') assert logmock.call_count == 2 assert pl._log_cache.currsize == 2 From 2cd54a59333feafbf64a664fe151a199e91a8ce4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Nov 2020 20:06:13 +0100 Subject: [PATCH 27/73] Allow disabling output from plugins --- freqtrade/mixins/logging_mixin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py index db2307ad3..a8dec2da7 100644 --- a/freqtrade/mixins/logging_mixin.py +++ b/freqtrade/mixins/logging_mixin.py @@ -8,6 +8,9 @@ class LoggingMixin(): Logging Mixin Shows similar messages only once every `refresh_period`. """ + # Disable output completely + show_output = True + def __init__(self, logger, refresh_period: int = 3600): """ :param refresh_period: in seconds - Show identical messages in this intervals @@ -31,4 +34,5 @@ class LoggingMixin(): # Log as debug first self.logger.debug(message) # Call hidden function. - _log_once(message) + if self.show_output: + _log_once(message) From 5e3d2401f5957de2cbea427f671cb4b013c0e1b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Nov 2020 20:34:29 +0100 Subject: [PATCH 28/73] Only call stop methods when they actually support this method --- freqtrade/plugins/protectionmanager.py | 24 ++++++++++--------- .../plugins/protections/cooldown_period.py | 5 ++++ freqtrade/plugins/protections/iprotection.py | 7 +++++- .../plugins/protections/low_profit_pairs.py | 5 ++++ .../plugins/protections/stoploss_guard.py | 9 +++++-- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index e58a50c80..d12f4ba80 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -50,22 +50,24 @@ class ProtectionManager(): now = datetime.now(timezone.utc) result = False for protection_handler in self._protection_handlers: - result, until, reason = protection_handler.global_stop(now) + if protection_handler.has_global_stop: + result, until, reason = protection_handler.global_stop(now) - # Early stopping - first positive result blocks further trades - if result and until: - if not PairLocks.is_global_lock(until): - PairLocks.lock_pair('*', until, reason, now=now) - result = True + # Early stopping - first positive result blocks further trades + if result and until: + if not PairLocks.is_global_lock(until): + PairLocks.lock_pair('*', until, reason, now=now) + result = True return result def stop_per_pair(self, pair) -> bool: now = datetime.now(timezone.utc) result = False for protection_handler in self._protection_handlers: - result, until, reason = protection_handler.stop_per_pair(pair, now) - if result and until: - if not PairLocks.is_pair_locked(pair, until): - PairLocks.lock_pair(pair, until, reason, now=now) - result = True + if protection_handler.has_local_stop: + result, until, reason = protection_handler.stop_per_pair(pair, now) + if result and until: + if not PairLocks.is_pair_locked(pair, until): + PairLocks.lock_pair(pair, until, reason, now=now) + result = True return result diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 447ca4363..4fe0a4fdc 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -12,6 +12,11 @@ logger = logging.getLogger(__name__) class CooldownPeriod(IProtection): + # Can globally stop the bot + has_global_stop: bool = False + # Can stop trading for one pair + has_local_stop: bool = True + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 8048fccf0..49fccb0e6 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -1,6 +1,6 @@ import logging -from abc import ABC, abstractmethod +from abc import ABC, abstractmethod, abstractproperty from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple @@ -15,6 +15,11 @@ ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]] class IProtection(LoggingMixin, ABC): + # Can globally stop the bot + has_global_stop: bool = False + # Can stop trading for one pair + has_local_stop: bool = False + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: self._config = config self._protection_config = protection_config diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 96fb2b08e..48efa3c9a 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -12,6 +12,11 @@ logger = logging.getLogger(__name__) class LowProfitPairs(IProtection): + # Can globally stop the bot + has_global_stop: bool = False + # Can stop trading for one pair + has_local_stop: bool = True + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 8b6871915..51a2fded8 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -15,6 +15,11 @@ logger = logging.getLogger(__name__) class StoplossGuard(IProtection): + # Can globally stop the bot + has_global_stop: bool = True + # Can stop trading for one pair + has_local_stop: bool = True + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) @@ -67,7 +72,7 @@ class StoplossGuard(IProtection): :return: Tuple of [bool, until, reason]. If true, all pairs will be locked with until """ - return self._stoploss_guard(date_now, pair=None) + return self._stoploss_guard(date_now, None) def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: """ @@ -76,4 +81,4 @@ class StoplossGuard(IProtection): :return: Tuple of [bool, until, reason]. If true, this pair will be locked with until """ - return False, None, None + return self._stoploss_guard(date_now, pair) From be57ceb2526a57cfda4ec209fbbf0d8efc358381 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 21 Nov 2020 14:46:27 +0100 Subject: [PATCH 29/73] Remove confusing entry (in this branch of the if statement, candle_date is empty --- freqtrade/strategy/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 81f4e7651..d14d3a35f 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -312,7 +312,7 @@ class IStrategy(ABC): if not candle_date: # Simple call ... - return PairLocks.is_pair_locked(pair, candle_date) + return PairLocks.is_pair_locked(pair) else: lock_time = timeframe_to_next_date(self.timeframe, candle_date) return PairLocks.is_pair_locked(pair, lock_time) From 8d9c66a638af5fa57d11bc5dde8563ebf2ede984 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 11:41:09 +0100 Subject: [PATCH 30/73] Add LogginMixin to freqtradebot class to avoid over-logging --- freqtrade/freqtradebot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f2ee4d7f0..24827a7e3 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,8 +19,9 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import safe_value_fallback, safe_value_fallback2 +from freqtrade.mixins import LoggingMixin from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db from freqtrade.plugins.protectionmanager import ProtectionManager @@ -35,7 +36,7 @@ from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) -class FreqtradeBot: +class FreqtradeBot(LoggingMixin): """ Freqtrade is the main class of the bot. This is from here the bot start its logic. @@ -104,6 +105,7 @@ class FreqtradeBot: self.rpc: RPCManager = RPCManager(self) # Protect sell-logic from forcesell and viceversa self._sell_lock = Lock() + LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) def notify_status(self, msg: str) -> None: """ @@ -365,7 +367,7 @@ class FreqtradeBot: "but checking to sell open trades.") return trades_created if PairLocks.is_global_lock(): - logger.info("Global pairlock active. Not creating new trades.") + self.log_once(logger.info, "Global pairlock active. Not creating new trades.") return trades_created # Create entity and execute trade for each pair from whitelist for pair in whitelist: @@ -551,7 +553,7 @@ class FreqtradeBot: analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) if self.strategy.is_pair_locked( pair, analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None): - logger.info(f"Pair {pair} is currently locked.") + self.log_once(logger.info, f"Pair {pair} is currently locked.") return False # get_free_open_trades is checked before create_trade is called From 8f958ef7238dcd1fa3046ce7307873d87ad85a54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 22 Nov 2020 11:49:41 +0100 Subject: [PATCH 31/73] Improve login-mixin structure --- freqtrade/freqtradebot.py | 4 ++-- freqtrade/mixins/logging_mixin.py | 5 +++-- freqtrade/pairlist/AgeFilter.py | 5 ++--- freqtrade/pairlist/PrecisionFilter.py | 4 ++-- freqtrade/pairlist/PriceFilter.py | 18 +++++++++--------- freqtrade/pairlist/SpreadFilter.py | 6 +++--- freqtrade/pairlist/VolumePairList.py | 2 +- freqtrade/pairlist/rangestabilityfilter.py | 6 +++--- .../plugins/protections/cooldown_period.py | 4 ++-- freqtrade/plugins/protections/iprotection.py | 2 +- .../plugins/protections/low_profit_pairs.py | 3 +-- .../plugins/protections/stoploss_guard.py | 4 ++-- tests/plugins/test_pairlist.py | 6 +++--- tests/plugins/test_protections.py | 1 + 14 files changed, 35 insertions(+), 35 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 24827a7e3..265a8ce10 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -367,7 +367,7 @@ class FreqtradeBot(LoggingMixin): "but checking to sell open trades.") return trades_created if PairLocks.is_global_lock(): - self.log_once(logger.info, "Global pairlock active. Not creating new trades.") + self.log_once("Global pairlock active. Not creating new trades.", logger.info) return trades_created # Create entity and execute trade for each pair from whitelist for pair in whitelist: @@ -553,7 +553,7 @@ class FreqtradeBot(LoggingMixin): analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) if self.strategy.is_pair_locked( pair, analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None): - self.log_once(logger.info, f"Pair {pair} is currently locked.") + self.log_once(f"Pair {pair} is currently locked.", logger.info) return False # get_free_open_trades is checked before create_trade is called diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py index a8dec2da7..e9921e1ec 100644 --- a/freqtrade/mixins/logging_mixin.py +++ b/freqtrade/mixins/logging_mixin.py @@ -1,5 +1,6 @@ +from typing import Callable from cachetools import TTLCache, cached @@ -19,12 +20,12 @@ class LoggingMixin(): self.refresh_period = refresh_period self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) - def log_once(self, logmethod, message: str) -> None: + def log_once(self, message: str, logmethod: Callable) -> None: """ Logs message - not more often than "refresh_period" to avoid log spamming Logs the log-message as debug as well to simplify debugging. - :param logmethod: Function that'll be called. Most likely `logger.info`. :param message: String containing the message to be sent to the function. + :param logmethod: Function that'll be called. Most likely `logger.info`. :return: None. """ @cached(cache=self._log_cache) diff --git a/freqtrade/pairlist/AgeFilter.py b/freqtrade/pairlist/AgeFilter.py index dd63c1147..ae2132637 100644 --- a/freqtrade/pairlist/AgeFilter.py +++ b/freqtrade/pairlist/AgeFilter.py @@ -76,9 +76,8 @@ class AgeFilter(IPairList): self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000 return True else: - self.log_once(logger.info, - f"Removed {ticker['symbol']} from whitelist, because age " + self.log_once(f"Removed {ticker['symbol']} from whitelist, because age " f"{len(daily_candles)} is less than {self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}") + f"{plural(self._min_days_listed, 'day')}", logger.info) return False return False diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index a28d54205..db05d5883 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -59,8 +59,8 @@ class PrecisionFilter(IPairList): logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: - self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, because " - f"stop price {sp} would be <= stop limit {stop_gap_price}") + self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + f"stop price {sp} would be <= stop limit {stop_gap_price}", logger.info) return False return True diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py index a5d73b728..3686cd138 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -64,9 +64,9 @@ class PriceFilter(IPairList): :return: True if the pair can stay, false if it should be removed """ if ticker['last'] is None or ticker['last'] == 0: - self.log_once(logger.info, - f"Removed {ticker['symbol']} from whitelist, because " - "ticker['last'] is empty (Usually no trade in the last 24h).") + self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + "ticker['last'] is empty (Usually no trade in the last 24h).", + logger.info) return False # Perform low_price_ratio check. @@ -74,22 +74,22 @@ class PriceFilter(IPairList): compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last']) changeperc = compare / ticker['last'] if changeperc > self._low_price_ratio: - self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") + self.log_once(f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%", logger.info) return False # Perform min_price check. if self._min_price != 0: if ticker['last'] < self._min_price: - self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because last price < {self._min_price:.8f}") + self.log_once(f"Removed {ticker['symbol']} from whitelist, " + f"because last price < {self._min_price:.8f}", logger.info) return False # Perform max_price check. if self._max_price != 0: if ticker['last'] > self._max_price: - self.log_once(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because last price > {self._max_price:.8f}") + self.log_once(f"Removed {ticker['symbol']} from whitelist, " + f"because last price > {self._max_price:.8f}", logger.info) return False return True diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py index 963ecb82a..6c4e9f12f 100644 --- a/freqtrade/pairlist/SpreadFilter.py +++ b/freqtrade/pairlist/SpreadFilter.py @@ -45,9 +45,9 @@ class SpreadFilter(IPairList): if 'bid' in ticker and 'ask' in ticker: spread = 1 - ticker['bid'] / ticker['ask'] if spread > self._max_spread_ratio: - self.log_once(logger.info, - f"Removed {ticker['symbol']} from whitelist, because spread " - f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%") + self.log_once(f"Removed {ticker['symbol']} from whitelist, because spread " + f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%", + logger.info) return False else: return True diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 24e1674fd..7056bc59d 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -111,6 +111,6 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] - self.log_once(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info) return pairs diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/pairlist/rangestabilityfilter.py index 7a1b69a1a..756368355 100644 --- a/freqtrade/pairlist/rangestabilityfilter.py +++ b/freqtrade/pairlist/rangestabilityfilter.py @@ -78,10 +78,10 @@ class RangeStabilityFilter(IPairList): if pct_change >= self._min_rate_of_change: result = True else: - self.log_once(logger.info, - f"Removed {pair} from whitelist, because rate of change " + self.log_once(f"Removed {pair} from whitelist, because rate of change " f"over {plural(self._days, 'day')} is {pct_change:.3f}, " - f"which is below the threshold of {self._min_rate_of_change}.") + f"which is below the threshold of {self._min_rate_of_change}.", + logger.info) result = False self._pair_cache[pair] = result diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 4fe0a4fdc..1abec7218 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -1,6 +1,6 @@ import logging -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from typing import Any, Dict from freqtrade.persistence import Trade @@ -46,7 +46,7 @@ class CooldownPeriod(IProtection): ] trade = Trade.get_trades(filters).first() if trade: - self.log_once(logger.info, f"Cooldown for {pair} for {self._stop_duration}.") + self.log_once(f"Cooldown for {pair} for {self._stop_duration}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) return True, until, self._reason() diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 49fccb0e6..0f539bbd3 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -1,6 +1,6 @@ import logging -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 48efa3c9a..c45ba3a39 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -58,9 +58,8 @@ class LowProfitPairs(IProtection): profit = sum(trade.close_profit for trade in trades) if profit < self._required_profit: self.log_once( - logger.info, f"Trading for {pair} stopped due to {profit:.2f} < {self._required_profit} " - f"within {self._lookback_period} minutes.") + f"within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) return True, until, self._reason(profit) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 51a2fded8..0645d366b 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -58,8 +58,8 @@ class StoplossGuard(IProtection): trades = Trade.get_trades(filters).all() if len(trades) > self._trade_limit: - self.log_once(logger.info, f"Trading stopped due to {self._trade_limit} " - f"stoplosses within {self._lookback_period} minutes.") + self.log_once(f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) return True, until, self._reason() diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 2f1617f6c..c2a4a69d7 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -102,14 +102,14 @@ def test_log_cached(mocker, static_pl_conf, markets, tickers): logmock = MagicMock() # Assign starting whitelist pl = freqtrade.pairlists._pairlist_handlers[0] - pl.log_once(logmock, 'Hello world') + pl.log_once('Hello world', logmock) assert logmock.call_count == 1 - pl.log_once(logmock, 'Hello world') + pl.log_once('Hello world', logmock) assert logmock.call_count == 1 assert pl._log_cache.currsize == 1 assert ('Hello world',) in pl._log_cache._Cache__data - pl.log_once(logmock, 'Hello world2') + pl.log_once('Hello world2', logmock) assert logmock.call_count == 2 assert pl._log_cache.currsize == 2 diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 3417b1a56..1a22d08a2 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -165,6 +165,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() + @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " From 32cde1cb7da970b3dde7874db35c57984f442409 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Nov 2020 10:48:54 +0100 Subject: [PATCH 32/73] Improve test for lowprofitpairs --- tests/plugins/test_protections.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 1a22d08a2..a02a0366c 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -11,6 +11,7 @@ from tests.conftest import get_patched_freqtradebot, log_has_re def generate_mock_trade(pair: str, fee: float, is_open: bool, sell_reason: str = SellType.SELL_SIGNAL, min_ago_open: int = None, min_ago_close: int = None, + profit_rate: float = 0.9 ): open_rate = random.random() @@ -28,8 +29,9 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, ) trade.recalc_open_trade_price() if not is_open: - trade.close(open_rate * (1 - 0.9)) + trade.close(open_rate * profit_rate) trade.sell_reason = sell_reason + return trade @@ -134,7 +136,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): Trade.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, - min_ago_open=800, min_ago_close=450, + min_ago_open=800, min_ago_close=450, profit_rate=0.9, )) # Not locked with 1 trade @@ -145,7 +147,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): Trade.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, - min_ago_open=200, min_ago_close=120, + min_ago_open=200, min_ago_close=120, profit_rate=0.9, )) # Not locked with 1 trade (first trade is outside of lookback_period) @@ -154,9 +156,17 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() + # Add positive trade + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=20, min_ago_close=10, profit_rate=1.15, + )) + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + Trade.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, - min_ago_open=110, min_ago_close=20, + min_ago_open=110, min_ago_close=20, profit_rate=0.8, )) # Locks due to 2nd trade @@ -166,6 +176,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() + @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " From dcdf4a0503281c02598b2d171c0e5f06f7878e15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Nov 2020 10:58:50 +0100 Subject: [PATCH 33/73] Improve tests --- tests/plugins/test_protections.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index a02a0366c..ce0ad7d5e 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -5,6 +5,7 @@ import pytest from freqtrade.persistence import PairLocks, Trade from freqtrade.strategy.interface import SellType +from freqtrade import constants from tests.conftest import get_patched_freqtradebot, log_has_re @@ -35,6 +36,19 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, return trade +def test_protectionmanager(mocker, default_conf): + default_conf['protections'] = [{'method': protection} + for protection in constants.AVAILABLE_PROTECTIONS] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + for handler in freqtrade.protections._protection_handlers: + assert handler.name in constants.AVAILABLE_PROTECTIONS + if not handler.has_global_stop: + assert handler.global_stop(datetime.utcnow()) == (False, None, None) + if not handler.has_local_stop: + assert handler.local_stop('XRP/BTC', datetime.utcnow()) == (False, None, None) + + @pytest.mark.usefixtures("init_persistence") def test_stoploss_guard(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ @@ -176,7 +190,6 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() - @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " From dce236467224f4d80ce7c6c90927796fdff54722 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Nov 2020 11:11:55 +0100 Subject: [PATCH 34/73] Add stoploss per pair support --- docs/includes/protections.md | 4 +- .../plugins/protections/stoploss_guard.py | 3 + tests/plugins/test_protections.py | 58 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 91b10cf65..644b98e64 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -17,6 +17,7 @@ Protections will protect your strategy from unexpected events and market conditi #### Stoploss Guard `StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. +This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. ```json "protections": [ @@ -24,7 +25,8 @@ Protections will protect your strategy from unexpected events and market conditi "method": "StoplossGuard", "lookback_period": 60, "trade_limit": 4, - "stop_duration": 60 + "stop_duration": 60, + "only_per_pair": false } ], ``` diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 0645d366b..1ad839f3d 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -26,6 +26,7 @@ class StoplossGuard(IProtection): self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 10) self._stop_duration = protection_config.get('stop_duration', 60) + self._disable_global_stop = protection_config.get('only_per_pair', False) def short_desc(self) -> str: """ @@ -72,6 +73,8 @@ class StoplossGuard(IProtection): :return: Tuple of [bool, until, reason]. If true, all pairs will be locked with until """ + if self._disable_global_stop: + return False, None, None return self._stoploss_guard(date_now, None) def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index ce0ad7d5e..7eac737ef 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -95,6 +95,64 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert PairLocks.is_global_lock() +@pytest.mark.parametrize('only_per_pair', [False, True]) +@pytest.mark.usefixtures("init_persistence") +def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair): + default_conf['protections'] = [{ + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 1, + "only_per_pair": only_per_pair + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + pair = 'XRP/BTC' + assert not freqtrade.protections.stop_per_pair(pair) + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, profit_rate=0.9, + )) + + assert not freqtrade.protections.stop_per_pair(pair) + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + # This trade does not count, as it's closed too long ago + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, profit_rate=0.9, + )) + # Trade does not count for per pair stop as it's the wrong pair. + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, profit_rate=0.9, + )) + # 3 Trades closed - but the 2nd has been closed too long ago. + assert not freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + if not only_per_pair: + assert log_has_re(message, caplog) + else: + assert not log_has_re(message, caplog) + + caplog.clear() + + # 2nd Trade that counts with correct pair + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, profit_rate=0.9, + )) + + assert freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + assert PairLocks.is_pair_locked(pair) + assert PairLocks.is_global_lock() != only_per_pair + + @pytest.mark.usefixtures("init_persistence") def test_CooldownPeriod(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ From 6d0f16920f47961f9bae3d3be0e316fbbf368bea Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Nov 2020 11:54:11 +0100 Subject: [PATCH 35/73] Get Longest lock logic --- freqtrade/freqtradebot.py | 20 +++++++++--- freqtrade/persistence/pairlock_middleware.py | 11 ++++++- tests/plugins/test_pairlocks.py | 32 ++++++++++++++++++++ tests/test_freqtradebot.py | 8 ++--- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 265a8ce10..1e0f5fdf0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -367,7 +367,13 @@ class FreqtradeBot(LoggingMixin): "but checking to sell open trades.") return trades_created if PairLocks.is_global_lock(): - self.log_once("Global pairlock active. Not creating new trades.", logger.info) + lock = PairLocks.get_pair_longest_lock('*') + if lock: + self.log_once(f"Global pairlock active until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. " + "Not creating new trades.", logger.info) + else: + self.log_once("Global pairlock active. Not creating new trades.", logger.info) return trades_created # Create entity and execute trade for each pair from whitelist for pair in whitelist: @@ -551,9 +557,15 @@ class FreqtradeBot(LoggingMixin): logger.debug(f"create_trade for pair {pair}") analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) - if self.strategy.is_pair_locked( - pair, analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None): - self.log_once(f"Pair {pair} is currently locked.", logger.info) + nowtime = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None + if self.strategy.is_pair_locked(pair, nowtime): + lock = PairLocks.get_pair_longest_lock(pair, nowtime) + if lock: + self.log_once(f"Pair {pair} is still locked until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.", + logger.info) + else: + self.log_once(f"Pair {pair} is still locked.", logger.info) return False # get_free_open_trades is checked before create_trade is called diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 38b5a5d63..de804f025 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -46,7 +46,7 @@ class PairLocks(): PairLocks.locks.append(lock) @staticmethod - def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: + def get_pair_locks(pair: str, now: Optional[datetime] = None) -> List[PairLock]: """ Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty @@ -66,6 +66,15 @@ class PairLocks(): )] return locks + @staticmethod + def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]: + """ + Get the lock that expires the latest for the pair given. + """ + locks = PairLocks.get_pair_locks(pair, now) + locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True) + return locks[0] if locks else None + @staticmethod def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: """ diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index 0b6b89717..db7d9f46f 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -80,3 +80,35 @@ def test_PairLocks(use_db): assert len(PairLock.query.all()) == 0 # Reset use-db variable PairLocks.use_db = True + + +@pytest.mark.parametrize('use_db', (False, True)) +@pytest.mark.usefixtures("init_persistence") +def test_PairLocks_getlongestlock(use_db): + PairLocks.timeframe = '5m' + # No lock should be present + if use_db: + assert len(PairLock.query.all()) == 0 + else: + PairLocks.use_db = False + + assert PairLocks.use_db == use_db + + pair = 'ETH/BTC' + assert not PairLocks.is_pair_locked(pair) + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + # ETH/BTC locked for 4 minutes + assert PairLocks.is_pair_locked(pair) + lock = PairLocks.get_pair_longest_lock(pair) + + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=3) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) < arrow.utcnow().shift(minutes=14) + + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=15).datetime) + assert PairLocks.is_pair_locked(pair) + + lock = PairLocks.get_pair_longest_lock(pair) + # Must be longer than above + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14) + + PairLocks.use_db = True diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 94ed06cd9..142729f4d 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -692,16 +692,16 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) n = freqtrade.enter_positions() - message = "Global pairlock active. Not creating new trades." + message = r"Global pairlock active until.* Not creating new trades." n = freqtrade.enter_positions() # 0 trades, but it's not because of pairlock. assert n == 0 - assert not log_has(message, caplog) + assert not log_has_re(message, caplog) PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') n = freqtrade.enter_positions() assert n == 0 - assert log_has(message, caplog) + assert log_has_re(message, caplog) def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @@ -3289,7 +3289,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo caplog.clear() freqtrade.enter_positions() - assert log_has(f"Pair {trade.pair} is currently locked.", caplog) + assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, From 12e84bda1e1940333ba8fb649d289c0fd5303a98 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 10:29:45 +0100 Subject: [PATCH 36/73] Add developer docs for Protections --- docs/developer.md | 47 +++++++++++++++++++++++++++++++++++- docs/includes/protections.md | 2 +- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 662905d65..86e9b1078 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -94,7 +94,7 @@ Below is an outline of exception inheritance hierarchy: +---+ StrategyError ``` -## Modules +## Plugins ### Pairlists @@ -173,6 +173,51 @@ In `VolumePairList`, this implements different methods of sorting, does early va return pairs ``` +### Protections + +Best read the [Protection documentation](configuration.md#protections) to understand protections. +This Guide is directed towards Developers who want to develop a new protection. + +No protection should use datetime directly, but use the provided `date_now` variable for date calculations. This preserves the ability to backtest protections. + +!!! Tip "Writing a new Protection" + Best copy one of the existing Protections to have a good example. + +#### Implementation of a new protection + +All Protection implementations must have `IProtection` as parent class. +For that reason, they must implement the following methods: + +* `short_desc()` +* `global_stop()` +* `stop_per_pair()`. + +`global_stop()` and `stop_per_pair()` must return a ProtectionReturn tuple, which consists of: + +* lock pair - boolean +* lock until - datetime - until when should the pair be locked (will be rounded up to the next new candle) +* reason - string, used for logging and storage in the database + +The `until` portion should be calculated using the provided `calculate_lock_end()` method. + +#### Global vs. local stops + +Protections can have 2 different ways to stop trading for a limited : + +* Per pair (local) +* For all Pairs (globally) + +##### Protections - per pair + +Protections that implement the per pair approach must set `has_local_stop=True`. +The method `stop_per_pair()` will be called once, whenever a sell order is closed, and the trade is therefore closed. + +##### Protections - global protection + +These Protections should do their evaluation across all pairs, and consequently will also lock all pairs from trading (called a global PairLock). +Global protection must set `has_global_stop=True` to be evaluated for global stops. +The method `global_stop()` will be called on every iteration, so they should not do too heavy calculations (or should cache the calculations across runs). + ## Implement a new Exchange (WIP) !!! Note diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 644b98e64..aaf5bbff4 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -1,6 +1,6 @@ ## Protections -Protections will protect your strategy from unexpected events and market conditions. +Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. !!! Note Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy. From 4351a26b4cc84abfab5b0fd901c60918d488175e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 10:32:23 +0100 Subject: [PATCH 37/73] Move stop_duration to parent class avoids reimplementation and enhances standardization --- docs/developer.md | 3 +++ freqtrade/plugins/protections/cooldown_period.py | 2 -- freqtrade/plugins/protections/iprotection.py | 2 ++ freqtrade/plugins/protections/low_profit_pairs.py | 1 - freqtrade/plugins/protections/stoploss_guard.py | 1 - 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 86e9b1078..ebfe8e013 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -200,6 +200,9 @@ For that reason, they must implement the following methods: The `until` portion should be calculated using the provided `calculate_lock_end()` method. +All Protections should use `"stop_duration"` to define how long a a pair (or all pairs) should be locked. +The content of this is made available as `self._stop_duration` to the each Protection. + #### Global vs. local stops Protections can have 2 different ways to stop trading for a limited : diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 1abec7218..18a73ef5b 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -20,8 +20,6 @@ class CooldownPeriod(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) - self._stop_duration = protection_config.get('stop_duration', 60) - def _reason(self) -> str: """ LockReason to use diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 0f539bbd3..2053ae741 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -23,6 +23,8 @@ class IProtection(LoggingMixin, ABC): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: self._config = config self._protection_config = protection_config + self._stop_duration = protection_config.get('stop_duration', 60) + LoggingMixin.__init__(self, logger) @property diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index c45ba3a39..cd850ca0c 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -22,7 +22,6 @@ class LowProfitPairs(IProtection): self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 1) - self._stop_duration = protection_config.get('stop_duration', 60) self._required_profit = protection_config.get('required_profit', 0.0) def short_desc(self) -> str: diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 1ad839f3d..65403d683 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -25,7 +25,6 @@ class StoplossGuard(IProtection): self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 10) - self._stop_duration = protection_config.get('stop_duration', 60) self._disable_global_stop = protection_config.get('only_per_pair', False) def short_desc(self) -> str: From 397a15cb617ffb668007632cb6cb9cc3c8717639 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 17:14:28 +0100 Subject: [PATCH 38/73] Improve protection documentation --- docs/includes/protections.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index aaf5bbff4..9722e70aa 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -1,6 +1,7 @@ ## Protections Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. +All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. !!! Note Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy. From ad746627b339d243b4bcd9a15a1a54d4a4cb3051 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 17:47:15 +0100 Subject: [PATCH 39/73] Fix lock-loop --- docs/developer.md | 4 ++-- freqtrade/freqtradebot.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index ebfe8e013..6ea641edd 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -213,13 +213,13 @@ Protections can have 2 different ways to stop trading for a limited : ##### Protections - per pair Protections that implement the per pair approach must set `has_local_stop=True`. -The method `stop_per_pair()` will be called once, whenever a sell order is closed, and the trade is therefore closed. +The method `stop_per_pair()` will be called whenever a trade closed (sell order completed). ##### Protections - global protection These Protections should do their evaluation across all pairs, and consequently will also lock all pairs from trading (called a global PairLock). Global protection must set `has_global_stop=True` to be evaluated for global stops. -The method `global_stop()` will be called on every iteration, so they should not do too heavy calculations (or should cache the calculations across runs). +The method `global_stop()` will be called whenever a trade closed (sell order completed). ## Implement a new Exchange (WIP) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1e0f5fdf0..ecc824a86 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -182,9 +182,6 @@ class FreqtradeBot(LoggingMixin): # First process current opened trades (positions) self.exit_positions(trades) - # Evaluate if protections should apply - self.protections.global_stop() - # Then looking for buy opportunities if self.get_free_open_trades(): self.enter_positions() @@ -1431,8 +1428,7 @@ class FreqtradeBot(LoggingMixin): # Updating wallets when order is closed if not trade.is_open: self.protections.stop_per_pair(trade.pair) - # Evaluate if protections should apply - # self.protections.global_stop() + self.protections.global_stop() self.wallets.update() return False From 9947dcd1da1660efed3d676c4e537f9c5bd0d045 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 17:51:58 +0100 Subject: [PATCH 40/73] Beta feature warning --- docs/includes/protections.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 9722e70aa..716addb55 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -1,5 +1,8 @@ ## Protections +!!! Warning "Beta feature" + This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Issue. + Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. From 768d7fa1966e3694ac97daba57833db8d3c08809 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 18:02:34 +0100 Subject: [PATCH 41/73] Readd optional for get_pair_locks - it's necessary --- freqtrade/persistence/pairlock_middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index de804f025..6ce91ee6b 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -46,7 +46,7 @@ class PairLocks(): PairLocks.locks.append(lock) @staticmethod - def get_pair_locks(pair: str, now: Optional[datetime] = None) -> List[PairLock]: + def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: """ Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty From 9d6f3a89ef26081ecab03c3455c9cfa4063ad856 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Nov 2020 11:36:16 +0100 Subject: [PATCH 42/73] Improve docs and fix typos --- docs/developer.md | 11 +++++++++++ freqtrade/constants.py | 4 ++-- tests/plugins/test_protections.py | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 6ea641edd..05b518184 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -94,6 +94,8 @@ Below is an outline of exception inheritance hierarchy: +---+ StrategyError ``` +--- + ## Plugins ### Pairlists @@ -203,6 +205,8 @@ The `until` portion should be calculated using the provided `calculate_lock_end( All Protections should use `"stop_duration"` to define how long a a pair (or all pairs) should be locked. The content of this is made available as `self._stop_duration` to the each Protection. +If your protection requires a look-back period, please use `"lookback_period"` to keep different protections aligned. + #### Global vs. local stops Protections can have 2 different ways to stop trading for a limited : @@ -221,6 +225,13 @@ These Protections should do their evaluation across all pairs, and consequently Global protection must set `has_global_stop=True` to be evaluated for global stops. The method `global_stop()` will be called whenever a trade closed (sell order completed). +##### Protections - calculating lock end time + +Protections should calculate the lock end time based on the last trade it considers. +This avoids relocking should the lookback-period be longer than the actual lock period. + +--- + ## Implement a new Exchange (WIP) !!! Note diff --git a/freqtrade/constants.py b/freqtrade/constants.py index bc8acc8b3..add9aae95 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -204,8 +204,8 @@ CONF_SCHEMA = { 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, 'stop_duration': {'type': 'number', 'minimum': 0.0}, - 'trade_limit': {'type': 'number', 'integer': 1}, - 'lookback_period': {'type': 'number', 'integer': 1}, + 'trade_limit': {'type': 'number', 'minimum': 1}, + 'lookback_period': {'type': 'number', 'minimum': 1}, }, 'required': ['method'], } diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 7eac737ef..24594c3ac 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -54,6 +54,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, + "stop_duration": 40, "trade_limit": 2 }] freqtrade = get_patched_freqtradebot(mocker, default_conf) From 089c463cfb04404a8ea2592dddaf489ed78615cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Nov 2020 08:05:48 +0100 Subject: [PATCH 43/73] Introduce max_drawdown protection --- freqtrade/constants.py | 2 +- .../plugins/protections/cooldown_period.py | 2 - .../plugins/protections/low_profit_pairs.py | 2 - .../protections/max_drawdown_protection.py | 91 +++++++++++++++++++ .../plugins/protections/stoploss_guard.py | 2 - 5 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 freqtrade/plugins/protections/max_drawdown_protection.py diff --git a/freqtrade/constants.py b/freqtrade/constants.py index add9aae95..dfc21b678 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -27,7 +27,7 @@ AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] -AVAILABLE_PROTECTIONS = ['StoplossGuard', 'CooldownPeriod', 'LowProfitPairs'] +AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 18a73ef5b..7b37b2303 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -12,9 +12,7 @@ logger = logging.getLogger(__name__) class CooldownPeriod(IProtection): - # Can globally stop the bot has_global_stop: bool = False - # Can stop trading for one pair has_local_stop: bool = True def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index cd850ca0c..515f81521 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -12,9 +12,7 @@ logger = logging.getLogger(__name__) class LowProfitPairs(IProtection): - # Can globally stop the bot has_global_stop: bool = False - # Can stop trading for one pair has_local_stop: bool = True def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py new file mode 100644 index 000000000..e8a920908 --- /dev/null +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -0,0 +1,91 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +import pandas as pd + +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class MaxDrawdown(IProtection): + + has_global_stop: bool = True + has_local_stop: bool = False + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._lookback_period = protection_config.get('lookback_period', 60) + self._trade_limit = protection_config.get('trade_limit', 1) + self._max_allowed_drawdown = protection_config.get('max_allowed_drawdown', 0.0) + # TODO: Implement checks to limit max_drawdown to sensible values + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Max drawdown protection, stop trading if drawdown is > " + f"{self._max_allowed_drawdown} within {self._lookback_period} minutes.") + + def _reason(self, drawdown: float) -> str: + """ + LockReason to use + """ + return (f'{drawdown} > {self._max_allowed_drawdown} in {self._lookback_period} min, ' + f'locking for {self._stop_duration} min.') + + def _max_drawdown(self, date_now: datetime, pair: str) -> ProtectionReturn: + """ + Evaluate recent trades for drawdown ... + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + filters = [ + Trade.is_open.is_(False), + Trade.close_date > look_back_until, + ] + if pair: + filters.append(Trade.pair == pair) + trades = Trade.get_trades(filters).all() + + trades_df = pd.DataFrame(trades) + + if len(trades) < self._trade_limit: + # Not enough trades in the relevant period + return False, None, None + + # Drawdown is always positive + drawdown, _, _ = calculate_max_drawdown(trades_df) + + if drawdown > self._max_allowed_drawdown: + self.log_once( + f"Trading for {pair} stopped due to {drawdown:.2f} < {self._max_allowed_drawdown} " + f"within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + + return True, until, self._reason(drawdown) + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + return self._max_drawdown(date_now) + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return False, None, None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 65403d683..b6f430085 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -15,9 +15,7 @@ logger = logging.getLogger(__name__) class StoplossGuard(IProtection): - # Can globally stop the bot has_global_stop: bool = True - # Can stop trading for one pair has_local_stop: bool = True def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: From f06b58dc91d946f4b929531ad4f01ba5754860db Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 30 Nov 2020 19:07:39 +0100 Subject: [PATCH 44/73] Test MaxDrawdown desc --- .../protections/max_drawdown_protection.py | 12 ++-- tests/plugins/test_protections.py | 72 ++++++++++++++++++- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index e8a920908..e5625733c 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -40,7 +40,7 @@ class MaxDrawdown(IProtection): return (f'{drawdown} > {self._max_allowed_drawdown} in {self._lookback_period} min, ' f'locking for {self._stop_duration} min.') - def _max_drawdown(self, date_now: datetime, pair: str) -> ProtectionReturn: + def _max_drawdown(self, date_now: datetime) -> ProtectionReturn: """ Evaluate recent trades for drawdown ... """ @@ -49,23 +49,21 @@ class MaxDrawdown(IProtection): Trade.is_open.is_(False), Trade.close_date > look_back_until, ] - if pair: - filters.append(Trade.pair == pair) trades = Trade.get_trades(filters).all() - trades_df = pd.DataFrame(trades) + trades_df = pd.DataFrame([trade.to_json() for trade in trades]) if len(trades) < self._trade_limit: # Not enough trades in the relevant period return False, None, None # Drawdown is always positive - drawdown, _, _ = calculate_max_drawdown(trades_df) + drawdown, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') if drawdown > self._max_allowed_drawdown: self.log_once( - f"Trading for {pair} stopped due to {drawdown:.2f} < {self._max_allowed_drawdown} " - f"within {self._lookback_period} minutes.", logger.info) + f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}" + f" within {self._lookback_period} minutes.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) return True, until, self._reason(drawdown) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 24594c3ac..e5bbec431 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -46,7 +46,7 @@ def test_protectionmanager(mocker, default_conf): if not handler.has_global_stop: assert handler.global_stop(datetime.utcnow()) == (False, None, None) if not handler.has_local_stop: - assert handler.local_stop('XRP/BTC', datetime.utcnow()) == (False, None, None) + assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None) @pytest.mark.usefixtures("init_persistence") @@ -249,6 +249,71 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() +@pytest.mark.usefixtures("init_persistence") +def test_MaxDrawdown(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "MaxDrawdown", + "lookback_period": 1000, + "stopduration": 60, + "trade_limit": 3, + "max_allowed_drawdown": 0.15 + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to Max.*" + + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=500, min_ago_close=400, profit_rate=0.9, + )) + # Not locked with one trade + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, + )) + + # Not locked with 1 trade (2nd trade is outside of lookback_period) + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + assert not log_has_re(message, caplog) + + # Winning trade ... (should not lock, does not change drawdown!) + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=320, min_ago_close=410, profit_rate=1.5, + )) + assert not freqtrade.protections.global_stop() + assert not PairLocks.is_global_lock() + + caplog.clear() + + # Add additional negative trade, causing a loss of > 15% + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=20, min_ago_close=10, profit_rate=0.8, + )) + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + # local lock not supported + assert not PairLocks.is_pair_locked('XRP/BTC') + assert freqtrade.protections.global_stop() + assert PairLocks.is_global_lock() + assert log_has_re(message, caplog) + + @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " @@ -264,6 +329,11 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): "profit < 0.0 within 60 minutes.'}]", None ), + ({"method": "MaxDrawdown", "stopduration": 60}, + "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " + "within 60 minutes.'}]", + None + ), ]) def test_protection_manager_desc(mocker, default_conf, protectionconf, desc_expected, exception_expected): From b36f333b2fed15bd864526b98a44a9604f96dc38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 1 Dec 2020 06:52:19 +0100 Subject: [PATCH 45/73] Add new protections to full sample, documentation --- config_full.json.example | 14 +++++++++ docs/includes/protections.md | 59 ++++++++++++++++++++++++++++-------- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 839f99dbd..737015b41 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -85,6 +85,20 @@ { "method": "CooldownPeriod", "stopduration": 20 + }, + { + "method": "MaxDrawdown", + "lookback_period": 2000, + "trade_limit": 20, + "stop_duration": 10, + "max_allowed_drawdown": 0.2 + }, + { + "method": "LowProfitPairs", + "lookback_period": 360, + "trade_limit": 1, + "stop_duration": 2, + "required_profit": 0.02 } ], "exchange": { diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 716addb55..526c4d0a3 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -15,6 +15,7 @@ All protection end times are rounded up to the next candle to avoid sudden, unex ### Available Protection Handlers * [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. +* [`MaxDrawdown`](#maxdrawdown) Stop trading if max-drawdown is reached. * [`LowProfitPairs`](#low-profit-pairs) Lock pairs with low profits * [`CooldownPeriod`](#cooldown-period) Don't enter a trade right after selling a trade. @@ -23,33 +24,56 @@ All protection end times are rounded up to the next candle to avoid sudden, unex `StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. +The below example stops trading for all pairs for 2 hours (120min) after the last trade if the bot hit stoploss 4 times within the last 24h. + ```json "protections": [ { "method": "StoplossGuard", - "lookback_period": 60, + "lookback_period": 1440, "trade_limit": 4, - "stop_duration": 60, + "stop_duration": 120, "only_per_pair": false } ], ``` !!! Note - `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the result was negative. + `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the resulting profit was negative. `trade_limit` and `lookback_period` will need to be tuned for your strategy. +#### MaxDrawdown + +`MaxDrawdown` uses all trades within `lookback_period` (in minutes) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes) after the last trade - assuming that the bot needs some time to let markets recover. + +The below sample stops trading for 12 hours (720min) if max-drawdown is > 20% considering all trades within the last 2 days (2880min). + +```json +"protections": [ + { + "method": "MaxDrawdown", + "lookback_period": 2880, + "trade_limit": 20, + "stop_duration": 720, + "max_allowed_drawdown": 0.2 + }, +], + +``` + #### Low Profit Pairs `LowProfitPairs` uses all trades for a pair within a `lookback_period` (in minutes) to determine the overall profit ratio. If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). +The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 hours (360min). + ```json "protections": [ { "method": "LowProfitPairs", - "lookback_period": 60, - "trade_limit": 4, + "lookback_period": 360, + "trade_limit": 2, "stop_duration": 60, "required_profit": 0.02 } @@ -79,10 +103,11 @@ All protections are evaluated in the sequence they are defined. The below example: -* stops trading if more than 4 stoploss occur for all pairs within a 1 hour (60 minute) limit (`StoplossGuard`). * Locks each pair after selling for an additional 10 minutes (`CooldownPeriod`), giving other pairs a chance to get filled. -* Locks all pairs that had 4 Trades within the last 6 hours with a combined profit ratio of below 0.02 (<2%). (`LowProfitPairs`) -* Locks all pairs for 120 minutes that had a profit of below 0.01 (<1%) within the last 24h (`60 * 24 = 1440`), a minimum of 7 trades +* Stops trading if the last 2 days had 20 trades, which caused a max-drawdown of more than 20%. (`MaxDrawdown`). +* Stops trading if more than 4 stoploss occur for all pairs within a 1 day (1440min) limit (`StoplossGuard`). +* Locks all pairs that had 4 Trades within the last 6 hours (`60 * 6 = 360`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). +* Locks all pairs for 120 minutes that had a profit of below 0.01 (<1%) within the last 24h (`60 * 24 = 1440`), a minimum of 4 trades. ```json "protections": [ @@ -90,23 +115,31 @@ The below example: "method": "CooldownPeriod", "stop_duration": 10 }, + { + "method": "MaxDrawdown", + "lookback_period": 2880, + "trade_limit": 20, + "stop_duration": 720, + "max_allowed_drawdown": 0.2 + }, { "method": "StoplossGuard", - "lookback_period": 60, + "lookback_period": 1440, "trade_limit": 4, - "stop_duration": 60 + "stop_duration": 120, + "only_per_pair": false }, { "method": "LowProfitPairs", "lookback_period": 360, - "trade_limit": 4, + "trade_limit": 2, "stop_duration": 60, "required_profit": 0.02 }, - { + { "method": "LowProfitPairs", "lookback_period": 1440, - "trade_limit": 7, + "trade_limit": 4, "stop_duration": 120, "required_profit": 0.01 } From f13e9ce5edb993b17e964d4ae3bae87baae97b68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 08:22:12 +0100 Subject: [PATCH 46/73] Improve docs --- docs/includes/protections.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 526c4d0a3..f5639565f 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -12,16 +12,21 @@ All protection end times are rounded up to the next candle to avoid sudden, unex !!! Tip Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). -### Available Protection Handlers +### Available Protections * [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. * [`MaxDrawdown`](#maxdrawdown) Stop trading if max-drawdown is reached. * [`LowProfitPairs`](#low-profit-pairs) Lock pairs with low profits * [`CooldownPeriod`](#cooldown-period) Don't enter a trade right after selling a trade. +### Common settings to all Protections + +* `stop_duration` (minutes) - how long should protections be locked. +* `lookback_period` (minutes) - Only trades that completed after `current_time - lookback_period` will be considered (may be ignored by some Protections). + #### Stoploss Guard -`StoplossGuard` selects all trades within a `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. +`StoplossGuard` selects all trades within `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. The below example stops trading for all pairs for 2 hours (120min) after the last trade if the bot hit stoploss 4 times within the last 24h. @@ -63,7 +68,7 @@ The below sample stops trading for 12 hours (720min) if max-drawdown is > 20% co #### Low Profit Pairs -`LowProfitPairs` uses all trades for a pair within a `lookback_period` (in minutes) to determine the overall profit ratio. +`LowProfitPairs` uses all trades for a pair within `lookback_period` (in minutes) to determine the overall profit ratio. If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 hours (360min). @@ -93,8 +98,9 @@ The below example will stop trading a pair for 60 minutes if the pair does not h ], ``` -!!! Note: +!!! Note This Protection applies only at pair-level, and will never lock all pairs globally. + This Protection does not consider `lookback_period` as it only looks at the latest trade. ### Full example of Protections From eb952d77be2c9eaff24cab9d42a5111d28fefd13 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 08:27:14 +0100 Subject: [PATCH 47/73] Move lookback_period to parent __init__ --- docs/includes/protections.md | 2 ++ freqtrade/plugins/protections/iprotection.py | 1 + freqtrade/plugins/protections/low_profit_pairs.py | 1 - freqtrade/plugins/protections/max_drawdown_protection.py | 1 - freqtrade/plugins/protections/stoploss_guard.py | 1 - 5 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index f5639565f..25d59a992 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -21,8 +21,10 @@ All protection end times are rounded up to the next candle to avoid sudden, unex ### Common settings to all Protections +* `method` - Protection name to use. * `stop_duration` (minutes) - how long should protections be locked. * `lookback_period` (minutes) - Only trades that completed after `current_time - lookback_period` will be considered (may be ignored by some Protections). +* `trade_limit` - How many trades are required at minimum (not used by all Protections). #### Stoploss Guard diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 2053ae741..60f83eea6 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -24,6 +24,7 @@ class IProtection(LoggingMixin, ABC): self._config = config self._protection_config = protection_config self._stop_duration = protection_config.get('stop_duration', 60) + self._lookback_period = protection_config.get('lookback_period', 60) LoggingMixin.__init__(self, logger) diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 515f81521..70ef5b080 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -18,7 +18,6 @@ class LowProfitPairs(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) - self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 1) self._required_profit = protection_config.get('required_profit', 0.0) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index e5625733c..2a83cdeba 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -21,7 +21,6 @@ class MaxDrawdown(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) - self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 1) self._max_allowed_drawdown = protection_config.get('max_allowed_drawdown', 0.0) # TODO: Implement checks to limit max_drawdown to sensible values diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index b6f430085..520607337 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -21,7 +21,6 @@ class StoplossGuard(IProtection): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: super().__init__(config, protection_config) - self._lookback_period = protection_config.get('lookback_period', 60) self._trade_limit = protection_config.get('trade_limit', 10) self._disable_global_stop = protection_config.get('only_per_pair', False) From a93bb6853bbd6359deaa733adaa0491de5546fd9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 10:21:03 +0100 Subject: [PATCH 48/73] Document *candles settings, implement validations --- docs/includes/protections.md | 12 ++++++++---- freqtrade/configuration/config_validation.py | 20 ++++++++++++++++++++ freqtrade/constants.py | 2 ++ tests/plugins/test_protections.py | 15 ++++++++------- tests/test_configuration.py | 18 ++++++++++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 25d59a992..2f704d83f 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -21,10 +21,14 @@ All protection end times are rounded up to the next candle to avoid sudden, unex ### Common settings to all Protections -* `method` - Protection name to use. -* `stop_duration` (minutes) - how long should protections be locked. -* `lookback_period` (minutes) - Only trades that completed after `current_time - lookback_period` will be considered (may be ignored by some Protections). -* `trade_limit` - How many trades are required at minimum (not used by all Protections). +| Parameter| Description | +|------------|-------------| +| method | Protection name to use.
**Datatype:** String, selected from [available Protections](#available-protections) +| stop_duration_candles | For how many candles should the lock be set?
**Datatype:** Positive integer (in candles) +| stop_duration | how many minutes should protections be locked.
Cannot be used together with `stop_duration_candles`.
**Datatype:** Float (in minutes) +| `lookback_period_candles` | Only trades that completed within the last `lookback_period_candles` candles will be considered. This setting may be ignored by some Protections.
**Datatype:** Positive integer (in candles). +| lookback_period | Only trades that completed after `current_time - lookback_period` will be considered.
Cannot be used together with `lookback_period_candles`.
This setting may be ignored by some Protections.
**Datatype:** Float (in minutes) +| trade_limit | Number of trades required at minimum (not used by all Protections).
**Datatype:** Positive integer #### Stoploss Guard diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index ab21bc686..a6435d0e6 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -74,6 +74,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: _validate_trailing_stoploss(conf) _validate_edge(conf) _validate_whitelist(conf) + _validate_protections(conf) _validate_unlimited_amount(conf) # validate configuration before returning @@ -155,3 +156,22 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if (pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") + + +def _validate_protections(conf: Dict[str, Any]) -> None: + """ + Validate protection configuration validity + """ + + for prot in conf.get('protections', []): + if ('stop_duration' in prot and 'stop_duration_candles' in prot): + raise OperationalException( + "Protections must specify either `stop_duration` or `stop_duration_candles`.\n" + f"Please fix the protection {prot.get('method')}" + ) + + if ('lookback_period' in prot and 'lookback_period_candle' in prot): + raise OperationalException( + "Protections must specify either `lookback_period` or `lookback_period_candles`.\n" + f"Please fix the protection {prot.get('method')}" + ) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index dfc21b678..e7d7e80f6 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -204,8 +204,10 @@ CONF_SCHEMA = { 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, 'stop_duration': {'type': 'number', 'minimum': 0.0}, + 'stop_duration_candles': {'type': 'number', 'minimum': 0}, 'trade_limit': {'type': 'number', 'minimum': 1}, 'lookback_period': {'type': 'number', 'minimum': 1}, + 'lookback_period_candles': {'type': 'number', 'minimum': 1}, }, 'required': ['method'], } diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index e5bbec431..29ff4e069 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -103,6 +103,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair "method": "StoplossGuard", "lookback_period": 60, "trade_limit": 1, + "stop_duration": 60, "only_per_pair": only_per_pair }] freqtrade = get_patched_freqtradebot(mocker, default_conf) @@ -158,7 +159,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair def test_CooldownPeriod(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ "method": "CooldownPeriod", - "stopduration": 60, + "stop_duration": 60, }] freqtrade = get_patched_freqtradebot(mocker, default_conf) message = r"Trading stopped due to .*" @@ -195,7 +196,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ "method": "LowProfitPairs", "lookback_period": 400, - "stopduration": 60, + "stop_duration": 60, "trade_limit": 2, "required_profit": 0.0, }] @@ -254,7 +255,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ "method": "MaxDrawdown", "lookback_period": 1000, - "stopduration": 60, + "stop_duration": 60, "trade_limit": 3, "max_allowed_drawdown": 0.15 }] @@ -315,21 +316,21 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): @pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ - ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2}, + ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}, "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " "2 stoplosses within 60 minutes.'}]", None ), - ({"method": "CooldownPeriod", "stopduration": 60}, + ({"method": "CooldownPeriod", "stop_duration": 60}, "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 min.'}]", None ), - ({"method": "LowProfitPairs", "stopduration": 60}, + ({"method": "LowProfitPairs", "lookback_period": 60, "stop_duration": 60}, "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " "profit < 0.0 within 60 minutes.'}]", None ), - ({"method": "MaxDrawdown", "stopduration": 60}, + ({"method": "MaxDrawdown", "lookback_period": 60, "stop_duration": 60}, "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " "within 60 minutes.'}]", None diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 167215f29..283f6a0f9 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -879,6 +879,24 @@ def test_validate_whitelist(default_conf): validate_config_consistency(conf) +@pytest.mark.parametrize('protconf,expected', [ + ([], None), + ([{"method": "StoplossGuard", "lookback_period": 2000, "stop_duration_candles": 10}], None), + ([{"method": "StoplossGuard", "lookback_period_candle": 20, "stop_duration": 10}], None), + ([{"method": "StoplossGuard", "lookback_period_candle": 20, "lookback_period": 2000, + "stop_duration": 10}], r'Protections must specify either `lookback_period`.*'), + ([{"method": "StoplossGuard", "lookback_period": 20, "stop_duration": 10, + "stop_duration_candles": 10}], r'Protections must specify either `stop_duration`.*'), +]) +def test_validate_protections(default_conf, protconf, expected): + conf = deepcopy(default_conf) + conf['protections'] = protconf + if expected: + with pytest.raises(OperationalException, match=expected): + validate_config_consistency(conf) + else: + validate_config_consistency(conf) + def test_load_config_test_comments() -> None: """ From d4799e6aa3897db275920b48b615e7f6733c32ef Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 10:45:35 +0100 Subject: [PATCH 49/73] Implement *candle definitions --- docs/includes/protections.md | 57 +++++++++++--------- freqtrade/configuration/config_validation.py | 2 +- freqtrade/plugins/protections/iprotection.py | 12 ++++- tests/plugins/test_protections.py | 32 ++++++++++- tests/test_configuration.py | 5 +- 5 files changed, 76 insertions(+), 32 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 2f704d83f..210765176 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -30,20 +30,24 @@ All protection end times are rounded up to the next candle to avoid sudden, unex | lookback_period | Only trades that completed after `current_time - lookback_period` will be considered.
Cannot be used together with `lookback_period_candles`.
This setting may be ignored by some Protections.
**Datatype:** Float (in minutes) | trade_limit | Number of trades required at minimum (not used by all Protections).
**Datatype:** Positive integer +!!! Note "Durations" + Durations (`stop_duration*` and `lookback_period*` can be defined in either minutes or candles). + For more flexibility when testing different timeframes, all below examples will use the "candle" definition. + #### Stoploss Guard -`StoplossGuard` selects all trades within `lookback_period` (in minutes), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. +`StoplossGuard` selects all trades within `lookback_period`, and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. -The below example stops trading for all pairs for 2 hours (120min) after the last trade if the bot hit stoploss 4 times within the last 24h. +The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. ```json "protections": [ { "method": "StoplossGuard", - "lookback_period": 1440, + "lookback_period_candles": 24, "trade_limit": 4, - "stop_duration": 120, + "stop_duration_candles": 4, "only_per_pair": false } ], @@ -57,15 +61,15 @@ The below example stops trading for all pairs for 2 hours (120min) after the las `MaxDrawdown` uses all trades within `lookback_period` (in minutes) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes) after the last trade - assuming that the bot needs some time to let markets recover. -The below sample stops trading for 12 hours (720min) if max-drawdown is > 20% considering all trades within the last 2 days (2880min). +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. ```json "protections": [ { "method": "MaxDrawdown", - "lookback_period": 2880, + "lookback_period_candles": 48, "trade_limit": 20, - "stop_duration": 720, + "stop_duration_candles": 12, "max_allowed_drawdown": 0.2 }, ], @@ -77,13 +81,13 @@ The below sample stops trading for 12 hours (720min) if max-drawdown is > 20% co `LowProfitPairs` uses all trades for a pair within `lookback_period` (in minutes) to determine the overall profit ratio. If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). -The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 hours (360min). +The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. ```json "protections": [ { "method": "LowProfitPairs", - "lookback_period": 360, + "lookback_period_candles": 6, "trade_limit": 2, "stop_duration": 60, "required_profit": 0.02 @@ -95,11 +99,13 @@ The below example will stop trading a pair for 60 minutes if the pair does not h `CooldownPeriod` locks a pair for `stop_duration` (in minutes) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. +The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". + ```json "protections": [ { "method": "CooldownPeriod", - "stop_duration": 60 + "stop_duration_candle": 2 } ], ``` @@ -113,46 +119,47 @@ The below example will stop trading a pair for 60 minutes if the pair does not h All protections can be combined at will, also with different parameters, creating a increasing wall for under-performing pairs. All protections are evaluated in the sequence they are defined. -The below example: +The below example assumes a timeframe of 1 hour: -* Locks each pair after selling for an additional 10 minutes (`CooldownPeriod`), giving other pairs a chance to get filled. -* Stops trading if the last 2 days had 20 trades, which caused a max-drawdown of more than 20%. (`MaxDrawdown`). -* Stops trading if more than 4 stoploss occur for all pairs within a 1 day (1440min) limit (`StoplossGuard`). -* Locks all pairs that had 4 Trades within the last 6 hours (`60 * 6 = 360`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). -* Locks all pairs for 120 minutes that had a profit of below 0.01 (<1%) within the last 24h (`60 * 24 = 1440`), a minimum of 4 trades. +* Locks each pair after selling for an additional 5 candles (`CooldownPeriod`), giving other pairs a chance to get filled. +* Stops trading for 4 hours (`4 * 1h candles`) if the last 2 days (`48 * 1h candles`) had 20 trades, which caused a max-drawdown of more than 20%. (`MaxDrawdown`). +* Stops trading if more than 4 stoploss occur for all pairs within a 1 day (`24 * 1h candles`) limit (`StoplossGuard`). +* Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). +* Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades. ```json +"timeframe": "1h", "protections": [ { "method": "CooldownPeriod", - "stop_duration": 10 + "stop_duration_candles": 5 }, { "method": "MaxDrawdown", - "lookback_period": 2880, + "lookback_period_candles": 48, "trade_limit": 20, - "stop_duration": 720, + "stop_duration_candles": 4, "max_allowed_drawdown": 0.2 }, { "method": "StoplossGuard", - "lookback_period": 1440, + "lookback_period_candles": 24, "trade_limit": 4, - "stop_duration": 120, + "stop_duration_candles": 2, "only_per_pair": false }, { "method": "LowProfitPairs", - "lookback_period": 360, + "lookback_period_candles": 6, "trade_limit": 2, - "stop_duration": 60, + "stop_duration_candles": 60, "required_profit": 0.02 }, { "method": "LowProfitPairs", - "lookback_period": 1440, + "lookback_period_candles": 24, "trade_limit": 4, - "stop_duration": 120, + "stop_duration_candles": 2, "required_profit": 0.01 } ], diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index a6435d0e6..b8829b80f 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -170,7 +170,7 @@ def _validate_protections(conf: Dict[str, Any]) -> None: f"Please fix the protection {prot.get('method')}" ) - if ('lookback_period' in prot and 'lookback_period_candle' in prot): + if ('lookback_period' in prot and 'lookback_period_candles' in prot): raise OperationalException( "Protections must specify either `lookback_period` or `lookback_period_candles`.\n" f"Please fix the protection {prot.get('method')}" diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 60f83eea6..7a5a87f47 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple +from freqtrade.exchange import timeframe_to_minutes from freqtrade.mixins import LoggingMixin from freqtrade.persistence import Trade @@ -23,8 +24,15 @@ class IProtection(LoggingMixin, ABC): def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: self._config = config self._protection_config = protection_config - self._stop_duration = protection_config.get('stop_duration', 60) - self._lookback_period = protection_config.get('lookback_period', 60) + tf_in_min = timeframe_to_minutes(config['timeframe']) + if 'stop_duration_candles' in protection_config: + self._stop_duration = (tf_in_min * protection_config.get('stop_duration_candles')) + else: + self._stop_duration = protection_config.get('stop_duration', 60) + if 'lookback_period_candles' in protection_config: + self._lookback_period = tf_in_min * protection_config.get('lookback_period_candles', 60) + else: + self._lookback_period = protection_config.get('lookback_period', 60) LoggingMixin.__init__(self, logger) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 29ff4e069..819ae805e 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -3,9 +3,10 @@ from datetime import datetime, timedelta import pytest -from freqtrade.persistence import PairLocks, Trade -from freqtrade.strategy.interface import SellType from freqtrade import constants +from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.protectionmanager import ProtectionManager +from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has_re @@ -49,6 +50,33 @@ def test_protectionmanager(mocker, default_conf): assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None) +@pytest.mark.parametrize('timeframe,expected,protconf', [ + ('1m', [20, 10], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 10}]), + ('5m', [100, 15], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 15}]), + ('1h', [1200, 40], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 40}]), + ('1d', [1440, 5], + [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration": 5}]), + ('1m', [20, 5], + [{"method": "StoplossGuard", "lookback_period": 20, "stop_duration_candles": 5}]), + ('5m', [15, 25], + [{"method": "StoplossGuard", "lookback_period": 15, "stop_duration_candles": 5}]), + ('1h', [50, 600], + [{"method": "StoplossGuard", "lookback_period": 50, "stop_duration_candles": 10}]), + ('1h', [60, 540], + [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration_candles": 9}]), +]) +def test_protections_init(mocker, default_conf, timeframe, expected, protconf): + default_conf['timeframe'] = timeframe + default_conf['protections'] = protconf + man = ProtectionManager(default_conf) + assert len(man._protection_handlers) == len(protconf) + assert man._protection_handlers[0]._lookback_period == expected[0] + assert man._protection_handlers[0]._stop_duration == expected[1] + + @pytest.mark.usefixtures("init_persistence") def test_stoploss_guard(mocker, default_conf, fee, caplog): default_conf['protections'] = [{ diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 283f6a0f9..bebbc1508 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -879,11 +879,12 @@ def test_validate_whitelist(default_conf): validate_config_consistency(conf) + @pytest.mark.parametrize('protconf,expected', [ ([], None), ([{"method": "StoplossGuard", "lookback_period": 2000, "stop_duration_candles": 10}], None), - ([{"method": "StoplossGuard", "lookback_period_candle": 20, "stop_duration": 10}], None), - ([{"method": "StoplossGuard", "lookback_period_candle": 20, "lookback_period": 2000, + ([{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 10}], None), + ([{"method": "StoplossGuard", "lookback_period_candles": 20, "lookback_period": 2000, "stop_duration": 10}], r'Protections must specify either `lookback_period`.*'), ([{"method": "StoplossGuard", "lookback_period": 20, "stop_duration": 10, "stop_duration_candles": 10}], r'Protections must specify either `stop_duration`.*'), From c993831a04c8a92af179eb35a7bd6983458d0b56 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 10:54:37 +0100 Subject: [PATCH 50/73] Add protections to startup messages --- freqtrade/freqtradebot.py | 2 +- freqtrade/rpc/rpc_manager.py | 7 ++++++- tests/rpc/test_rpc_manager.py | 10 +++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ecc824a86..9fc342056 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -137,7 +137,7 @@ class FreqtradeBot(LoggingMixin): Called on startup and after reloading the bot - triggers notifications and performs startup tasks """ - self.rpc.startup_messages(self.config, self.pairlists) + self.rpc.startup_messages(self.config, self.pairlists, self.protections) if not self.edge: # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index b97a5357b..ab5e09ddd 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -62,7 +62,7 @@ class RPCManager: except NotImplementedError: logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.") - def startup_messages(self, config: Dict[str, Any], pairlist) -> None: + def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None: if config['dry_run']: self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, @@ -90,3 +90,8 @@ class RPCManager: 'status': f'Searching for {stake_currency} pairs to buy and sell ' f'based on {pairlist.short_desc()}' }) + if len(protections.name_list) > 0: + self.send_msg({ + 'type': RPCMessageType.STARTUP_NOTIFICATION, + 'status': f'Using Protections {protections.short_desc()}' + }) diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 4b715fc37..06706120f 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -137,7 +137,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) assert telegram_mock.call_count == 3 assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] @@ -147,10 +147,14 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: default_conf['whitelist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} } + default_conf['protections'] = [{"method": "StoplossGuard", + "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}] + freqtradebot = get_patched_freqtradebot(mocker, default_conf) - rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) - assert telegram_mock.call_count == 3 + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) + assert telegram_mock.call_count == 4 assert "Dry run is enabled." in telegram_mock.call_args_list[0][0][0]['status'] + assert 'StoplossGuard' in telegram_mock.call_args_list[-1][0][0]['status'] def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: From 0e2a43ab4dbd49e74f1aaca0da2d19dd41aa1dd9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 11:08:54 +0100 Subject: [PATCH 51/73] Add duration_explanation functions --- .../plugins/protections/cooldown_period.py | 6 ++-- freqtrade/plugins/protections/iprotection.py | 33 +++++++++++++++++-- .../plugins/protections/low_profit_pairs.py | 6 ++-- .../protections/max_drawdown_protection.py | 8 ++--- .../plugins/protections/stoploss_guard.py | 2 +- tests/plugins/test_protections.py | 22 ++++++++++++- 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 7b37b2303..e5eae01dd 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -22,13 +22,13 @@ class CooldownPeriod(IProtection): """ LockReason to use """ - return (f'Cooldown period for {self._stop_duration} min.') + return (f'Cooldown period for {self.stop_duration_str}.') def short_desc(self) -> str: """ Short method description - used for startup-messages """ - return (f"{self.name} - Cooldown period of {self._stop_duration} min.") + return (f"{self.name} - Cooldown period of {self.stop_duration_str}.") def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: """ @@ -42,7 +42,7 @@ class CooldownPeriod(IProtection): ] trade = Trade.get_trades(filters).first() if trade: - self.log_once(f"Cooldown for {pair} for {self._stop_duration}.", logger.info) + self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) return True, until, self._reason() diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 7a5a87f47..684bf6cd3 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import plural from freqtrade.mixins import LoggingMixin from freqtrade.persistence import Trade @@ -26,12 +27,16 @@ class IProtection(LoggingMixin, ABC): self._protection_config = protection_config tf_in_min = timeframe_to_minutes(config['timeframe']) if 'stop_duration_candles' in protection_config: - self._stop_duration = (tf_in_min * protection_config.get('stop_duration_candles')) + self._stop_duration_candles = protection_config.get('stop_duration_candles', 1) + self._stop_duration = (tf_in_min * self._stop_duration_candles) else: + self._stop_duration_candles = None self._stop_duration = protection_config.get('stop_duration', 60) if 'lookback_period_candles' in protection_config: - self._lookback_period = tf_in_min * protection_config.get('lookback_period_candles', 60) + self._lookback_period_candles = protection_config.get('lookback_period_candles', 1) + self._lookback_period = tf_in_min * self._lookback_period_candles else: + self._lookback_period_candles = None self._lookback_period = protection_config.get('lookback_period', 60) LoggingMixin.__init__(self, logger) @@ -40,6 +45,30 @@ class IProtection(LoggingMixin, ABC): def name(self) -> str: return self.__class__.__name__ + @property + def stop_duration_str(self) -> str: + """ + Output configured stop duration in either candles or minutes + """ + if self._stop_duration_candles: + return (f"{self._stop_duration_candles} " + f"{plural(self._stop_duration_candles, 'candle', 'candles')}") + else: + return (f"{self._stop_duration} " + f"{plural(self._stop_duration, 'minute', 'minutes')}") + + @property + def lookback_period_str(self) -> str: + """ + Output configured lookback period in either candles or minutes + """ + if self._lookback_period_candles: + return (f"{self._lookback_period_candles} " + f"{plural(self._lookback_period_candles, 'candle', 'candles')}") + else: + return (f"{self._lookback_period} " + f"{plural(self._lookback_period, 'minute', 'minutes')}") + @abstractmethod def short_desc(self) -> str: """ diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 70ef5b080..4721ea1a2 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -26,14 +26,14 @@ class LowProfitPairs(IProtection): Short method description - used for startup-messages """ return (f"{self.name} - Low Profit Protection, locks pairs with " - f"profit < {self._required_profit} within {self._lookback_period} minutes.") + f"profit < {self._required_profit} within {self.lookback_period_str}.") def _reason(self, profit: float) -> str: """ LockReason to use """ - return (f'{profit} < {self._required_profit} in {self._lookback_period} min, ' - f'locking for {self._stop_duration} min.') + return (f'{profit} < {self._required_profit} in {self.lookback_period_str}, ' + f'locking for {self.stop_duration_str}.') def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn: """ diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index 2a83cdeba..e0c91243b 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -30,14 +30,14 @@ class MaxDrawdown(IProtection): Short method description - used for startup-messages """ return (f"{self.name} - Max drawdown protection, stop trading if drawdown is > " - f"{self._max_allowed_drawdown} within {self._lookback_period} minutes.") + f"{self._max_allowed_drawdown} within {self.lookback_period_str}.") def _reason(self, drawdown: float) -> str: """ LockReason to use """ - return (f'{drawdown} > {self._max_allowed_drawdown} in {self._lookback_period} min, ' - f'locking for {self._stop_duration} min.') + return (f'{drawdown} > {self._max_allowed_drawdown} in {self.lookback_period_str}, ' + f'locking for {self.stop_duration_str}.') def _max_drawdown(self, date_now: datetime) -> ProtectionReturn: """ @@ -62,7 +62,7 @@ class MaxDrawdown(IProtection): if drawdown > self._max_allowed_drawdown: self.log_once( f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}" - f" within {self._lookback_period} minutes.", logger.info) + f" within {self.lookback_period_str}.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) return True, until, self._reason(drawdown) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 520607337..7a13ead57 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -29,7 +29,7 @@ class StoplossGuard(IProtection): Short method description - used for startup-messages """ return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses " - f"within {self._lookback_period} minutes.") + f"within {self.lookback_period_str}.") def _reason(self) -> str: """ diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 819ae805e..22fe33e19 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -350,7 +350,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): None ), ({"method": "CooldownPeriod", "stop_duration": 60}, - "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 min.'}]", + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 minutes.'}]", None ), ({"method": "LowProfitPairs", "lookback_period": 60, "stop_duration": 60}, @@ -363,6 +363,26 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): "within 60 minutes.'}]", None ), + ({"method": "StoplossGuard", "lookback_period_candles": 12, "trade_limit": 2, + "stop_duration": 60}, + "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " + "2 stoplosses within 12 candles.'}]", + None + ), + ({"method": "CooldownPeriod", "stop_duration_candles": 5}, + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 5 candles.'}]", + None + ), + ({"method": "LowProfitPairs", "lookback_period_candles": 11, "stop_duration": 60}, + "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " + "profit < 0.0 within 11 candles.'}]", + None + ), + ({"method": "MaxDrawdown", "lookback_period_candles": 20, "stop_duration": 60}, + "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " + "within 20 candles.'}]", + None + ), ]) def test_protection_manager_desc(mocker, default_conf, protectionconf, desc_expected, exception_expected): From 64d6c7bb651765ff5ab1071879f5197e7fbba025 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 11:17:11 +0100 Subject: [PATCH 52/73] Update developer docs --- docs/developer.md | 6 ++++-- freqtrade/plugins/protectionmanager.py | 3 --- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 05b518184..f1d658ab8 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -202,10 +202,10 @@ For that reason, they must implement the following methods: The `until` portion should be calculated using the provided `calculate_lock_end()` method. -All Protections should use `"stop_duration"` to define how long a a pair (or all pairs) should be locked. +All Protections should use `"stop_duration"` / `"stop_duration_candles"` to define how long a a pair (or all pairs) should be locked. The content of this is made available as `self._stop_duration` to the each Protection. -If your protection requires a look-back period, please use `"lookback_period"` to keep different protections aligned. +If your protection requires a look-back period, please use `"lookback_period"` / `"lockback_period_candles"` to keep all protections aligned. #### Global vs. local stops @@ -230,6 +230,8 @@ The method `global_stop()` will be called whenever a trade closed (sell order co Protections should calculate the lock end time based on the last trade it considers. This avoids relocking should the lookback-period be longer than the actual lock period. +The `IProtection` parent class provides a helper method for this in `calculate_lock_end()`. + --- ## Implement a new Exchange (WIP) diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index d12f4ba80..03a09cc58 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -20,9 +20,6 @@ class ProtectionManager(): self._protection_handlers: List[IProtection] = [] for protection_handler_config in self._config.get('protections', []): - if 'method' not in protection_handler_config: - logger.warning(f"No method found in {protection_handler_config}, ignoring.") - continue protection_handler = ProtectionResolver.load_protection( protection_handler_config['method'], config=config, From 3426e99b8b3ad3f93eac53f474c84fc20c461c8e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 11:37:57 +0100 Subject: [PATCH 53/73] Improve formatting of protection startup message --- freqtrade/rpc/rpc_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index ab5e09ddd..c42878f99 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -91,7 +91,8 @@ class RPCManager: f'based on {pairlist.short_desc()}' }) if len(protections.name_list) > 0: + prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()]) self.send_msg({ 'type': RPCMessageType.STARTUP_NOTIFICATION, - 'status': f'Using Protections {protections.short_desc()}' + 'status': f'Using Protections: \n{prots}' }) From 98c88fa58e890630a2c9221d48a31c42423a28ca Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Nov 2020 20:09:34 +0100 Subject: [PATCH 54/73] Prepare protections for backtesting --- freqtrade/persistence/models.py | 41 +++++++++++++++++++ freqtrade/plugins/protectionmanager.py | 12 +++--- .../plugins/protections/cooldown_period.py | 17 ++++---- .../plugins/protections/low_profit_pairs.py | 16 ++++---- .../protections/max_drawdown_protection.py | 7 +--- .../plugins/protections/stoploss_guard.py | 25 ++++++----- 6 files changed, 84 insertions(+), 34 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 04d5a7695..d262a6784 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -202,6 +202,10 @@ class Trade(_DECL_BASE): """ __tablename__ = 'trades' + use_db: bool = True + # Trades container for backtesting + trades: List['Trade'] = [] + id = Column(Integer, primary_key=True) orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") @@ -562,6 +566,43 @@ class Trade(_DECL_BASE): else: return Trade.query + @staticmethod + def get_trades_proxy(*, pair: str = None, is_open: bool = None, + open_date: datetime = None, close_date: datetime = None, + ) -> List['Trade']: + """ + Helper function to query Trades. + Returns a List of trades, filtered on the parameters given. + In live mode, converts the filter to a database query and returns all rows + In Backtest mode, uses filters on Trade.trades to get the result. + + :return: unsorted List[Trade] + """ + if Trade.use_db: + trade_filter = [] + if pair: + trade_filter.append(Trade.pair == pair) + if open_date: + trade_filter.append(Trade.open_date > open_date) + if close_date: + trade_filter.append(Trade.close_date > close_date) + if is_open is not None: + trade_filter.append(Trade.is_open.is_(is_open)) + return Trade.get_trades(trade_filter).all() + else: + # Offline mode - without database + sel_trades = [trade for trade in Trade.trades] + if pair: + sel_trades = [trade for trade in sel_trades if trade.pair == pair] + if open_date: + sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] + if close_date: + sel_trades = [trade for trade in sel_trades if trade.close_date + and trade.close_date > close_date] + if is_open is not None: + sel_trades = [trade for trade in sel_trades if trade.is_open == is_open] + return sel_trades + @staticmethod def get_open_trades() -> List[Any]: """ diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index 03a09cc58..a8edd4e4b 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -3,7 +3,7 @@ Protection manager class """ import logging from datetime import datetime, timezone -from typing import Dict, List +from typing import Dict, List, Optional from freqtrade.persistence import PairLocks from freqtrade.plugins.protections import IProtection @@ -43,8 +43,9 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self) -> bool: - now = datetime.now(timezone.utc) + def global_stop(self, now: Optional[datetime] = None) -> bool: + if not now: + now = datetime.now(timezone.utc) result = False for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: @@ -57,8 +58,9 @@ class ProtectionManager(): result = True return result - def stop_per_pair(self, pair) -> bool: - now = datetime.now(timezone.utc) + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + if not now: + now = datetime.now(timezone.utc) result = False for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index e5eae01dd..2d7d7b4c7 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -35,13 +35,16 @@ class CooldownPeriod(IProtection): Get last trade for this pair """ look_back_until = date_now - timedelta(minutes=self._stop_duration) - filters = [ - Trade.is_open.is_(False), - Trade.close_date > look_back_until, - Trade.pair == pair, - ] - trade = Trade.get_trades(filters).first() - if trade: + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # Trade.pair == pair, + # ] + # trade = Trade.get_trades(filters).first() + trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + if trades: + # Get latest trade + trade = sorted(trades, key=lambda t: t.close_date)[-1] self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 4721ea1a2..9d5ed35b4 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -40,13 +40,15 @@ class LowProfitPairs(IProtection): Evaluate recent trades for pair """ look_back_until = date_now - timedelta(minutes=self._lookback_period) - filters = [ - Trade.is_open.is_(False), - Trade.close_date > look_back_until, - ] - if pair: - filters.append(Trade.pair == pair) - trades = Trade.get_trades(filters).all() + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # ] + # if pair: + # filters.append(Trade.pair == pair) + + trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + # trades = Trade.get_trades(filters).all() if len(trades) < self._trade_limit: # Not enough trades in the relevant period return False, None, None diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index e0c91243b..f1c77d1d9 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -44,11 +44,8 @@ class MaxDrawdown(IProtection): Evaluate recent trades for drawdown ... """ look_back_until = date_now - timedelta(minutes=self._lookback_period) - filters = [ - Trade.is_open.is_(False), - Trade.close_date > look_back_until, - ] - trades = Trade.get_trades(filters).all() + + trades = Trade.get_trades_proxy(is_open=False, close_date=look_back_until) trades_df = pd.DataFrame([trade.to_json() for trade in trades]) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 7a13ead57..4dbc71048 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -43,16 +43,21 @@ class StoplossGuard(IProtection): Evaluate recent trades """ look_back_until = date_now - timedelta(minutes=self._lookback_period) - filters = [ - Trade.is_open.is_(False), - Trade.close_date > look_back_until, - or_(Trade.sell_reason == SellType.STOP_LOSS.value, - and_(Trade.sell_reason == SellType.TRAILING_STOP_LOSS.value, - Trade.close_profit < 0)) - ] - if pair: - filters.append(Trade.pair == pair) - trades = Trade.get_trades(filters).all() + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # or_(Trade.sell_reason == SellType.STOP_LOSS.value, + # and_(Trade.sell_reason == SellType.TRAILING_STOP_LOSS.value, + # Trade.close_profit < 0)) + # ] + # if pair: + # filters.append(Trade.pair == pair) + # trades = Trade.get_trades(filters).all() + + trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + trades = [trade for trade in trades1 if trade.sell_reason == SellType.STOP_LOSS + or (trade.sell_reason == SellType.TRAILING_STOP_LOSS + and trade.close_profit < 0)] if len(trades) > self._trade_limit: self.log_once(f"Trading stopped due to {self._trade_limit} " From b606936eb70131856a347d54540a784e23035085 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Nov 2020 20:17:47 +0100 Subject: [PATCH 55/73] Make changes to backtesting to incorporate protections --- freqtrade/optimize/backtesting.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 883f7338c..1d183152c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -21,7 +21,8 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_stats) from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.persistence import Trade +from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType @@ -115,6 +116,11 @@ class Backtesting: else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) + Trade.use_db = False + PairLocks.timeframe = self.config['timeframe'] + PairLocks.use_db = False + self.protections = ProtectionManager(self.config) + # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) # Load one (first) strategy @@ -235,6 +241,10 @@ class Backtesting: trade_dur = int((sell_row[DATE_IDX] - trade.open_date).total_seconds() // 60) closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) + trade.close_date = sell_row[DATE_IDX] + trade.sell_reason = sell.sell_type + trade.close(closerate) + return BacktestResult(pair=trade.pair, profit_percent=trade.calc_profit_ratio(rate=closerate), profit_abs=trade.calc_profit(rate=closerate), @@ -261,6 +271,7 @@ class Backtesting: if len(open_trades[pair]) > 0: for trade in open_trades[pair]: sell_row = data[pair][-1] + trade_entry = BacktestResult(pair=trade.pair, profit_percent=trade.calc_profit_ratio( rate=sell_row[OPEN_IDX]), @@ -320,6 +331,8 @@ class Backtesting: while tmp <= end_date: open_trade_count_start = open_trade_count + self.protections.global_stop(tmp) + for i, pair in enumerate(data): if pair not in indexes: indexes[pair] = 0 @@ -342,7 +355,8 @@ class Backtesting: if ((position_stacking or len(open_trades[pair]) == 0) and (max_open_trades <= 0 or open_trade_count_start < max_open_trades) and tmp != end_date - and row[BUY_IDX] == 1 and row[SELL_IDX] != 1): + and row[BUY_IDX] == 1 and row[SELL_IDX] != 1 + and not PairLocks.is_pair_locked(pair, row[DATE_IDX])): # Enter trade trade = Trade( pair=pair, @@ -361,6 +375,7 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") open_trades[pair].append(trade) + Trade.trades.append(trade) for trade in open_trades[pair]: # since indexes has been incremented before, we need to go one step back to @@ -372,6 +387,7 @@ class Backtesting: open_trade_count -= 1 open_trades[pair].remove(trade) trades.append(trade_entry) + self.protections.stop_per_pair(pair, row[DATE_IDX]) # Move time one configured time_interval ahead. tmp += timedelta(minutes=self.timeframe_min) From 9f34aebdaa4ebdbeee78130d03e35f55352551e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 16 Nov 2020 20:21:32 +0100 Subject: [PATCH 56/73] Allow closing trades without message --- freqtrade/optimize/backtesting.py | 2 +- freqtrade/persistence/models.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1d183152c..f80976a20 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -243,7 +243,7 @@ class Backtesting: trade.close_date = sell_row[DATE_IDX] trade.sell_reason = sell.sell_type - trade.close(closerate) + trade.close(closerate, show_msg=False) return BacktestResult(pair=trade.pair, profit_percent=trade.calc_profit_ratio(rate=closerate), diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index d262a6784..9b8f561b8 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -411,7 +411,7 @@ class Trade(_DECL_BASE): raise ValueError(f'Unknown order type: {order_type}') cleanup_db() - def close(self, rate: float) -> None: + def close(self, rate: float, *, show_msg: bool = False) -> None: """ Sets close_rate to the given rate, calculates total profit and marks trade as closed @@ -423,10 +423,11 @@ class Trade(_DECL_BASE): self.is_open = False self.sell_order_status = 'closed' self.open_order_id = None - logger.info( - 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', - self - ) + if show_msg: + logger.info( + 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', + self + ) def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], side: str) -> None: From 32189d27c82db0a3239075c6259a1770cb78e5c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 19 Nov 2020 20:05:56 +0100 Subject: [PATCH 57/73] Disable output from plugins in backtesting --- freqtrade/optimize/backtesting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f80976a20..7ead5ca24 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -25,6 +25,7 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) @@ -68,6 +69,8 @@ class Backtesting: """ def __init__(self, config: Dict[str, Any]) -> None: + + LoggingMixin.show_output = False self.config = config # Reset keys for backtesting From e2d15f40824cdc26dc55081eda1e5fa0493cbc06 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 23 Nov 2020 20:29:29 +0100 Subject: [PATCH 58/73] Add parameter to enable protections for backtesting --- freqtrade/commands/arguments.py | 6 ++++-- freqtrade/commands/cli_options.py | 8 ++++++++ freqtrade/configuration/configuration.py | 3 +++ freqtrade/optimize/backtesting.py | 11 ++++++++--- freqtrade/optimize/hyperopt.py | 2 ++ 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index aa58ff585..a7ae969f4 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -20,11 +20,13 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", + "enable_protections", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", - "position_stacking", "epochs", "spaces", - "use_max_market_positions", "print_all", + "position_stacking", "use_max_market_positions", + "enable_protections", + "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_loss"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 619a300ae..668b4abf5 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -144,6 +144,14 @@ AVAILABLE_CLI_OPTIONS = { action='store_false', default=True, ), + "enable_protections": Arg( + '--enable-protections', '--enableprotections', + help='Enable protections for backtesting.' + 'Will slow backtesting down by a considerable amount, but will include ' + 'configured protections', + action='store_true', + default=False, + ), "strategy_list": Arg( '--strategy-list', help='Provide a space-separated list of strategies to backtest. ' diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 1ca3187fb..7bf3e6bf2 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -211,6 +211,9 @@ class Configuration: self._args_to_config(config, argname='position_stacking', logstring='Parameter --enable-position-stacking detected ...') + self._args_to_config( + config, argname='enable_protections', + logstring='Parameter --enable-protections detected, enabling Protections. ...') # Setting max_open_trades to infinite if -1 if config.get('max_open_trades') == -1: config['max_open_trades'] = float('inf') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7ead5ca24..56cc426ac 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -297,7 +297,8 @@ class Backtesting: def backtest(self, processed: Dict, stake_amount: float, start_date: datetime, end_date: datetime, - max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: + max_open_trades: int = 0, position_stacking: bool = False, + enable_protections: bool = False) -> DataFrame: """ Implement backtesting functionality @@ -311,6 +312,7 @@ class Backtesting: :param end_date: backtesting timerange end datetime :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited :param position_stacking: do we allow position stacking? + :param enable_protections: Should protections be enabled? :return: DataFrame with trades (results of backtesting) """ logger.debug(f"Run backtest, stake_amount: {stake_amount}, " @@ -334,7 +336,8 @@ class Backtesting: while tmp <= end_date: open_trade_count_start = open_trade_count - self.protections.global_stop(tmp) + if enable_protections: + self.protections.global_stop(tmp) for i, pair in enumerate(data): if pair not in indexes: @@ -390,7 +393,8 @@ class Backtesting: open_trade_count -= 1 open_trades[pair].remove(trade) trades.append(trade_entry) - self.protections.stop_per_pair(pair, row[DATE_IDX]) + if enable_protections: + self.protections.stop_per_pair(pair, row[DATE_IDX]) # Move time one configured time_interval ahead. tmp += timedelta(minutes=self.timeframe_min) @@ -446,6 +450,7 @@ class Backtesting: end_date=max_date.datetime, max_open_trades=max_open_trades, position_stacking=position_stacking, + enable_protections=self.config.get('enable_protections'), ) all_results[self.strategy.get_strategy_name()] = { 'results': results, diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7870ba1cf..2a2f5b472 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -542,6 +542,8 @@ class Hyperopt: end_date=max_date.datetime, max_open_trades=self.max_open_trades, position_stacking=self.position_stacking, + enable_protections=self.config.get('enable_protections', False), + ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) From 946fb094553ab32854d6992f9ecc58e6db26ea42 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Nov 2020 06:51:54 +0100 Subject: [PATCH 59/73] Update help command output --- docs/bot-usage.md | 43 +++++++++++++++++++++++++----------- docs/includes/protections.md | 3 +++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4d07435c7..5820b3cc7 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -213,9 +213,11 @@ Backtesting also uses the config specified via `-c/--config`. usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TIMEFRAME] - [--timerange TIMERANGE] [--max-open-trades INT] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [--eps] [--dmmp] + [--eps] [--dmmp] [--enable-protections] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] @@ -226,6 +228,9 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -241,6 +246,10 @@ optional arguments: Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number). + --enable-protections, --enableprotections + Enable protections for backtesting.Will slow + backtesting down by a considerable amount, but will + include configured protections --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to backtest. Please note that ticker-interval needs to be @@ -296,13 +305,14 @@ to find optimal parameter values for your strategy. usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--hyperopt NAME] [--hyperopt-path PATH] [--eps] - [-e INT] + [--dmmp] [--enable-protections] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] - [--dmmp] [--print-all] [--no-color] [--print-json] - [-j JOBS] [--random-state INT] [--min-trades INT] + [--print-all] [--no-color] [--print-json] [-j JOBS] + [--random-state INT] [--min-trades INT] [--hyperopt-loss NAME] optional arguments: @@ -312,6 +322,9 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -327,14 +340,18 @@ optional arguments: --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). - -e INT, --epochs INT Specify number of epochs (default: 100). - --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] - Specify which parameters to hyperopt. Space-separated - list. --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number). + --enable-protections, --enableprotections + Enable protections for backtesting.Will slow + backtesting down by a considerable amount, but will + include configured protections + -e INT, --epochs INT Specify number of epochs (default: 100). + --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] + Specify which parameters to hyperopt. Space-separated + list. --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. @@ -353,10 +370,10 @@ optional arguments: class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in - Hyperopt-loss-functions are: ShortTradeDurHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss, - SharpeHyperOptLossDaily, SortinoHyperOptLoss, - SortinoHyperOptLossDaily. + Hyperopt-loss-functions are: + ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, + SharpeHyperOptLoss, SharpeHyperOptLossDaily, + SortinoHyperOptLoss, SortinoHyperOptLossDaily Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 210765176..351cfcac3 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -12,6 +12,9 @@ All protection end times are rounded up to the next candle to avoid sudden, unex !!! Tip Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). +!!! Note "Backtesting" + Protections are supported by backtesting and hyperopt, but must be enabled by using the `--enable-protections` flag. + ### Available Protections * [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. From a3f9cd2c26cfaf70033f99d4d4a1e8cffc5f9c54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 Nov 2020 07:38:09 +0100 Subject: [PATCH 60/73] Only load protections when necessary --- freqtrade/optimize/backtesting.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 56cc426ac..1819e5617 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -122,7 +122,8 @@ class Backtesting: Trade.use_db = False PairLocks.timeframe = self.config['timeframe'] PairLocks.use_db = False - self.protections = ProtectionManager(self.config) + if self.config.get('enable_protections', False): + self.protections = ProtectionManager(self.config) # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) @@ -450,7 +451,7 @@ class Backtesting: end_date=max_date.datetime, max_open_trades=max_open_trades, position_stacking=position_stacking, - enable_protections=self.config.get('enable_protections'), + enable_protections=self.config.get('enable_protections', False), ) all_results[self.strategy.get_strategy_name()] = { 'results': results, From 75a5161650072fc5592bcb63e6f11cc8e2aab07f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 25 Nov 2020 09:53:13 +0100 Subject: [PATCH 61/73] Support multis-strategy backtests with protections --- freqtrade/optimize/backtesting.py | 14 ++++++++ freqtrade/persistence/models.py | 8 +++++ freqtrade/persistence/pairlock_middleware.py | 8 +++++ .../plugins/protections/stoploss_guard.py | 4 +-- tests/optimize/test_backtesting.py | 34 +++++++++++++++++-- 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 1819e5617..e3f5e7671 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -120,8 +120,10 @@ class Backtesting: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) Trade.use_db = False + Trade.reset_trades() PairLocks.timeframe = self.config['timeframe'] PairLocks.use_db = False + PairLocks.reset_locks() if self.config.get('enable_protections', False): self.protections = ProtectionManager(self.config) @@ -130,6 +132,11 @@ class Backtesting: # Load one (first) strategy self._set_strategy(self.strategylist[0]) + def __del__(self): + LoggingMixin.show_output = True + PairLocks.use_db = True + Trade.use_db = True + def _set_strategy(self, strategy): """ Load strategy into backtesting @@ -321,6 +328,13 @@ class Backtesting: f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" ) trades = [] + PairLocks.use_db = False + Trade.use_db = False + if enable_protections: + # Reset persisted data - used for protections only + + PairLocks.reset_locks() + Trade.reset_trades() # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 9b8f561b8..07f4b5a4f 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -327,6 +327,14 @@ class Trade(_DECL_BASE): 'open_order_id': self.open_order_id, } + @staticmethod + def reset_trades() -> None: + """ + Resets all trades. Only active for backtesting mode. + """ + if not Trade.use_db: + Trade.trades = [] + def adjust_min_max_rates(self, current_price: float) -> None: """ Adjust the max_rate and min_rate. diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 6ce91ee6b..8644146d8 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -21,6 +21,14 @@ class PairLocks(): timeframe: str = '' + @staticmethod + def reset_locks() -> None: + """ + Resets all locks. Only active for backtesting mode. + """ + if not PairLocks.use_db: + PairLocks.locks = [] + @staticmethod def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: """ diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 4dbc71048..71e74880c 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -55,8 +55,8 @@ class StoplossGuard(IProtection): # trades = Trade.get_trades(filters).all() trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) - trades = [trade for trade in trades1 if trade.sell_reason == SellType.STOP_LOSS - or (trade.sell_reason == SellType.TRAILING_STOP_LOSS + trades = [trade for trade in trades1 if str(trade.sell_reason) == SellType.STOP_LOSS.value + or (str(trade.sell_reason) == SellType.TRAILING_STOP_LOSS.value and trade.close_profit < 0)] if len(trades) > self._trade_limit: diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 45cbea68e..15ad18bf9 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -95,6 +95,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: end_date=max_date, max_open_trades=1, position_stacking=False, + enable_protections=config.get('enable_protections', False), ) # results :: assert len(results) == num_results @@ -532,10 +533,39 @@ def test_processed(default_conf, mocker, testdatadir) -> None: def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir) -> None: - # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 19], ['lower', 0], ['sine', 35]] + tests = [ + ['sine', 35], + ['raise', 19], + ['lower', 0], + ['sine', 35], + ['raise', 19] + ] + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results + for [contour, numres] in tests: + simple_backtest(default_conf, contour, numres, mocker, testdatadir) + +def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatadir) -> None: + # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic + default_conf['protections'] = [ + { + "method": "CooldownPeriod", + "stop_duration": 3, + }] + + default_conf['enable_protections'] = True + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + tests = [ + ['sine', 9], + ['raise', 10], + ['lower', 0], + ['sine', 9], + ['raise', 10], + ] + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results for [contour, numres] in tests: simple_backtest(default_conf, contour, numres, mocker, testdatadir) From bb51da82978efe7592ed7a14619ab74a91eef4df Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 27 Nov 2020 17:38:15 +0100 Subject: [PATCH 62/73] Fix slow backtest due to protections --- freqtrade/optimize/backtesting.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e3f5e7671..2684a249c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -173,6 +173,17 @@ class Backtesting: return data, timerange + def prepare_backtest(self, enable_protections): + """ + Backtesting setup method - called once for every call to "backtest()". + """ + PairLocks.use_db = False + Trade.use_db = False + if enable_protections: + # Reset persisted data - used for protections only + PairLocks.reset_locks() + Trade.reset_trades() + def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ Helper function to convert a processed dataframes into lists for performance reasons. @@ -328,13 +339,7 @@ class Backtesting: f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" ) trades = [] - PairLocks.use_db = False - Trade.use_db = False - if enable_protections: - # Reset persisted data - used for protections only - - PairLocks.reset_locks() - Trade.reset_trades() + self.prepare_backtest(enable_protections) # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) @@ -351,9 +356,6 @@ class Backtesting: while tmp <= end_date: open_trade_count_start = open_trade_count - if enable_protections: - self.protections.global_stop(tmp) - for i, pair in enumerate(data): if pair not in indexes: indexes[pair] = 0 @@ -410,6 +412,7 @@ class Backtesting: trades.append(trade_entry) if enable_protections: self.protections.stop_per_pair(pair, row[DATE_IDX]) + self.protections.global_stop(tmp) # Move time one configured time_interval ahead. tmp += timedelta(minutes=self.timeframe_min) From 57a4044eb0a74209c206941ff9ceb3b763bcf713 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 29 Nov 2020 11:37:10 +0100 Subject: [PATCH 63/73] Enhance test verifying that locks are not replaced --- tests/plugins/test_protections.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 22fe33e19..e997c5526 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -1,5 +1,5 @@ import random -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import pytest @@ -123,6 +123,12 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert log_has_re(message, caplog) assert PairLocks.is_global_lock() + # Test 5m after lock-period - this should try and relock the pair, but end-time + # should be the previous end-time + end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) + assert freqtrade.protections.global_stop(end_time) + assert not PairLocks.is_global_lock(end_time) + @pytest.mark.parametrize('only_per_pair', [False, True]) @pytest.mark.usefixtures("init_persistence") From 5849d07497f1d36bfa0380c18f241ba6feabc8e1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 1 Dec 2020 06:51:59 +0100 Subject: [PATCH 64/73] Export locks as part of backtesting --- freqtrade/optimize/backtesting.py | 1 + freqtrade/optimize/optimize_reports.py | 1 + 2 files changed, 2 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2684a249c..5bb7eaf74 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -473,6 +473,7 @@ class Backtesting: all_results[self.strategy.get_strategy_name()] = { 'results': results, 'config': self.strategy.config, + 'locks': PairLocks.locks, } stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index b3799856e..d029ecd13 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -266,6 +266,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], backtest_days = (max_date - min_date).days strat_stats = { 'trades': results.to_dict(orient='records'), + 'locks': [lock.to_json() for lock in content['locks']], 'best_pair': best_pair, 'worst_pair': worst_pair, 'results_per_pair': pair_results, From effc96e92b352f819571a74aab5bcd8db1669803 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Dec 2020 07:42:39 +0100 Subject: [PATCH 65/73] Improve tests for backtest protections --- docs/developer.md | 5 +- docs/includes/protections.md | 4 +- freqtrade/mixins/logging_mixin.py | 2 - freqtrade/plugins/__init__.py | 2 - .../plugins/protections/stoploss_guard.py | 2 - tests/optimize/test_backtesting.py | 48 +++++++++++-------- tests/optimize/test_optimize_reports.py | 3 +- tests/plugins/test_protections.py | 2 +- 8 files changed, 37 insertions(+), 31 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index f1d658ab8..48b021027 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -121,8 +121,8 @@ The base-class provides an instance of the exchange (`self._exchange`) the pairl self._pairlist_pos = pairlist_pos ``` -!!! Note - You'll need to register your pairlist in `constants.py` under the variable `AVAILABLE_PAIRLISTS` - otherwise it will not be selectable. +!!! Tip + Don't forget to register your pairlist in `constants.py` under the variable `AVAILABLE_PAIRLISTS` - otherwise it will not be selectable. Now, let's step through the methods which require actions: @@ -184,6 +184,7 @@ No protection should use datetime directly, but use the provided `date_now` vari !!! Tip "Writing a new Protection" Best copy one of the existing Protections to have a good example. + Don't forget to register your protection in `constants.py` under the variable `AVAILABLE_PROTECTIONS` - otherwise it will not be selectable. #### Implementation of a new protection diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 351cfcac3..a8caf55b1 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -1,7 +1,7 @@ ## Protections !!! Warning "Beta feature" - This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Issue. + This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Github Issue. Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. @@ -13,7 +13,7 @@ All protection end times are rounded up to the next candle to avoid sudden, unex Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). !!! Note "Backtesting" - Protections are supported by backtesting and hyperopt, but must be enabled by using the `--enable-protections` flag. + Protections are supported by backtesting and hyperopt, but must be explicitly enabled by using the `--enable-protections` flag. ### Available Protections diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py index e9921e1ec..2e1c20a52 100644 --- a/freqtrade/mixins/logging_mixin.py +++ b/freqtrade/mixins/logging_mixin.py @@ -1,5 +1,3 @@ - - from typing import Callable from cachetools import TTLCache, cached diff --git a/freqtrade/plugins/__init__.py b/freqtrade/plugins/__init__.py index 96943268b..e69de29bb 100644 --- a/freqtrade/plugins/__init__.py +++ b/freqtrade/plugins/__init__.py @@ -1,2 +0,0 @@ -# flake8: noqa: F401 -# from freqtrade.plugins.protectionmanager import ProtectionManager diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 71e74880c..193907ddc 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,8 +3,6 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict -from sqlalchemy import and_, or_ - from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn from freqtrade.strategy.interface import SellType diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 15ad18bf9..547e55db8 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -79,7 +79,7 @@ def load_data_test(what, testdatadir): fill_missing=True)} -def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: +def simple_backtest(config, contour, mocker, testdatadir) -> None: patch_exchange(mocker) config['timeframe'] = '1m' backtesting = Backtesting(config) @@ -98,7 +98,7 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: enable_protections=config.get('enable_protections', False), ) # results :: - assert len(results) == num_results + return results # FIX: fixturize this? @@ -532,23 +532,9 @@ def test_processed(default_conf, mocker, testdatadir) -> None: assert col in cols -def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [ - ['sine', 35], - ['raise', 19], - ['lower', 0], - ['sine', 35], - ['raise', 19] - ] - # While buy-signals are unrealistic, running backtesting - # over and over again should not cause different results - for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres, mocker, testdatadir) - - def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatadir) -> None: - # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic + # While this test IS a copy of test_backtest_pricecontours, it's needed to ensure + # results do not carry-over to the next run, which is not given by using parametrize. default_conf['protections'] = [ { "method": "CooldownPeriod", @@ -567,7 +553,31 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad # While buy-signals are unrealistic, running backtesting # over and over again should not cause different results for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres, mocker, testdatadir) + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == numres + + +@pytest.mark.parametrize('protections,contour,expected', [ + (None, 'sine', 35), + (None, 'raise', 19), + (None, 'lower', 0), + (None, 'sine', 35), + (None, 'raise', 19), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'sine', 9), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'raise', 10), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'lower', 0), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'sine', 9), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'raise', 10), +]) +def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, + protections, contour, expected) -> None: + if protections: + default_conf['protections'] = protections + default_conf['enable_protections'] = True + + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == expected def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d04929164..a0e1932ff 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -76,7 +76,8 @@ def test_generate_backtest_stats(default_conf, testdatadir): "sell_reason": [SellType.ROI, SellType.STOP_LOSS, SellType.ROI, SellType.FORCE_SELL] }), - 'config': default_conf} + 'config': default_conf, + 'locks': []} } timerange = TimeRange.parse_timerange('1510688220-1510700340') min_date = Arrow.fromtimestamp(1510688220) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index e997c5526..2ad03a97c 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -1,5 +1,5 @@ import random -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta import pytest From f047297995a3e8875382a1445bbb53d6a1024810 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 15:45:02 +0100 Subject: [PATCH 66/73] Improve wording, fix bug --- docs/includes/protections.md | 2 +- freqtrade/mixins/logging_mixin.py | 1 + freqtrade/optimize/backtesting.py | 2 +- .../plugins/protections/max_drawdown_protection.py | 5 ++++- tests/plugins/test_protections.py | 12 ++++++++++++ 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index a8caf55b1..7378a590c 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -7,7 +7,7 @@ Protections will protect your strategy from unexpected events and market conditi All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. !!! Note - Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy. + Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. !!! Tip Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py index 2e1c20a52..06935d5f6 100644 --- a/freqtrade/mixins/logging_mixin.py +++ b/freqtrade/mixins/logging_mixin.py @@ -1,4 +1,5 @@ from typing import Callable + from cachetools import TTLCache, cached diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5bb7eaf74..de9c52dad 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -18,6 +18,7 @@ from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from freqtrade.mixins import LoggingMixin from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_stats) from freqtrade.pairlist.pairlistmanager import PairListManager @@ -25,7 +26,6 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType -from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index f1c77d1d9..d54e6699b 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -54,7 +54,10 @@ class MaxDrawdown(IProtection): return False, None, None # Drawdown is always positive - drawdown, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') + try: + drawdown, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') + except ValueError: + return False, None, None if drawdown > self._max_allowed_drawdown: self.log_once( diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 2ad03a97c..82b6e4500 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -304,6 +304,18 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + Trade.session.add(generate_mock_trade( + 'NEO/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + # No losing trade yet ... so max_drawdown will raise exception + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + Trade.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=500, min_ago_close=400, profit_rate=0.9, From de2cc9708dc5d6e9f9e0bfa83d11e31a31442ca6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 16:01:29 +0100 Subject: [PATCH 67/73] Fix test leakage --- tests/plugins/test_pairlocks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index db7d9f46f..bd103b21e 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -79,6 +79,7 @@ def test_PairLocks(use_db): # Nothing was pushed to the database assert len(PairLock.query.all()) == 0 # Reset use-db variable + PairLocks.reset_locks() PairLocks.use_db = True @@ -111,4 +112,5 @@ def test_PairLocks_getlongestlock(use_db): # Must be longer than above assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14) + PairLocks.reset_locks() PairLocks.use_db = True From b5289d5f0ed48ba5cbbb916ca30a388619bf62e7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 16:02:55 +0100 Subject: [PATCH 68/73] Update full config with correct protection keys --- config_full.json.example | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 737015b41..b6170bceb 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -78,26 +78,26 @@ "protections": [ { "method": "StoplossGuard", - "lookback_period": 60, + "lookback_period_candles": 60, "trade_limit": 4, - "stopduration": 60 + "stop_duration_candles": 60 }, { "method": "CooldownPeriod", - "stopduration": 20 + "stop_duration_candles": 20 }, { "method": "MaxDrawdown", - "lookback_period": 2000, + "lookback_period_candles": 200, "trade_limit": 20, - "stop_duration": 10, + "stop_duration_candles": 10, "max_allowed_drawdown": 0.2 }, { "method": "LowProfitPairs", - "lookback_period": 360, + "lookback_period_candles": 360, "trade_limit": 1, - "stop_duration": 2, + "stop_duration_candles": 2, "required_profit": 0.02 } ], From c37bc307e29d26d9db8ceec2dd3e920354dafca2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 16:07:00 +0100 Subject: [PATCH 69/73] Small finetunings to documentation --- docs/developer.md | 2 +- freqtrade/pairlist/pairlistmanager.py | 3 --- freqtrade/persistence/models.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 48b021027..dcbaa3ca9 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -229,7 +229,7 @@ The method `global_stop()` will be called whenever a trade closed (sell order co ##### Protections - calculating lock end time Protections should calculate the lock end time based on the last trade it considers. -This avoids relocking should the lookback-period be longer than the actual lock period. +This avoids re-locking should the lookback-period be longer than the actual lock period. The `IProtection` parent class provides a helper method for this in `calculate_lock_end()`. diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 89bab99be..810a22300 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -26,9 +26,6 @@ class PairListManager(): self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): - if 'method' not in pairlist_handler_config: - logger.warning(f"No method found in {pairlist_handler_config}, ignoring.") - continue pairlist_handler = PairListResolver.load_pairlist( pairlist_handler_config['method'], exchange=exchange, diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 07f4b5a4f..bcda6368a 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -419,7 +419,7 @@ class Trade(_DECL_BASE): raise ValueError(f'Unknown order type: {order_type}') cleanup_db() - def close(self, rate: float, *, show_msg: bool = False) -> None: + def close(self, rate: float, *, show_msg: bool = True) -> None: """ Sets close_rate to the given rate, calculates total profit and marks trade as closed From 82bc6973feab3937010e99cb2301cbad30651724 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 16:16:33 +0100 Subject: [PATCH 70/73] Add last key to config_full --- config_full.json.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config_full.json.example b/config_full.json.example index b6170bceb..e69e52469 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -80,7 +80,8 @@ "method": "StoplossGuard", "lookback_period_candles": 60, "trade_limit": 4, - "stop_duration_candles": 60 + "stop_duration_candles": 60, + "only_per_pair": false }, { "method": "CooldownPeriod", From f897b683c7ec3031cc0aeb6dd863c5891a6e86c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Dec 2020 19:22:14 +0100 Subject: [PATCH 71/73] Add seperate page describing plugins --- docs/plugins.md | 3 +++ mkdocs.yml | 1 + 2 files changed, 4 insertions(+) create mode 100644 docs/plugins.md diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 000000000..1f785bbaa --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,3 @@ +# Plugins +--8<-- "includes/pairlists.md" +--8<-- "includes/protections.md" diff --git a/mkdocs.yml b/mkdocs.yml index c791386ae..a7ae0cc96 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - Backtesting: backtesting.md - Hyperopt: hyperopt.md - Edge Positioning: edge.md + - Plugins: plugins.md - Utility Subcommands: utils.md - FAQ: faq.md - Data Analysis: From 8a2fbf65923283efd0db0c61c316cc5df193089f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Dec 2020 10:15:16 +0100 Subject: [PATCH 72/73] Small cleanup of protection stuff --- docs/includes/protections.md | 12 ++++++------ freqtrade/resolvers/protection_resolver.py | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 7378a590c..87db17fd8 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -26,12 +26,12 @@ All protection end times are rounded up to the next candle to avoid sudden, unex | Parameter| Description | |------------|-------------| -| method | Protection name to use.
**Datatype:** String, selected from [available Protections](#available-protections) -| stop_duration_candles | For how many candles should the lock be set?
**Datatype:** Positive integer (in candles) -| stop_duration | how many minutes should protections be locked.
Cannot be used together with `stop_duration_candles`.
**Datatype:** Float (in minutes) +| `method` | Protection name to use.
**Datatype:** String, selected from [available Protections](#available-protections) +| `stop_duration_candles` | For how many candles should the lock be set?
**Datatype:** Positive integer (in candles) +| `stop_duration` | how many minutes should protections be locked.
Cannot be used together with `stop_duration_candles`.
**Datatype:** Float (in minutes) | `lookback_period_candles` | Only trades that completed within the last `lookback_period_candles` candles will be considered. This setting may be ignored by some Protections.
**Datatype:** Positive integer (in candles). -| lookback_period | Only trades that completed after `current_time - lookback_period` will be considered.
Cannot be used together with `lookback_period_candles`.
This setting may be ignored by some Protections.
**Datatype:** Float (in minutes) -| trade_limit | Number of trades required at minimum (not used by all Protections).
**Datatype:** Positive integer +| `lookback_period` | Only trades that completed after `current_time - lookback_period` will be considered.
Cannot be used together with `lookback_period_candles`.
This setting may be ignored by some Protections.
**Datatype:** Float (in minutes) +| `trade_limit` | Number of trades required at minimum (not used by all Protections).
**Datatype:** Positive integer !!! Note "Durations" Durations (`stop_duration*` and `lookback_period*` can be defined in either minutes or candles). @@ -108,7 +108,7 @@ The below example will stop trading a pair for 2 candles after closing a trade, "protections": [ { "method": "CooldownPeriod", - "stop_duration_candle": 2 + "stop_duration_candles": 2 } ], ``` diff --git a/freqtrade/resolvers/protection_resolver.py b/freqtrade/resolvers/protection_resolver.py index 928bd4633..c54ae1011 100644 --- a/freqtrade/resolvers/protection_resolver.py +++ b/freqtrade/resolvers/protection_resolver.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=attribute-defined-outside-init - """ This module load custom pairlists """ From 9cd1be8f93f59b332d6317fc777033343ce39036 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Dec 2020 10:33:45 +0100 Subject: [PATCH 73/73] Update usage of open_trade_price to open_trade_value --- tests/plugins/test_protections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index 82b6e4500..e36900a96 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -29,7 +29,7 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, amount=0.01 / open_rate, exchange='bittrex', ) - trade.recalc_open_trade_price() + trade.recalc_open_trade_value() if not is_open: trade.close(open_rate * profit_rate) trade.sell_reason = sell_reason