Merge branch 'freqtrade:develop' into develop

config_examples/config_full.example.json

@@ -149,7 +149,9 @@
             },
             "sell_fill": "on",
             "buy_cancel": "on",
-            "sell_cancel": "on"
+            "sell_cancel": "on",
+            "protection_trigger": "off",
+            "protection_trigger_global": "on"
         },
         "reload": true,
         "balance_dust_level": 0.01

docs/requirements-docs.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.2
-mkdocs-material==7.2.6
+mkdocs-material==7.3.0
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2

docs/strategy-advanced.md

@@ -701,3 +701,33 @@ The variable 'content', will contain the strategy file in a BASE64 encoded form.
 ```
 
 Please ensure that 'NameOfStrategy' is identical to the strategy name!
+
+## Performance warning
+
+When executing a strategy, one can sometimes be greeted by the following in the logs
+
+> PerformanceWarning: DataFrame is highly fragmented.
+
+This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say:
+use `pd.concat(axis=1)`.
+This can have slight performance implications, which are usually only visible during hyperopt (when optimizing an indicator).
+
+For example:
+
+```python
+for val in self.buy_ema_short.range:
+    dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val)
+```
+
+should be rewritten to
+
+```python
+frames = [dataframe]
+for val in self.buy_ema_short.range:
+    frames.append({
+        f'ema_short_{val}': ta.EMA(dataframe, timeperiod=val)
+    })
+
+# Append columns to existing dataframe
+merged_frame = pd.concat(frames, axis=1)
+```

docs/strategy-customization.md

@@ -942,6 +942,8 @@ Printing more than a few rows is also possible (simply use  `print(dataframe)` i
 
 ## Common mistakes when developing strategies
 
+### Peeking into the future while backtesting
+
 Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future.
 This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions.
 

docs/telegram-usage.md

@@ -93,7 +93,9 @@ Example configuration showing the different settings:
         "buy_cancel": "silent",
         "sell_cancel": "on",
         "buy_fill": "off",
-        "sell_fill": "off"
+        "sell_fill": "off",
+        "protection_trigger": "off",
+        "protection_trigger_global": "on"
     },
     "reload": true,
     "balance_dust_level": 0.01
@@ -103,6 +105,7 @@ Example configuration showing the different settings:
 `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange.
 `sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange.
 `*_fill` notifications are off by default and must be explicitly enabled.
+`protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered.
 
 
 `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.

freqtrade/constants.py

@@ -113,7 +113,7 @@ CONF_SCHEMA = {
         },
         'tradable_balance_ratio': {
             'type': 'number',
-            'minimum': 0.1,
+            'minimum': 0.0,
             'maximum': 1,
             'default': 0.99
         },
@@ -287,6 +287,15 @@ CONF_SCHEMA = {
                             'enum': TELEGRAM_SETTING_OPTIONS,
                             'default': 'off'
                         },
+                        'protection_trigger': {
+                            'type': 'string',
+                            'enum': TELEGRAM_SETTING_OPTIONS,
+                            'default': 'off'
+                        },
+                        'protection_trigger_global': {
+                            'type': 'string',
+                            'enum': TELEGRAM_SETTING_OPTIONS,
+                        },
                     }
                 },
                 'reload': {'type': 'boolean'},

freqtrade/enums/rpcmessagetype.py

@@ -11,6 +11,8 @@ class RPCMessageType(Enum):
     SELL = 'sell'
     SELL_FILL = 'sell_fill'
     SELL_CANCEL = 'sell_cancel'
+    PROTECTION_TRIGGER = 'protection_trigger'
+    PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global'
 
     def __repr__(self):
         return self.value

freqtrade/freqtradebot.py

@@ -1217,7 +1217,7 @@ class FreqtradeBot(LoggingMixin):
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
             'gain': gain,
-            'limit': profit_rate,
+            'limit': profit_rate or 0,
             'order_type': order_type,
             'amount': trade.amount,
             'open_rate': trade.open_rate,
@@ -1226,7 +1226,7 @@ class FreqtradeBot(LoggingMixin):
             'profit_ratio': profit_ratio,
             'sell_reason': trade.sell_reason,
             'open_date': trade.open_date,
-            'close_date': trade.close_date,
+            'close_date': trade.close_date or datetime.now(timezone.utc),
             'stake_currency': self.config['stake_currency'],
             'fiat_currency': self.config.get('fiat_display_currency', None),
             'reason': reason,
@@ -1292,8 +1292,7 @@ class FreqtradeBot(LoggingMixin):
         if not trade.is_open:
             if not stoploss_order and not trade.open_order_id:
                 self._notify_exit(trade, '', True)
-            self.protections.stop_per_pair(trade.pair)
+            self.handle_protections(trade.pair)
-            self.protections.global_stop()
             self.wallets.update()
         elif not trade.open_order_id:
             # Buy fill
@@ -1301,6 +1300,19 @@ class FreqtradeBot(LoggingMixin):
 
         return False
 
+    def handle_protections(self, pair: str) -> None:
+        prot_trig = self.protections.stop_per_pair(pair)
+        if prot_trig:
+            msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }
+            msg.update(prot_trig.to_json())
+            self.rpc.send_msg(msg)
+
+        prot_trig_glb = self.protections.global_stop()
+        if prot_trig_glb:
+            msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, }
+            msg.update(prot_trig_glb.to_json())
+            self.rpc.send_msg(msg)
+
     def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,
                               amount: float, fee_abs: float) -> float:
         """

freqtrade/persistence/pairlock_middleware.py

@@ -30,7 +30,8 @@ class PairLocks():
             PairLocks.locks = []
 
     @staticmethod
-    def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None:
+    def lock_pair(pair: str, until: datetime, reason: str = None, *,
+                  now: datetime = None) -> PairLock:
         """
         Create PairLock from now to "until".
         Uses database by default, unless PairLocks.use_db is set to False,
@@ -52,6 +53,7 @@ class PairLocks():
             PairLock.query.session.commit()
         else:
             PairLocks.locks.append(lock)
+        return lock
 
     @staticmethod
     def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:

freqtrade/plugins/protectionmanager.py

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
 from typing import Dict, List, Optional
 
 from freqtrade.persistence import PairLocks
+from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.protections import IProtection
 from freqtrade.resolvers import ProtectionResolver
 
@@ -43,30 +44,28 @@ class ProtectionManager():
         """
         return [{p.name: p.short_desc()} for p in self._protection_handlers]
 
-    def global_stop(self, now: Optional[datetime] = None) -> bool:
+    def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]:
         if not now:
             now = datetime.now(timezone.utc)
-        result = False
+        result = None
         for protection_handler in self._protection_handlers:
             if protection_handler.has_global_stop:
-                result, until, reason = protection_handler.global_stop(now)
+                lock, until, reason = protection_handler.global_stop(now)
 
                 # Early stopping - first positive result blocks further trades
-                if result and until:
+                if lock and until:
                     if not PairLocks.is_global_lock(until):
-                        PairLocks.lock_pair('*', until, reason, now=now)
+                        result = PairLocks.lock_pair('*', until, reason, now=now)
-                    result = True
         return result
 
-    def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool:
+    def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]:
         if not now:
             now = datetime.now(timezone.utc)
-        result = False
+        result = None
         for protection_handler in self._protection_handlers:
             if protection_handler.has_local_stop:
-                result, until, reason = protection_handler.stop_per_pair(pair, now)
+                lock, until, reason = protection_handler.stop_per_pair(pair, now)
-                if result and until:
+                if lock and until:
                     if not PairLocks.is_pair_locked(pair, until):
-                        PairLocks.lock_pair(pair, until, reason, now=now)
+                        result = PairLocks.lock_pair(pair, until, reason, now=now)
-                    result = True
         return result

freqtrade/rpc/telegram.py

@@ -260,6 +260,50 @@ class Telegram(RPCHandler):
 
         return message
 
+    def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str:
+
+        if msg_type == RPCMessageType.BUY:
+            message = self._format_buy_msg(msg)
+
+        elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
+            msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell'
+            message = ("\N{WARNING SIGN} *{exchange}:* "
+                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
+                       "Reason: {reason}.".format(**msg))
+
+        elif msg_type == RPCMessageType.BUY_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Buy order for {pair} (#{trade_id}) filled "
+                       "for {open_rate}.".format(**msg))
+        elif msg_type == RPCMessageType.SELL_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Sell order for {pair} (#{trade_id}) filled "
+                       "for {close_rate}.".format(**msg))
+        elif msg_type == RPCMessageType.SELL:
+            message = self._format_sell_msg(msg)
+        elif msg_type == RPCMessageType.PROTECTION_TRIGGER:
+            message = (
+                "*Protection* triggered due to {reason}. "
+                "`{pair}` will be locked until `{lock_end_time}`."
+            ).format(**msg)
+        elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL:
+            message = (
+                "*Protection* triggered due to {reason}. "
+                "*All pairs* will be locked until `{lock_end_time}`."
+            ).format(**msg)
+        elif msg_type == RPCMessageType.STATUS:
+            message = '*Status:* `{status}`'.format(**msg)
+
+        elif msg_type == RPCMessageType.WARNING:
+            message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
+
+        elif msg_type == RPCMessageType.STARTUP:
+            message = '{status}'.format(**msg)
+
+        else:
+            raise NotImplementedError('Unknown message type: {}'.format(msg_type))
+        return message
+
     def send_msg(self, msg: Dict[str, Any]) -> None:
         """ Send a message to telegram channel """
 
@@ -284,37 +328,7 @@ class Telegram(RPCHandler):
             # Notification disabled
             return
 
-        if msg_type == RPCMessageType.BUY:
+        message = self.compose_message(msg, msg_type)
-            message = self._format_buy_msg(msg)
-
-        elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
-            msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell'
-            message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
-                       "Reason: {reason}.".format(**msg))
-
-        elif msg_type == RPCMessageType.BUY_FILL:
-            message = ("\N{LARGE CIRCLE} *{exchange}:* "
-                       "Buy order for {pair} (#{trade_id}) filled "
-                       "for {open_rate}.".format(**msg))
-        elif msg_type == RPCMessageType.SELL_FILL:
-            message = ("\N{LARGE CIRCLE} *{exchange}:* "
-                       "Sell order for {pair} (#{trade_id}) filled "
-                       "for {close_rate}.".format(**msg))
-        elif msg_type == RPCMessageType.SELL:
-            message = self._format_sell_msg(msg)
-
-        elif msg_type == RPCMessageType.STATUS:
-            message = '*Status:* `{status}`'.format(**msg)
-
-        elif msg_type == RPCMessageType.WARNING:
-            message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
-
-        elif msg_type == RPCMessageType.STARTUP:
-            message = '{status}'.format(**msg)
-
-        else:
-            raise NotImplementedError('Unknown message type: {}'.format(msg_type))
 
         self._send_msg(message, disable_notification=(noti == 'silent'))
 

requirements-dev.txt

@@ -23,5 +23,5 @@ nbconvert==6.1.0
 # mypy types
 types-cachetools==4.2.0
 types-filelock==0.1.5
-types-requests==2.25.6
+types-requests==2.25.8
 types-tabulate==0.8.2

requirements-hyperopt.txt

@@ -8,4 +8,4 @@ scikit-optimize==0.8.1
 filelock==3.0.12
 joblib==1.0.1
 psutil==5.8.0
-progressbar2==3.53.2
+progressbar2==3.53.3

requirements.txt

@@ -1,16 +1,17 @@
 numpy==1.21.2
 pandas==1.3.3
+pandas-ta==0.3.14b
 
-ccxt==1.56.30
+ccxt==1.56.86
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.8
 aiohttp==3.7.4.post0
-SQLAlchemy==1.4.23
+SQLAlchemy==1.4.25
 python-telegram-bot==13.7
 arrow==1.1.1
 cachetools==4.2.2
 requests==2.26.0
-urllib3==1.26.6
+urllib3==1.26.7
 wrapt==1.12.1
 jsonschema==3.2.0
 TA-Lib==0.4.21

setup.py

@@ -54,6 +54,7 @@ setup(
         'wrapt',
         'jsonschema',
         'TA-Lib',
+        'pandas-ta',
         'technical',
         'tabulate',
         'pycoingecko',

tests/plugins/test_protections.py

@@ -125,7 +125,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog):
     # Test 5m after lock-period - this should try and relock the pair, but end-time
     # should be the previous end-time
     end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5)
-    assert freqtrade.protections.global_stop(end_time)
+    freqtrade.protections.global_stop(end_time)
     assert not PairLocks.is_global_lock(end_time)
 
 
@@ -182,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair
         min_ago_open=180, min_ago_close=30, profit_rate=0.9,
     ))
 
-    assert freqtrade.protections.stop_per_pair(pair)
+    freqtrade.protections.stop_per_pair(pair)
     assert freqtrade.protections.global_stop() != only_per_pair
     assert PairLocks.is_pair_locked(pair)
     assert PairLocks.is_global_lock() != only_per_pair

tests/rpc/test_rpc_telegram.py

@@ -1313,6 +1313,34 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
             'Reason: cancelled due to timeout.')
 
 
+def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None:
+
+    default_conf['telegram']['notification_settings']['protection_trigger'] = 'on'
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+    time_machine.move_to("2021-09-01 05:00:00 +00:00")
+    lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason')
+    msg = {
+        'type': RPCMessageType.PROTECTION_TRIGGER,
+    }
+    msg.update(lock.to_json())
+    telegram.send_msg(msg)
+    assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. "
+            "`ETH/BTC` will be locked until `2021-09-01 05:10:00`.")
+
+    msg_mock.reset_mock()
+    # Test global protection
+
+    msg = {
+        'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL,
+    }
+    lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason')
+    msg.update(lock.to_json())
+    telegram.send_msg(msg)
+    assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. "
+            "*All pairs* will be locked until `2021-09-01 06:45:00`.")
+
+
 def test_send_msg_buy_fill_notification(default_conf, mocker) -> None:
 
     default_conf['telegram']['notification_settings']['buy_fill'] = 'on'

tests/test_freqtradebot.py

@@ -416,6 +416,29 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order,
     assert log_has_re(message, caplog)
 
 
+def test_handle_protections(mocker, default_conf, fee):
+    default_conf['protections'] = [
+        {"method": "CooldownPeriod", "stop_duration": 60},
+        {
+            "method": "StoplossGuard",
+            "lookback_period_candles": 24,
+            "trade_limit": 4,
+            "stop_duration_candles": 4,
+            "only_per_pair": False
+        }
+    ]
+
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    freqtrade.protections._protection_handlers[1].global_stop = MagicMock(
+        return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf"))
+    create_mock_trades(fee)
+    freqtrade.handle_protections('ETC/BTC')
+    send_msg_mock = freqtrade.rpc.send_msg
+    assert send_msg_mock.call_count == 2
+    assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER
+    assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL
+
+
 def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
     default_conf['dry_run'] = True