From c17595b314fb1e6dcfff38695c51e5c07f2b38fe Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sat, 28 Aug 2021 19:39:50 +0200 Subject: [PATCH 01/19] Docs: Mention Performance Warning for strategies Related to #5408 --- docs/strategy-customization.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index cfea60d22..194517b2b 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -781,6 +781,8 @@ Printing more than a few rows is also possible (simply use `print(dataframe)` i ## Common mistakes when developing strategies +### Peeking into the future while backtesting + Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions. @@ -791,6 +793,33 @@ The following lists some common patterns which should be avoided to prevent frus - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling().mean()` instead - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead. +### Performance warning + +When executing a strategy, one can sometimes be greeted by the following in the logs + +> PerformanceWarning: DataFrame is highly fragmented. + +This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: + use `pd.concat(axis=1)`. For example + +```python +for i in range(100): + dataframe[i] = ta.indicator(dataframe, param=i) +``` + +should be rewritten to + +```python +frames = [dataframe] +for i in range(100): + frames.append({ + str(i): ta.indicator(dataframe, param=i) + }) + +# Append columns to existing dataframe +merged_frame = pd.concat(frames, axis=1) +``` + ## Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. From 879bf47b32ff692e35f2509665b1ef9899eb3621 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Sep 2021 19:25:36 +0200 Subject: [PATCH 02/19] Refactor telegram.py to simplify send_msg --- freqtrade/rpc/telegram.py | 51 +++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 19c58b63d..898446ea0 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -260,29 +260,7 @@ class Telegram(RPCHandler): return message - def send_msg(self, msg: Dict[str, Any]) -> None: - """ Send a message to telegram channel """ - - default_noti = 'on' - - msg_type = msg['type'] - noti = '' - if msg_type == RPCMessageType.SELL: - sell_noti = self._config['telegram'] \ - .get('notification_settings', {}).get(str(msg_type), {}) - # For backward compatibility sell still can be string - if isinstance(sell_noti, str): - noti = sell_noti - else: - noti = sell_noti.get(str(msg['sell_reason']), default_noti) - else: - noti = self._config['telegram'] \ - .get('notification_settings', {}).get(str(msg_type), default_noti) - - if noti == 'off': - logger.info(f"Notification '{msg_type}' not sent.") - # Notification disabled - return + def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str: if msg_type == RPCMessageType.BUY: message = self._format_buy_msg(msg) @@ -315,6 +293,33 @@ class Telegram(RPCHandler): else: raise NotImplementedError('Unknown message type: {}'.format(msg_type)) + return message + + def send_msg(self, msg: Dict[str, Any]) -> None: + """ Send a message to telegram channel """ + + default_noti = 'on' + + msg_type = msg['type'] + noti = '' + if msg_type == RPCMessageType.SELL: + sell_noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), {}) + # For backward compatibility sell still can be string + if isinstance(sell_noti, str): + noti = sell_noti + else: + noti = sell_noti.get(str(msg['sell_reason']), default_noti) + else: + noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), default_noti) + + if noti == 'off': + logger.info(f"Notification '{msg_type}' not sent.") + # Notification disabled + return + + message = self.compose_message(msg, msg_type) self._send_msg(message, disable_notification=(noti == 'silent')) From 1da091dea3006b67e2400cf1390981354626f4fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 19 Sep 2021 19:41:19 +0200 Subject: [PATCH 03/19] ProtectionManager should return the lock just created --- freqtrade/persistence/pairlock_middleware.py | 4 +++- freqtrade/plugins/protectionmanager.py | 23 ++++++++++---------- tests/plugins/test_protections.py | 4 ++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index af904f693..8662fc36d 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -30,7 +30,8 @@ class PairLocks(): PairLocks.locks = [] @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + def lock_pair(pair: str, until: datetime, reason: str = None, *, + now: datetime = None) -> PairLock: """ Create PairLock from now to "until". Uses database by default, unless PairLocks.use_db is set to False, @@ -52,6 +53,7 @@ class PairLocks(): PairLock.query.session.commit() else: PairLocks.locks.append(lock) + return lock @staticmethod def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py index f33e5b4bc..2510d6fee 100644 --- a/freqtrade/plugins/protectionmanager.py +++ b/freqtrade/plugins/protectionmanager.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from typing import Dict, List, Optional from freqtrade.persistence import PairLocks +from freqtrade.persistence.models import PairLock from freqtrade.plugins.protections import IProtection from freqtrade.resolvers import ProtectionResolver @@ -43,30 +44,28 @@ class ProtectionManager(): """ return [{p.name: p.short_desc()} for p in self._protection_handlers] - def global_stop(self, now: Optional[datetime] = None) -> bool: + def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_global_stop: - result, until, reason = protection_handler.global_stop(now) + lock, until, reason = protection_handler.global_stop(now) # Early stopping - first positive result blocks further trades - if result and until: + if lock and until: if not PairLocks.is_global_lock(until): - PairLocks.lock_pair('*', until, reason, now=now) - result = True + result = PairLocks.lock_pair('*', until, reason, now=now) return result - def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]: if not now: now = datetime.now(timezone.utc) - result = False + result = None for protection_handler in self._protection_handlers: if protection_handler.has_local_stop: - result, until, reason = protection_handler.stop_per_pair(pair, now) - if result and until: + lock, until, reason = protection_handler.stop_per_pair(pair, now) + if lock and until: if not PairLocks.is_pair_locked(pair, until): - PairLocks.lock_pair(pair, until, reason, now=now) - result = True + result = PairLocks.lock_pair(pair, until, reason, now=now) return result diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index c0a9ae72a..a3cb29c9d 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -125,7 +125,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): # Test 5m after lock-period - this should try and relock the pair, but end-time # should be the previous end-time end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) - assert freqtrade.protections.global_stop(end_time) + freqtrade.protections.global_stop(end_time) assert not PairLocks.is_global_lock(end_time) @@ -182,7 +182,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair min_ago_open=180, min_ago_close=30, profit_rate=0.9, )) - assert freqtrade.protections.stop_per_pair(pair) + freqtrade.protections.stop_per_pair(pair) assert freqtrade.protections.global_stop() != only_per_pair assert PairLocks.is_pair_locked(pair) assert PairLocks.is_global_lock() != only_per_pair From a0fb43c6caca7d0b0f06cf9814d43f69341339b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 19:12:59 +0200 Subject: [PATCH 04/19] Add pairlock-notification --- freqtrade/enums/rpcmessagetype.py | 1 + freqtrade/freqtradebot.py | 16 ++++++++++++++-- freqtrade/rpc/telegram.py | 5 +++++ tests/rpc/test_rpc_telegram.py | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 9c59f6108..889d67cf4 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -11,6 +11,7 @@ class RPCMessageType(Enum): SELL = 'sell' SELL_FILL = 'sell_fill' SELL_CANCEL = 'sell_cancel' + PROTECTION_TRIGGER = 'protection_trigger' def __repr__(self): return self.value diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1cb8988ff..451b5764a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1292,8 +1292,7 @@ class FreqtradeBot(LoggingMixin): if not trade.is_open: if not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) - self.protections.stop_per_pair(trade.pair) - self.protections.global_stop() + self.handle_protections(trade.pair) self.wallets.update() elif not trade.open_order_id: # Buy fill @@ -1301,6 +1300,19 @@ class FreqtradeBot(LoggingMixin): return False + def handle_protections(self, pair: str) -> None: + prot_trig = self.protections.stop_per_pair(pair) + if prot_trig: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg.update(prot_trig.to_json()) + self.rpc.send_msg(msg) + + prot_trig_glb = self.protections.global_stop() + if prot_trig_glb: + msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg.update(prot_trig_glb.to_json()) + self.rpc.send_msg(msg) + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, amount: float, fee_abs: float) -> float: """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 898446ea0..934a5182a 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -281,6 +281,11 @@ class Telegram(RPCHandler): "for {close_rate}.".format(**msg)) elif msg_type == RPCMessageType.SELL: message = self._format_sell_msg(msg) + elif msg_type == RPCMessageType.PROTECTION_TRIGGER: + message = ( + "*Protection* triggered due to {reason}. " + "{pair} will be locked until {lock_end_time}." + ).format(**msg) elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 21f1cd000..0675e4d5f 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1313,6 +1313,20 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: 'Reason: cancelled due to timeout.') +def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None: + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + time_machine.move_to("2021-09-01 05:00:00 +00:00") + lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER, + } + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "ETH/BTC will be locked until 2021-09-01 05:10:00.") + + def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: default_conf['telegram']['notification_settings']['buy_fill'] = 'on' From dd0db7ee5db04fb98b542e9541d46768b9bb2c05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 19:23:40 +0200 Subject: [PATCH 05/19] Split protection-notification into global and per-pair --- config_examples/config_full.example.json | 4 +++- docs/telegram-usage.md | 4 +++- freqtrade/enums/rpcmessagetype.py | 1 + freqtrade/freqtradebot.py | 2 +- freqtrade/rpc/telegram.py | 6 +++++- tests/rpc/test_rpc_telegram.py | 13 +++++++++++++ 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index d0f3f0df6..c415d70b0 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -149,7 +149,9 @@ }, "sell_fill": "on", "buy_cancel": "on", - "sell_cancel": "on" + "sell_cancel": "on", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index b020b00db..1e6fa9dae 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -93,7 +93,9 @@ Example configuration showing the different settings: "buy_cancel": "silent", "sell_cancel": "on", "buy_fill": "off", - "sell_fill": "off" + "sell_fill": "off", + "protection_trigger": "off", + "protection_trigger_global": "on" }, "reload": true, "balance_dust_level": 0.01 diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py index 889d67cf4..4e3f693e5 100644 --- a/freqtrade/enums/rpcmessagetype.py +++ b/freqtrade/enums/rpcmessagetype.py @@ -12,6 +12,7 @@ class RPCMessageType(Enum): SELL_FILL = 'sell_fill' SELL_CANCEL = 'sell_cancel' PROTECTION_TRIGGER = 'protection_trigger' + PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global' def __repr__(self): return self.value diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 451b5764a..37be3173a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1309,7 +1309,7 @@ class FreqtradeBot(LoggingMixin): prot_trig_glb = self.protections.global_stop() if prot_trig_glb: - msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } + msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, } msg.update(prot_trig_glb.to_json()) self.rpc.send_msg(msg) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 934a5182a..0687f95a2 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -286,7 +286,11 @@ class Telegram(RPCHandler): "*Protection* triggered due to {reason}. " "{pair} will be locked until {lock_end_time}." ).format(**msg) - + elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: + message = ( + "*Protection* triggered due to {reason}. " + "All pairs will be locked until {lock_end_time}." + ).format(**msg) elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 0675e4d5f..d37a74187 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1326,6 +1326,19 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " "ETH/BTC will be locked until 2021-09-01 05:10:00.") + msg_mock.reset_mock() + # Test global protection + + msg = { + 'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, + } + lock = PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=100).datetime, 'randreason') + msg.update(lock.to_json()) + telegram.send_msg(msg) + assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " + "All pairs will be locked until 2021-09-01 06:45:00.") + + def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: From fd23ab3d647db7444d12129755ddf001e014dbcd Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 19:49:18 +0200 Subject: [PATCH 06/19] improve formatting, add tests --- freqtrade/rpc/telegram.py | 4 ++-- tests/rpc/test_rpc_telegram.py | 5 ++--- tests/test_freqtradebot.py | 23 +++++++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0687f95a2..059ba9c41 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -284,12 +284,12 @@ class Telegram(RPCHandler): elif msg_type == RPCMessageType.PROTECTION_TRIGGER: message = ( "*Protection* triggered due to {reason}. " - "{pair} will be locked until {lock_end_time}." + "`{pair}` will be locked until `{lock_end_time}`." ).format(**msg) elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL: message = ( "*Protection* triggered due to {reason}. " - "All pairs will be locked until {lock_end_time}." + "*All pairs* will be locked until `{lock_end_time}`." ).format(**msg) elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index d37a74187..8c285a76e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1324,7 +1324,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> msg.update(lock.to_json()) telegram.send_msg(msg) assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " - "ETH/BTC will be locked until 2021-09-01 05:10:00.") + "`ETH/BTC` will be locked until `2021-09-01 05:10:00`.") msg_mock.reset_mock() # Test global protection @@ -1336,8 +1336,7 @@ def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> msg.update(lock.to_json()) telegram.send_msg(msg) assert (msg_mock.call_args[0][0] == "*Protection* triggered due to randreason. " - "All pairs will be locked until 2021-09-01 06:45:00.") - + "*All pairs* will be locked until `2021-09-01 06:45:00`.") def test_send_msg_buy_fill_notification(default_conf, mocker) -> None: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index eb3c77cc7..e1df6dad9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -526,6 +526,29 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, assert log_has_re(message, caplog) +def test_handle_protections(mocker, default_conf, fee): + default_conf['protections'] = [ + {"method": "CooldownPeriod", "stop_duration": 60}, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": False + } + ] + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.protections._protection_handlers[1].global_stop = MagicMock( + return_value=(True, arrow.utcnow().shift(hours=1).datetime, "asdf")) + create_mock_trades(fee) + freqtrade.handle_protections('ETC/BTC') + send_msg_mock = freqtrade.rpc.send_msg + assert send_msg_mock.call_count == 2 + assert send_msg_mock.call_args_list[0][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER + assert send_msg_mock.call_args_list[1][0][0]['type'] == RPCMessageType.PROTECTION_TRIGGER_GLOBAL + + def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True From 3ce05c0d548119fd5618ba2700313a69ec10f3c5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 20 Sep 2021 20:08:48 +0200 Subject: [PATCH 07/19] Add "sane" defaults to protection triggers --- docs/telegram-usage.md | 1 + freqtrade/constants.py | 9 +++++++++ tests/rpc/test_rpc_telegram.py | 2 ++ 3 files changed, 12 insertions(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 1e6fa9dae..b9d01a236 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -105,6 +105,7 @@ Example configuration showing the different settings: `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. `sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange. `*_fill` notifications are off by default and must be explicitly enabled. +`protection_trigger` notifications are sent when a protection triggers and `protection_trigger_global` notifications trigger when global protections are triggered. `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 9ca43d459..4997108bc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -284,6 +284,15 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' }, + 'protection_trigger': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, + 'protection_trigger_global': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + }, } }, 'reload': {'type': 'boolean'}, diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 8c285a76e..7dde7b803 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1315,6 +1315,8 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: def test_send_msg_protection_notification(default_conf, mocker, time_machine) -> None: + default_conf['telegram']['notification_settings']['protection_trigger'] = 'on' + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) time_machine.move_to("2021-09-01 05:00:00 +00:00") lock = PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=6).datetime, 'randreason') From b0de4d333e86bfe23f065ab365e2ab18bb7ca41d Mon Sep 17 00:00:00 2001 From: Peter Willemsen Date: Tue, 21 Sep 2021 23:20:40 +0200 Subject: [PATCH 08/19] fixed webhook error --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1cb8988ff..95f2280ac 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1217,7 +1217,7 @@ class FreqtradeBot(LoggingMixin): 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, - 'limit': profit_rate, + 'limit': profit_rate or 0, 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, @@ -1226,7 +1226,7 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date, + 'close_date': trade.close_date or datetime.utcnow(), 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, From d7903f012f7af5f3c19df527780456bab0ca986f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Sep 2021 07:25:11 +0200 Subject: [PATCH 09/19] Move PerformanceWarning to advanced section rewrite to use strategy parameters instead of plain range --- docs/strategy-advanced.md | 30 ++++++++++++++++++++++++++++++ docs/strategy-customization.md | 27 --------------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 4409af6ea..d7d0dd04a 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -695,3 +695,33 @@ The variable 'content', will contain the strategy file in a BASE64 encoded form. ``` Please ensure that 'NameOfStrategy' is identical to the strategy name! + +## Performance warning + +When executing a strategy, one can sometimes be greeted by the following in the logs + +> PerformanceWarning: DataFrame is highly fragmented. + +This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: +use `pd.concat(axis=1)`. +This can have slight performance implications, which are usually only visible during hyperopt (when optimizing an indicator). + +For example: + +```python +for val in self.buy_ema_short.range: + dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val) +``` + +should be rewritten to + +```python +frames = [dataframe] +for val in self.buy_ema_short.range: + frames.append({ + f'ema_short_{val}': ta.EMA(dataframe, timeperiod=val) + }) + +# Append columns to existing dataframe +merged_frame = pd.concat(frames, axis=1) +``` diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 194517b2b..6d6eff2aa 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -793,33 +793,6 @@ The following lists some common patterns which should be avoided to prevent frus - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling().mean()` instead - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead. -### Performance warning - -When executing a strategy, one can sometimes be greeted by the following in the logs - -> PerformanceWarning: DataFrame is highly fragmented. - -This is a warning from [`pandas`](https://github.com/pandas-dev/pandas) and as the warning continues to say: - use `pd.concat(axis=1)`. For example - -```python -for i in range(100): - dataframe[i] = ta.indicator(dataframe, param=i) -``` - -should be rewritten to - -```python -frames = [dataframe] -for i in range(100): - frames.append({ - str(i): ta.indicator(dataframe, param=i) - }) - -# Append columns to existing dataframe -merged_frame = pd.concat(frames, axis=1) -``` - ## Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. From 692e91a26dfa2ec1c746923d481421cc87a15b8c Mon Sep 17 00:00:00 2001 From: Peter Willemsen Date: Thu, 23 Sep 2021 10:28:15 +0200 Subject: [PATCH 10/19] changed close date from datetime.utcnow() to datetime.now(timezone.utc) --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 95f2280ac..9cfd219a8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1226,7 +1226,7 @@ class FreqtradeBot(LoggingMixin): 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date or datetime.utcnow(), + 'close_date': trade.close_date or datetime.now(timezone.utc), 'stake_currency': self.config['stake_currency'], 'fiat_currency': self.config.get('fiat_display_currency', None), 'reason': reason, From 422d5601890f9749ab7f69a903d2de3f68ea9257 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:05:55 +0000 Subject: [PATCH 11/19] Bump types-requests from 2.25.6 to 2.25.8 Bumps [types-requests](https://github.com/python/typeshed) from 2.25.6 to 2.25.8. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4859e1cc6..d8d8ce916 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,5 +23,5 @@ nbconvert==6.1.0 # mypy types types-cachetools==4.2.0 types-filelock==0.1.5 -types-requests==2.25.6 +types-requests==2.25.8 types-tabulate==0.8.2 From e85dc6326302e83da897b06941ca725f6d5f242c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:01 +0000 Subject: [PATCH 12/19] Bump mkdocs-material from 7.2.6 to 7.3.0 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.2.6 to 7.3.0. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.2.6...7.3.0) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9927740c2..9b7c12a43 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.6 +mkdocs-material==7.3.0 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 766ef90b56cf190b8beeba1a249d9b53792fbba2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:16 +0000 Subject: [PATCH 13/19] Bump sqlalchemy from 1.4.23 to 1.4.25 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.23 to 1.4.25. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aa729dd9f..74ce391ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ccxt==1.56.30 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.23 +SQLAlchemy==1.4.25 python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 From 90d5af9a35211a51756603cfb3ffcb3cf67edc17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:20 +0000 Subject: [PATCH 14/19] Bump urllib3 from 1.26.6 to 1.26.7 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.6 to 1.26.7. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/1.26.7/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.6...1.26.7) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aa729dd9f..2f6a4a104 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 requests==2.26.0 -urllib3==1.26.6 +urllib3==1.26.7 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.21 From 0353f070f9714655e1ccb06448ffa03dbe3f5fb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 05:06:24 +0000 Subject: [PATCH 15/19] Bump progressbar2 from 3.53.2 to 3.53.3 Bumps [progressbar2](https://github.com/WoLpH/python-progressbar) from 3.53.2 to 3.53.3. - [Release notes](https://github.com/WoLpH/python-progressbar/releases) - [Changelog](https://github.com/WoLpH/python-progressbar/blob/develop/CHANGES.rst) - [Commits](https://github.com/WoLpH/python-progressbar/compare/v3.53.2...v3.53.3) --- updated-dependencies: - dependency-name: progressbar2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 7dc55a9fc..9feec80f1 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -8,4 +8,4 @@ scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.1 psutil==5.8.0 -progressbar2==3.53.2 +progressbar2==3.53.3 From 954c468191b3c130c62d7bedfde1ff138f427908 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 07:12:38 +0200 Subject: [PATCH 16/19] Add pandas-ta to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index aa729dd9f..f0da0cdb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ numpy==1.21.2 pandas==1.3.3 +pandas-ta==0.3.14b ccxt==1.56.30 # Pin cryptography for now due to rust build errors with piwheels From 72a1e27fc6cc0454dfb3035f4957820dd6c6423d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 07:31:20 +0000 Subject: [PATCH 17/19] Bump ccxt from 1.56.30 to 1.56.86 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.56.30 to 1.56.86. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.56.30...1.56.86) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b004dfaa..ede0c74e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.56.30 +ccxt==1.56.86 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From b59906b117f300aefd828fa02c8b5c5658388df1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 19:24:33 +0200 Subject: [PATCH 18/19] Update minimum for tradable_balance_ratio to 0.0 --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 4997108bc..fca319a0f 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -110,7 +110,7 @@ CONF_SCHEMA = { }, 'tradable_balance_ratio': { 'type': 'number', - 'minimum': 0.1, + 'minimum': 0.0, 'maximum': 1, 'default': 0.99 }, From 4c268847d42043d53fc59751ffd1717125d9e0bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 24 Sep 2021 19:32:22 +0200 Subject: [PATCH 19/19] Add pandas-ta to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 727c40c7c..cf381bdd3 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ setup( 'wrapt', 'jsonschema', 'TA-Lib', + 'pandas-ta', 'technical', 'tabulate', 'pycoingecko',