From fb6ae174b97536a589fd857b68699e0518c4662f Mon Sep 17 00:00:00 2001 From: Spat Date: Sun, 28 Nov 2021 11:42:57 +1100 Subject: [PATCH 1/5] Added raw config and retry config to webhook --- docs/webhook-config.md | 33 ++++++++++++++++++++++++-- freqtrade/rpc/webhook.py | 44 +++++++++++++++++++++++++---------- tests/rpc/test_rpc_webhook.py | 11 +++++++++ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/webhook-config.md b/docs/webhook-config.md index ec944cb50..bea555385 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -50,7 +50,7 @@ Sample configuration (tested using IFTTT). The url in `webhook.url` should point to the correct url for your webhook. If you're using [IFTTT](https://ifttt.com) (as shown in the sample above) please insert your event and key to the url. -You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use `"format": "form"` or `"format": "json"` respectively. Example configuration for Mattermost Cloud integration: +You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw data. Use `"format": "form"`, `"format": "json"`, or `"format": "raw"` respectively. Example configuration for Mattermost Cloud integration: ```json "webhook": { @@ -63,7 +63,36 @@ You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use }, ``` -The result would be POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. +The result would be a POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. + +When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be ouput in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example: + +```json + "webhook": { + "enabled": true, + "url": "https://", + "format": "raw", + "webhookstatus": { + "data": "Status: {status}" + } + }, +``` + +The result would be a POST request with e.g. `Status: running` body and `Content-Type: text/plain` header. + +Optional parameters are available to enable automatic retries for webhook messages. The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook. Example configuration for retries: + +```json + "webhook": { + "enabled": true, + "url": "https://", + "retries": 3, + "retry_delay": 0.2, + "webhookstatus": { + "status": "Status: {status}" + } + }, +``` Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called. diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index b4c55649e..99077948e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -2,6 +2,7 @@ This module manages webhook communication """ import logging +import time from typing import Any, Dict from requests import RequestException, post @@ -28,12 +29,16 @@ class Webhook(RPCHandler): super().__init__(rpc, config) self._url = self._config['webhook']['url'] - self._format = self._config['webhook'].get('format', 'form') + self._retries = self._config['webhook'].get('retries', 0) + self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) - if self._format != 'form' and self._format != 'json': + if self._retries < 0: self._retries = 0 + if self._retry_delay < 0: self._retry_delay = 0 + + if not (self._format in ['form', 'json', 'raw']): raise NotImplementedError('Unknown webhook format `{}`, possible values are ' - '`form` (default) and `json`'.format(self._format)) + '`form` (default), `json`, and `raw`'.format(self._format)) def cleanup(self) -> None: """ @@ -77,13 +82,28 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - try: - if self._format == 'form': - post(self._url, data=payload) - elif self._format == 'json': - post(self._url, json=payload) - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) + success = False + attempts = 0 + while not success and attempts <= self._retries: + if attempts: + if self._retry_delay: time.sleep(self._retry_delay) + logger.info("Retrying webhook...") - except RequestException as exc: - logger.warning("Could not call webhook url. Exception: %s", exc) + attempts += 1 + + try: + if self._format == 'form': + response = post(self._url, data=payload) + elif self._format == 'json': + response = post(self._url, json=payload) + elif self._format == 'raw': + response = post(self._url, data=payload['data'], headers={'Content-Type': 'text/plain'}) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + + """throw a RequestException if the post was not successful""" + response.raise_for_status() + success = True + + except RequestException as exc: + logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 04e63a3be..735d2ada2 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -292,3 +292,14 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog): webhook._send_msg(msg) assert post.call_args[1] == {'json': msg} + +def test__send_msg_with_raw_format(default_conf, mocker, caplog): + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["format"] = "raw" + webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + msg = {'data': 'Hello'} + post = MagicMock() + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + + assert post.call_args[1] == {'data': msg['data'], 'headers': {'Content-Type': 'text/plain'}} From 0fa5bf54cd92c08a14e7ad2a194f7faa3f30f0ec Mon Sep 17 00:00:00 2001 From: Spat Date: Mon, 29 Nov 2021 10:30:41 +1100 Subject: [PATCH 2/5] Changed comment --- freqtrade/rpc/webhook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 99077948e..f76d50b0e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -101,7 +101,7 @@ class Webhook(RPCHandler): else: raise NotImplementedError('Unknown format: {}'.format(self._format)) - """throw a RequestException if the post was not successful""" + # Throw a RequestException if the post was not successful response.raise_for_status() success = True From 29180a1d2b2d1b02d99d46bcf6904aaa5a62bee5 Mon Sep 17 00:00:00 2001 From: Spat Date: Mon, 29 Nov 2021 10:48:35 +1100 Subject: [PATCH 3/5] Moved retry config to constants --- freqtrade/constants.py | 2 ++ freqtrade/rpc/webhook.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e775e39fc..51ded6c49 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -312,6 +312,8 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'enabled': {'type': 'boolean'}, + 'retries': {'type': 'integer', 'minimum': 0}, + 'retry_delay': {'type': 'number', 'minimum': 0}, 'webhookbuy': {'type': 'object'}, 'webhookbuycancel': {'type': 'object'}, 'webhooksell': {'type': 'object'}, diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index f76d50b0e..1973f212e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -33,9 +33,6 @@ class Webhook(RPCHandler): self._retries = self._config['webhook'].get('retries', 0) self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) - if self._retries < 0: self._retries = 0 - if self._retry_delay < 0: self._retry_delay = 0 - if not (self._format in ['form', 'json', 'raw']): raise NotImplementedError('Unknown webhook format `{}`, possible values are ' '`form` (default), `json`, and `raw`'.format(self._format)) From 018407852a1cca078e0cbdb6c5b31f1f5eb914e2 Mon Sep 17 00:00:00 2001 From: Spat Date: Mon, 29 Nov 2021 18:17:59 +1100 Subject: [PATCH 4/5] Added missing webhook config params to constants --- freqtrade/constants.py | 6 ++++++ freqtrade/rpc/webhook.py | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 51ded6c49..e074718ca 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -50,6 +50,8 @@ USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] +WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw'] + ENV_VAR_PREFIX = 'FREQTRADE__' NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') @@ -312,12 +314,16 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'enabled': {'type': 'boolean'}, + 'url': {'type': 'string'}, + 'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'}, 'retries': {'type': 'integer', 'minimum': 0}, 'retry_delay': {'type': 'number', 'minimum': 0}, 'webhookbuy': {'type': 'object'}, 'webhookbuycancel': {'type': 'object'}, + 'webhookbuyfill': {'type': 'object'}, 'webhooksell': {'type': 'object'}, 'webhooksellcancel': {'type': 'object'}, + 'webhooksellfill': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, }, }, diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 1973f212e..2a848787d 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -33,10 +33,6 @@ class Webhook(RPCHandler): self._retries = self._config['webhook'].get('retries', 0) self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) - if not (self._format in ['form', 'json', 'raw']): - raise NotImplementedError('Unknown webhook format `{}`, possible values are ' - '`form` (default), `json`, and `raw`'.format(self._format)) - def cleanup(self) -> None: """ Cleanup pending module resources. From dfb148f8d7cdbe6f3a5f8ca345f54f74943538dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 29 Nov 2021 19:54:54 +0100 Subject: [PATCH 5/5] Fix formatting --- freqtrade/rpc/webhook.py | 8 +++++--- tests/rpc/test_rpc_webhook.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 2a848787d..58b75769e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -79,7 +79,8 @@ class Webhook(RPCHandler): attempts = 0 while not success and attempts <= self._retries: if attempts: - if self._retry_delay: time.sleep(self._retry_delay) + if self._retry_delay: + time.sleep(self._retry_delay) logger.info("Retrying webhook...") attempts += 1 @@ -90,10 +91,11 @@ class Webhook(RPCHandler): elif self._format == 'json': response = post(self._url, json=payload) elif self._format == 'raw': - response = post(self._url, data=payload['data'], headers={'Content-Type': 'text/plain'}) + response = post(self._url, data=payload['data'], + headers={'Content-Type': 'text/plain'}) else: raise NotImplementedError('Unknown format: {}'.format(self._format)) - + # Throw a RequestException if the post was not successful response.raise_for_status() success = True diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 735d2ada2..17d1baca9 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -293,6 +293,7 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog): assert post.call_args[1] == {'json': msg} + def test__send_msg_with_raw_format(default_conf, mocker, caplog): default_conf["webhook"] = get_webhook_dict() default_conf["webhook"]["format"] = "raw"