From 41f97a73c97fa573c9ae0d61911a246014d032f8 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Fri, 25 Oct 2019 09:50:37 +0300 Subject: [PATCH 01/10] Add logging to syslog and journald --- freqtrade/loggers.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 90b8905e5..a98da25c2 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -1,9 +1,12 @@ import logging import sys -from logging.handlers import RotatingFileHandler +from logging import Formatter +from logging.handlers import RotatingFileHandler, SysLogHandler from typing import Any, Dict, List +from freqtrade import OperationalException + logger = logging.getLogger(__name__) @@ -36,10 +39,38 @@ def setup_logging(config: Dict[str, Any]) -> None: # Log to stdout, not stderr log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stdout)] - if config.get('logfile'): - log_handlers.append(RotatingFileHandler(config['logfile'], - maxBytes=1024 * 1024, # 1Mb - backupCount=10)) + logfile = config.get('logfile') + if logfile: + s = logfile.split(':') + if s[0] == 'syslog': + # Address can be either a string (socket filename) for Unix domain socket or + # a tuple (hostname, port) for UDP socket. + # Address can be omitted (i.e. simple 'syslog' used as the value of + # config['logfilename']), which defaults to '/dev/log', applicable for most + # of the systems. + address = (s[1], int(s[2])) if len(s) > 2 else s[1] if len(s) > 1 else '/dev/log' + handler = SysLogHandler(address=address) + # No datetime field for logging into syslog, to allow syslog + # to perform reduction of repeating messages if this is set in the + # syslog config. The messages should be equal for this. + handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) + log_handlers.append(handler) + elif s[0] == 'journald': + try: + from systemd.journal import JournaldLogHandler + except ImportError: + raise OperationalException("You need the systemd python package be installed in " + "order to use logging to journald.") + handler = JournaldLogHandler() + # No datetime field for logging into journald, to allow syslog + # to perform reduction of repeating messages if this is set in the + # syslog config. The messages should be equal for this. + handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) + log_handlers.append(handler) + else: + log_handlers.append(RotatingFileHandler(logfile, + maxBytes=1024 * 1024, # 1Mb + backupCount=10)) logging.basicConfig( level=logging.INFO if verbosity < 1 else logging.DEBUG, From 3a7553eef63aeffb25ea64d9287dcbb387240fd6 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 26 Oct 2019 12:45:05 +0300 Subject: [PATCH 02/10] Adjust option helpstring --- freqtrade/configuration/cli_options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/configuration/cli_options.py index 697e048db..4c6608c08 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/configuration/cli_options.py @@ -36,7 +36,8 @@ AVAILABLE_CLI_OPTIONS = { ), "logfile": Arg( '--logfile', - help='Log to the file specified.', + help="Log to the file specified. Special values are: 'syslog', 'journald'. " + "See the documentation for more details.", metavar='FILE', ), "version": Arg( From 8b4fea4b710e5384d6df3d94454c6ee141016d11 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sun, 27 Oct 2019 02:06:10 +0300 Subject: [PATCH 03/10] Update installation.md --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index fcbce571e..d45b47b3e 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -194,7 +194,7 @@ freqtrade -c config.json #### 7. (Optional) Post-installation Tasks -On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details. +On Linux, as an optional post-installation task, you may wish to setup the bot to run as a `systemd` service or configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Post-installation Tasks](advanced-setup.md) for details. ------ From a183162d8b7248fde6b8dd91e4c652dd224b8916 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 23 Nov 2019 03:37:29 +0300 Subject: [PATCH 04/10] Add description into Advanced Setup section --- docs/advanced-setup.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index e6334d2c1..c50527c8b 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -31,3 +31,40 @@ as the watchdog. !!! Note The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. + +## Logging to syslog or journald + +On many systems the bot can be configured to send its log messages to `syslog` or `journald`. The special values for the `--logfilename` option can be used for this: + +* `--logfilename journald` -- send log messages to `journald`. This needs the `systemd` python package installed as the dependency. Not available on Windows. + +* `--logfilename syslog:` -- send log messages to `syslog` server using the `` as syslog address. + +The syslog address can be either a Unix domain socket (socket filename) or a UDP socket specification, consisting of IP address and UDP port, separated by the ':' character. + +So, the following are the examples of possible usages: + +* `--logfilename syslog:/dev/log` -- log to syslog (rsyslog) using the `/dev/log` socket, suitable for most systems. +* `--logfilename syslog` -- same as above, the shortcut for `/dev/log`. +* `--logfilename syslog:/var/run/syslog` -- log to syslog (rsyslog) using the `/var/run/syslog` socket. Use this on MacOS. +* `--logfilename syslog:localhost:514` -- log to local syslog using UDP socket, if it listens on port 514. +* `--logfilename syslog::514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to external syslog server. + +Log messages are send to `journald` and `syslog` with the `user` facility. So you can see them with the following commands: + +* `tail -f /var/log/user`, or install a comprehansive graphical viewer (for instance, 'Log File Viewer' for Ubuntu) for the `syslog` case; +* `journalctl -f` when logging to `journald`. + +On many systems rsyslog (syslog) fetches data from journald, so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both journalctl and the syslog viewer utility. + +For rsyslog the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add +``` +if $programname startswith "freqtrade" then -/var/log/freqtrade.log +``` +to one of the rsyslog configuration files, for example at the end of the `/etc/rsyslog.d/50-default.conf`. + +For syslog (rsyslog), the reduction mode can be switched on. This will reduce the number of repeating messages. For instance, multiple bot Heartbeat messages will be reduced to the single message when nothing else happens with the bot. To achieve this, set in `/etc/rsyslog.conf`: +``` +# Filter duplicated messages +$RepeatedMsgReduction on +``` From 31c598f88aac05cc373b755ebe10f2b78125b79a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Nov 2019 14:09:42 +0100 Subject: [PATCH 05/10] Add tests for advanced logging setup --- tests/conftest.py | 2 +- tests/test_configuration.py | 50 ++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 305221d6d..8e2b4e976 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1215,7 +1215,7 @@ def import_fails() -> None: realimport = builtins.__import__ def mockedimport(name, *args, **kwargs): - if name in ["filelock"]: + if name in ["filelock", 'systemd.journal']: raise ImportError(f"No module named '{name}'") return realimport(name, *args, **kwargs) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2aa805fe6..041cb95aa 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -21,7 +21,7 @@ from freqtrade.configuration.directory_operations import (create_datadir, create_userdata_dir) from freqtrade.configuration.load_config import load_config_file from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL -from freqtrade.loggers import _set_loggers +from freqtrade.loggers import _set_loggers, setup_logging from freqtrade.state import RunMode from tests.conftest import (log_has, log_has_re, patched_configuration_load_config_file) @@ -604,6 +604,54 @@ def test_set_loggers() -> None: assert logging.getLogger('telegram').level is logging.INFO +def test_set_loggers_syslog(mocker): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'syslog:/dev/log', + } + + setup_logging(config) + assert len(logger.handlers) == 2 + assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler] + assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +def test_set_loggers_journald(mocker): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + + setup_logging(config) + assert len(logger.handlers) == 2 + assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"] + assert [x for x in logger.handlers if type(x) == logging.StreamHandler] + # reset handlers to not break pytest + logger.handlers = orig_handlers + + +def test_set_loggers_journald_importerror(mocker, import_fails): + logger = logging.getLogger() + orig_handlers = logger.handlers + logger.handlers = [] + + config = {'verbosity': 2, + 'logfile': 'journald', + } + with pytest.raises(OperationalException, + match=r'You need the systemd python package.*'): + setup_logging(config) + logger.handlers = orig_handlers + + def test_set_logfile(default_conf, mocker): patched_configuration_load_config_file(mocker, default_conf) From 1242263d255e8d8da09a99e18262b86d01245864 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Nov 2019 14:20:41 +0100 Subject: [PATCH 06/10] Make test OS dependent --- requirements-dev.txt | 1 + tests/test_configuration.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f5cde59e8..f7a68b664 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,6 +13,7 @@ pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest-mock==1.11.1 pytest-random-order==1.0.4 +systemd==0.16.1; sys_platform != 'win32' # Convert jupyter notebooks to markdown documents nbconvert==5.6.0 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 041cb95aa..9601a5b50 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name import json import logging +import sys import warnings from copy import deepcopy from pathlib import Path @@ -14,9 +15,9 @@ from freqtrade.configuration import (Arguments, Configuration, validate_config_consistency) from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_validation import validate_config_schema -from freqtrade.configuration.deprecated_settings import (check_conflicting_settings, - process_deprecated_setting, - process_temporary_deprecated_settings) +from freqtrade.configuration.deprecated_settings import ( + check_conflicting_settings, process_deprecated_setting, + process_temporary_deprecated_settings) from freqtrade.configuration.directory_operations import (create_datadir, create_userdata_dir) from freqtrade.configuration.load_config import load_config_file @@ -621,6 +622,7 @@ def test_set_loggers_syslog(mocker): logger.handlers = orig_handlers +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") def test_set_loggers_journald(mocker): logger = logging.getLogger() orig_handlers = logger.handlers From c7c7a1c2aaa204b390f464657be0e195ede48a36 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Nov 2019 14:25:32 +0100 Subject: [PATCH 07/10] skip test due to no journald installed --- requirements-dev.txt | 1 - tests/test_configuration.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f7a68b664..f5cde59e8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,6 @@ pytest-asyncio==0.10.0 pytest-cov==2.8.1 pytest-mock==1.11.1 pytest-random-order==1.0.4 -systemd==0.16.1; sys_platform != 'win32' # Convert jupyter notebooks to markdown documents nbconvert==5.6.0 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 9601a5b50..5d05a1e68 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -605,6 +605,7 @@ def test_set_loggers() -> None: assert logging.getLogger('telegram').level is logging.INFO +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") def test_set_loggers_syslog(mocker): logger = logging.getLogger() orig_handlers = logger.handlers @@ -622,7 +623,7 @@ def test_set_loggers_syslog(mocker): logger.handlers = orig_handlers -@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") +@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.") def test_set_loggers_journald(mocker): logger = logging.getLogger() orig_handlers = logger.handlers From b040cbffdd45b6b412bb50268857da3740edede2 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 30 Nov 2019 22:28:48 +0300 Subject: [PATCH 08/10] syslog and journald cases splitted --- docs/advanced-setup.md | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 5890ae6ab..df331a508 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -35,15 +35,17 @@ as the watchdog. !!! Note The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container. -## Logging to syslog or journald +## Advanced Logging -On many systems the bot can be configured to send its log messages to `syslog` or `journald`. The special values for the `--logfilename` option can be used for this: +On many Linux systems the bot can be configured to send its log messages to `syslog` or `journald` system services. Logging to a remote `syslog` server is also available on Windows. The special values for the `--logfilename` command line option can be used for this. -* `--logfilename journald` -- send log messages to `journald`. This needs the `systemd` python package installed as the dependency. Not available on Windows. +### Logging to syslog -* `--logfilename syslog:` -- send log messages to `syslog` server using the `` as syslog address. +To send Freqtrade log messages to a local or remote `syslog` service use the `--logfilename` command line option with the value in the following format: -The syslog address can be either a Unix domain socket (socket filename) or a UDP socket specification, consisting of IP address and UDP port, separated by the ':' character. +* `--logfilename syslog:` -- send log messages to `syslog` service using the `` as the syslog address. + +The syslog address can be either a Unix domain socket (socket filename) or a UDP socket specification, consisting of IP address and UDP port, separated by the `:` character. So, the following are the examples of possible usages: @@ -51,23 +53,37 @@ So, the following are the examples of possible usages: * `--logfilename syslog` -- same as above, the shortcut for `/dev/log`. * `--logfilename syslog:/var/run/syslog` -- log to syslog (rsyslog) using the `/var/run/syslog` socket. Use this on MacOS. * `--logfilename syslog:localhost:514` -- log to local syslog using UDP socket, if it listens on port 514. -* `--logfilename syslog::514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to external syslog server. +* `--logfilename syslog::514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to an external syslog server. -Log messages are send to `journald` and `syslog` with the `user` facility. So you can see them with the following commands: +Log messages are send to `syslog` with the `user` facility. So you can see them with the following commands: -* `tail -f /var/log/user`, or install a comprehansive graphical viewer (for instance, 'Log File Viewer' for Ubuntu) for the `syslog` case; -* `journalctl -f` when logging to `journald`. +* `tail -f /var/log/user`, or +* install a comprehensive graphical viewer (for instance, 'Log File Viewer' for Ubuntu). -On many systems rsyslog (syslog) fetches data from journald, so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both journalctl and the syslog viewer utility. +On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. -For rsyslog the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add +For `rsyslog` the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add ``` if $programname startswith "freqtrade" then -/var/log/freqtrade.log ``` to one of the rsyslog configuration files, for example at the end of the `/etc/rsyslog.d/50-default.conf`. -For syslog (rsyslog), the reduction mode can be switched on. This will reduce the number of repeating messages. For instance, multiple bot Heartbeat messages will be reduced to the single message when nothing else happens with the bot. To achieve this, set in `/etc/rsyslog.conf`: +For `syslog` (`rsyslog`), the reduction mode can be switched on. This will reduce the number of repeating messages. For instance, multiple bot Heartbeat messages will be reduced to a single message when nothing else happens with the bot. To achieve this, set in `/etc/rsyslog.conf`: ``` # Filter duplicated messages $RepeatedMsgReduction on ``` + +## Logging to journald + +This needs the `systemd` python package installed as the dependency, which is not available on Windows. Hence, the whole journald logging functionality is not available for a bot running on Windows. + +To send Freqtrade log messages to `journald` system service use the `--logfilename` command line option with the value in the following format: + +* `--logfilename journald` -- send log messages to `journald`. + +Log messages are send to `journald` with the `user` facility. So you can see them with the following commands: + +* `journalctl -f` -- shows Freqtrade log messages sent to `journald` along with other log messages fetched by `journald`. There are many options to filter the messages, see manual pages for the `journalctl` utility. + +On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. From eafccb445cc424918ff42e0ebb6d20c7ebc4d6d3 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 30 Nov 2019 22:32:12 +0300 Subject: [PATCH 09/10] Add command sample for journalctl with -u --- docs/advanced-setup.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index df331a508..b1a3a66ae 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -84,6 +84,9 @@ To send Freqtrade log messages to `journald` system service use the `--logfilena Log messages are send to `journald` with the `user` facility. So you can see them with the following commands: -* `journalctl -f` -- shows Freqtrade log messages sent to `journald` along with other log messages fetched by `journald`. There are many options to filter the messages, see manual pages for the `journalctl` utility. +* `journalctl -f` -- shows Freqtrade log messages sent to `journald` along with other log messages fetched by `journald`. +* `journalctl -f -u freqtrade.service` -- this command can be used when the bot is run as a `systemd` service. + +There are many other options in the `journalctl` utility to filter the messages, see manual pages for this utility. On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. From 7a3c3c4ddf32ccc3d4cc0b4f7294f31b09939e04 Mon Sep 17 00:00:00 2001 From: hroff-1902 <47309513+hroff-1902@users.noreply.github.com> Date: Sat, 30 Nov 2019 22:35:13 +0300 Subject: [PATCH 10/10] Add directlink to the section --- docs/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.md b/docs/installation.md index 15094232d..27b7a94c5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -201,7 +201,7 @@ freqtrade trade -c config.json #### 7. (Optional) Post-installation Tasks -On Linux, as an optional post-installation task, you may wish to setup the bot to run as a `systemd` service or configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Post-installation Tasks](advanced-setup.md) for details. +On Linux, as an optional post-installation task, you may wish to setup the bot to run as a `systemd` service or configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Logging](advanced-setup.md#advanced-logging) for details. ------