Merge pull request #2418 from hroff-1902/logging-syslog

Add logging to syslog and journald

docs/advanced-setup.md

@@ -34,3 +34,59 @@ as the watchdog.
 
 !!! Note
     The sd_notify communication between the bot and the systemd service manager will not work if the bot runs in a Docker container.
+
+## Advanced Logging
+
+On many Linux systems the bot can be configured to send its log messages to `syslog` or `journald` system services. Logging to a remote `syslog` server is also available on Windows. The special values for the `--logfilename` command line option can be used for this.
+
+### Logging to syslog
+
+To send Freqtrade log messages to a local or remote `syslog` service use the `--logfilename` command line option with the value in the following format:
+
+* `--logfilename syslog:<syslog_address>` -- send log messages to `syslog` service using the `<syslog_address>` as the syslog address.
+
+The syslog address can be either a Unix domain socket (socket filename) or a UDP socket specification, consisting of IP address and UDP port, separated by the `:` character.
+
+So, the following are the examples of possible usages:
+
+* `--logfilename syslog:/dev/log` -- log to syslog (rsyslog) using the `/dev/log` socket, suitable for most systems.
+* `--logfilename syslog` -- same as above, the shortcut for `/dev/log`.
+* `--logfilename syslog:/var/run/syslog` -- log to syslog (rsyslog) using the `/var/run/syslog` socket. Use this on MacOS.
+* `--logfilename syslog:localhost:514` -- log to local syslog using UDP socket, if it listens on port 514.
+* `--logfilename syslog:<ip>:514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to an external syslog server.
+
+Log messages are send to `syslog` with the `user` facility. So you can see them with the following commands:
+
+* `tail -f /var/log/user`, or 
+* install a comprehensive graphical viewer (for instance, 'Log File Viewer' for Ubuntu).
+
+On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better.
+
+For `rsyslog` the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add
+```
+if $programname startswith "freqtrade" then -/var/log/freqtrade.log
+```
+to one of the rsyslog configuration files, for example at the end of the `/etc/rsyslog.d/50-default.conf`.
+
+For `syslog` (`rsyslog`), the reduction mode can be switched on. This will reduce the number of repeating messages. For instance, multiple bot Heartbeat messages will be reduced to a single message when nothing else happens with the bot. To achieve this, set in `/etc/rsyslog.conf`:
+```
+# Filter duplicated messages
+$RepeatedMsgReduction on
+```
+
+## Logging to journald
+
+This needs the `systemd` python package installed as the dependency, which is not available on Windows. Hence, the whole journald logging functionality is not available for a bot running on Windows.
+
+To send Freqtrade log messages to `journald` system service use the `--logfilename` command line option with the value in the following format:
+
+* `--logfilename journald` -- send log messages to `journald`.
+
+Log messages are send to `journald` with the `user` facility. So you can see them with the following commands:
+
+* `journalctl -f` -- shows Freqtrade log messages sent to `journald` along with other log messages fetched by `journald`.
+* `journalctl -f -u freqtrade.service` -- this command can be used when the bot is run as a `systemd` service.
+
+There are many other options in the `journalctl` utility to filter the messages, see manual pages for this utility.
+
+On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better.

docs/installation.md

@@ -201,7 +201,7 @@ freqtrade trade -c config.json
 
 #### 7. (Optional) Post-installation Tasks
 
-On Linux, as an optional post-installation task, you can setup the bot to run as a `systemd` service. See [Advanced Post-installation Tasks](advanced-setup.md) for details.
+On Linux, as an optional post-installation task, you may wish to setup the bot to run as a `systemd` service or configure it to send the log messages to the `syslog`/`rsyslog` or `journald` daemons. See [Advanced Logging](advanced-setup.md#advanced-logging) for details.
 
 ------
 

freqtrade/configuration/cli_options.py

@@ -36,7 +36,8 @@ AVAILABLE_CLI_OPTIONS = {
     ),
     "logfile": Arg(
         '--logfile',
-        help='Log to the file specified.',
+        help="Log to the file specified. Special values are: 'syslog', 'journald'. "
+             "See the documentation for more details.",
         metavar='FILE',
     ),
     "version": Arg(

freqtrade/loggers.py

@@ -1,9 +1,12 @@
 import logging
 import sys
 
-from logging.handlers import RotatingFileHandler
+from logging import Formatter
+from logging.handlers import RotatingFileHandler, SysLogHandler
 from typing import Any, Dict, List
 
+from freqtrade import OperationalException
+
 
 logger = logging.getLogger(__name__)
 
@@ -36,8 +39,36 @@ def setup_logging(config: Dict[str, Any]) -> None:
     # Log to stderr
     log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)]
 
-    if config.get('logfile'):
-        log_handlers.append(RotatingFileHandler(config['logfile'],
+    logfile = config.get('logfile')
+    if logfile:
+        s = logfile.split(':')
+        if s[0] == 'syslog':
+            # Address can be either a string (socket filename) for Unix domain socket or
+            # a tuple (hostname, port) for UDP socket.
+            # Address can be omitted (i.e. simple 'syslog' used as the value of
+            # config['logfilename']), which defaults to '/dev/log', applicable for most
+            # of the systems.
+            address = (s[1], int(s[2])) if len(s) > 2 else s[1] if len(s) > 1 else '/dev/log'
+            handler = SysLogHandler(address=address)
+            # No datetime field for logging into syslog, to allow syslog
+            # to perform reduction of repeating messages if this is set in the
+            # syslog config. The messages should be equal for this.
+            handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
+            log_handlers.append(handler)
+        elif s[0] == 'journald':
+            try:
+                from systemd.journal import JournaldLogHandler
+            except ImportError:
+                raise OperationalException("You need the systemd python package be installed in "
+                                           "order to use logging to journald.")
+            handler = JournaldLogHandler()
+            # No datetime field for logging into journald, to allow syslog
+            # to perform reduction of repeating messages if this is set in the
+            # syslog config. The messages should be equal for this.
+            handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
+            log_handlers.append(handler)
+        else:
+            log_handlers.append(RotatingFileHandler(logfile,
                                                     maxBytes=1024 * 1024,  # 1Mb
                                                     backupCount=10))
 

tests/conftest.py

@@ -1387,7 +1387,7 @@ def import_fails() -> None:
     realimport = builtins.__import__
 
     def mockedimport(name, *args, **kwargs):
-        if name in ["filelock"]:
+        if name in ["filelock", 'systemd.journal']:
             raise ImportError(f"No module named '{name}'")
         return realimport(name, *args, **kwargs)
 

tests/test_configuration.py

@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, protected-access, invalid-name
 import json
 import logging
+import sys
 import warnings
 from copy import deepcopy
 from pathlib import Path
@@ -19,7 +20,7 @@ from freqtrade.configuration.deprecated_settings import (
     process_temporary_deprecated_settings)
 from freqtrade.configuration.load_config import load_config_file
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
-from freqtrade.loggers import _set_loggers
+from freqtrade.loggers import _set_loggers, setup_logging
 from freqtrade.state import RunMode
 from tests.conftest import (log_has, log_has_re,
                             patched_configuration_load_config_file)
@@ -638,6 +639,56 @@ def test_set_loggers() -> None:
     assert logging.getLogger('telegram').level is logging.INFO
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows")
+def test_set_loggers_syslog(mocker):
+    logger = logging.getLogger()
+    orig_handlers = logger.handlers
+    logger.handlers = []
+
+    config = {'verbosity': 2,
+              'logfile': 'syslog:/dev/log',
+              }
+
+    setup_logging(config)
+    assert len(logger.handlers) == 2
+    assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler]
+    assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
+    # reset handlers to not break pytest
+    logger.handlers = orig_handlers
+
+
+@pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.")
+def test_set_loggers_journald(mocker):
+    logger = logging.getLogger()
+    orig_handlers = logger.handlers
+    logger.handlers = []
+
+    config = {'verbosity': 2,
+              'logfile': 'journald',
+              }
+
+    setup_logging(config)
+    assert len(logger.handlers) == 2
+    assert [x for x in logger.handlers if type(x).__name__ == "JournaldLogHandler"]
+    assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
+    # reset handlers to not break pytest
+    logger.handlers = orig_handlers
+
+
+def test_set_loggers_journald_importerror(mocker, import_fails):
+    logger = logging.getLogger()
+    orig_handlers = logger.handlers
+    logger.handlers = []
+
+    config = {'verbosity': 2,
+              'logfile': 'journald',
+              }
+    with pytest.raises(OperationalException,
+                       match=r'You need the systemd python package.*'):
+        setup_logging(config)
+    logger.handlers = orig_handlers
+
+
 def test_set_logfile(default_conf, mocker):
     patched_configuration_load_config_file(mocker, default_conf)