Implement environment variable substitution within configuration

freqtrade/configuration/load_config.py

@@ -4,6 +4,7 @@ This module contain functions to load the configuration file
 import logging
 import re
 import sys
+from os import environ
 from pathlib import Path
 from typing import Any, Dict
 
@@ -14,30 +15,57 @@ from freqtrade.exceptions import OperationalException
 
 logger = logging.getLogger(__name__)
 
-
 CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS
 
 
-def log_config_error_range(path: str, errmsg: str) -> str:
+class SubstitutionException(Exception):
     """
-    Parses configuration file and prints range around error
+    Indicates that a variable within the configuration couldn't be substituted.
     """
-    if path != '-':
+
-        offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
+    def __init__(self, key: str, offset: int):
-        if offsetlist:
+        self.offset = offset
-            offset = int(offsetlist[0])
+        self.err = f'Environment variable {key} was requested for substitution, but is not set.'
-            text = Path(path).read_text()
+        super().__init__(self.err)
-            # Fetch an offset of 80 characters around the error line
+
-            subtext = text[offset-min(80, offset):offset+80]
+
-            segments = subtext.split('\n')
+def log_config_error_range(path: str, offset: int) -> str:
-            if len(segments) > 3:
+    """
-                # Remove first and last lines, to avoid odd truncations
+    Parses configuration file and prints range around the specified offset
-                return '\n'.join(segments[1:-1])
+    """
-            else:
+    if path != '-' and offset != -1:
-                return subtext
+        text = Path(path).read_text()
+        # Fetch an offset of 80 characters around the error line
+        subtext = text[offset - min(80, offset):offset + 80]
+        segments = subtext.split('\n')
+        if len(segments) > 3:
+            # Remove first and last lines, to avoid odd truncations
+            return '\n'.join(segments[1:-1])
+        else:
+            return subtext
+
     return ''
 
 
+def substitute_environment_variable(match: re.Match) -> str:
+    """
+    Substitutes a matched environment variable with its value
+    """
+    key = match.group(1).strip()
+    if key not in environ:
+        raise SubstitutionException(key, match.start(0))
+
+    return environ[key]
+
+
+def extract_error_offset(errmsg: str) -> int:
+    offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
+    if offsetlist:
+        return int(offsetlist[0])
+
+    return -1
+
+
 def load_config_file(path: str) -> Dict[str, Any]:
     """
     Loads a config file from the given path
@@ -47,13 +75,22 @@ def load_config_file(path: str) -> Dict[str, Any]:
     try:
         # Read config from stdin if requested in the options
         with open(path) if path != '-' else sys.stdin as file:
-            config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
+            content = re.sub(r'\${(.*?)}', substitute_environment_variable, file.read())
+            config = rapidjson.loads(content, parse_mode=CONFIG_PARSE_MODE)
     except FileNotFoundError:
         raise OperationalException(
             f'Config file "{path}" not found!'
             ' Please create a config file or check whether it exists.')
     except rapidjson.JSONDecodeError as e:
-        err_range = log_config_error_range(path, str(e))
+        err_offset = extract_error_offset(str(e))
+        err_range = log_config_error_range(path, err_offset)
+        raise OperationalException(
+            f'{e}\n'
+            f'Please verify the following segment of your configuration:\n{err_range}'
+            if err_range else 'Please verify your configuration file for syntax errors.'
+        )
+    except SubstitutionException as e:
+        err_range = log_config_error_range(path, e.offset)
         raise OperationalException(
             f'{e}\n'
             f'Please verify the following segment of your configuration:\n{err_range}'

tests/config_test_environment.json

@@ -0,0 +1,127 @@
+{
+    /* Single-line C-style comment */
+    "max_open_trades": 3,
+    /*
+     * Multi-line C-style comment
+     */
+    "stake_currency": "BTC",
+    "stake_amount": 0.05,
+    "fiat_display_currency": "USD",             // C++-style comment
+    "amount_reserve_percent" : 0.05,		// And more, tabs before this comment
+    "dry_run": false,
+    "timeframe": "5m",
+    "trailing_stop": false,
+    "trailing_stop_positive": 0.005,
+    "trailing_stop_positive_offset": 0.0051,
+    "trailing_only_offset_is_reached": false,
+    "minimal_roi": {
+        "40":  0.0,
+        "30":  0.01,
+        "20":  0.02,
+        "0":  0.04
+    },
+    "stoploss": -0.10,
+    "unfilledtimeout": {
+        "buy": 10,
+        "sell": 30,				// Trailing comma should also be accepted now
+    },
+    "bid_strategy": {
+        "use_order_book": false,
+        "ask_last_balance": 0.0,
+        "order_book_top": 1,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "ask_strategy":{
+        "use_order_book": false,
+        "order_book_min": 1,
+        "order_book_max": 9
+    },
+    "order_types": {
+        "buy": "limit",
+        "sell": "limit",
+        "stoploss": "market",
+        "stoploss_on_exchange": false,
+        "stoploss_on_exchange_interval": 60
+    },
+    "order_time_in_force": {
+        "buy": "gtc",
+        "sell": "gtc"
+    },
+    "pairlist": {
+        "method": "VolumePairList",
+        "config": {
+            "number_assets": 20,
+            "sort_key": "quoteVolume",
+            "precision_filter": false
+        }
+    },
+    "exchange": {
+        "name": "bittrex",
+        "sandbox": false,
+        "key": "your_exchange_key",
+        "secret": "your_exchange_secret",
+        "password": "",
+        "ccxt_config": {"enableRateLimit": true},
+        "ccxt_async_config": {
+            "enableRateLimit": false,
+            "rateLimit": 500,
+            "aiohttp_trust_env": false
+        },
+        "pair_whitelist": [
+            "ETH/BTC",
+            "LTC/BTC",
+            "ETC/BTC",
+            "DASH/BTC",
+            "ZEC/BTC",
+            "XLM/BTC",
+            "NXT/BTC",
+            "TRX/BTC",
+            "ADA/BTC",
+            "XMR/BTC"
+        ],
+        "pair_blacklist": [
+            "DOGE/BTC"
+        ],
+        "outdated_offset": 5,
+        "markets_refresh_interval": 60
+    },
+    "edge": {
+        "enabled": false,
+        "process_throttle_secs": 3600,
+        "calculate_since_number_of_days": 7,
+        "allowed_risk": 0.01,
+        "stoploss_range_min": -0.01,
+        "stoploss_range_max": -0.1,
+        "stoploss_range_step": -0.01,
+        "minimum_winrate": 0.60,
+        "minimum_expectancy": 0.20,
+        "min_trade_number": 10,
+        "max_trade_duration_minute": 1440,
+        "remove_pumps": false
+    },
+    "telegram": {
+// We can now comment out some settings
+//        "enabled": true,
+        "enabled": false,
+        "token": "${TELEGRAM_TOKEN}",
+        "chat_id": "${TELEGRAM_CHAT}"
+    },
+    "api_server": {
+        "enabled": false,
+        "listen_ip_address": "127.0.0.1",
+        "listen_port": 8080,
+        "username": "freqtrader",
+        "password": "SuperSecurePassword"
+    },
+    "db_url": "sqlite:///tradesv3.sqlite",
+    "initial_state": "running",
+    "forcebuy_enable": false,
+    "internals": {
+        "process_throttle_secs": 5
+    },
+    "strategy": "DefaultStrategy",
+    "strategy_path": "user_data/strategies/"
+}

tests/test_configuration.py

@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, protected-access, invalid-name
 import json
 import logging
+import os
 import sys
 import warnings
 from copy import deepcopy
@@ -18,7 +19,8 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti
                                                          process_deprecated_setting,
                                                          process_removed_setting,
                                                          process_temporary_deprecated_settings)
-from freqtrade.configuration.load_config import load_config_file, log_config_error_range
+from freqtrade.configuration.load_config import (extract_error_offset, load_config_file,
+                                                 log_config_error_range)
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
 from freqtrade.exceptions import OperationalException
 from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre
@@ -82,7 +84,8 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None:
         '"stake_amount": 0.001,', '"stake_amount": .001,')
     mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata))
 
-    x = log_config_error_range('somefile', 'Parse error at offset 64: Invalid value.')
+    offset = extract_error_offset('Parse error at offset 64: Invalid value.')
+    x = log_config_error_range('somefile', offset)
     assert isinstance(x, str)
     assert (x == '{"max_open_trades": 1, "stake_currency": "BTC", '
             '"stake_amount": .001, "fiat_display_currency": "USD", '
@@ -909,6 +912,32 @@ def test_load_config_test_comments() -> None:
     assert conf
 
 
+def test_load_config_test_env_variables(mocker) -> None:
+    """
+    Load config with environment variables
+    """
+    token = "17264728:eW91dHUuYmUvMDAwYWw3cnUzbXMg"
+    chat_id = "17263827"
+
+    mocker.patch.dict(os.environ, {'TELEGRAM_TOKEN': token, 'TELEGRAM_CHAT': chat_id})
+    config_file = Path(__file__).parents[0] / "config_test_environment.json"
+    conf = load_config_file(str(config_file))
+
+    assert conf
+    assert conf['telegram']['token'] == token
+    assert conf['telegram']['chat_id'] == chat_id
+
+
+def test_load_config_test_substitution_error() -> None:
+    """
+    Load config with environment variables without setting them
+    """
+
+    config_file = Path(__file__).parents[0] / "config_test_environment.json"
+    with pytest.raises(OperationalException, match=r'.*Environment variable TELEGRAM_TOKEN*'):
+        load_config_file(str(config_file))
+
+
 def test_load_config_default_exchange(all_conf) -> None:
     """
     config['exchange'] subtree has required options in it