From aa07a50468a1babe159928d7c4190ad470b33c65 Mon Sep 17 00:00:00 2001 From: Filip Krakowski Date: Sat, 20 Feb 2021 21:18:50 +0100 Subject: [PATCH] Implement environment variable substitution within configuration --- freqtrade/configuration/load_config.py | 73 ++++++++++---- tests/config_test_environment.json | 127 +++++++++++++++++++++++++ tests/test_configuration.py | 33 ++++++- 3 files changed, 213 insertions(+), 20 deletions(-) create mode 100644 tests/config_test_environment.json diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 726126034..e8f393dc3 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -4,6 +4,7 @@ This module contain functions to load the configuration file import logging import re import sys +from os import environ from pathlib import Path from typing import Any, Dict @@ -14,30 +15,57 @@ from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) - CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS -def log_config_error_range(path: str, errmsg: str) -> str: +class SubstitutionException(Exception): """ - Parses configuration file and prints range around error + Indicates that a variable within the configuration couldn't be substituted. """ - if path != '-': - offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg) - if offsetlist: - offset = int(offsetlist[0]) - text = Path(path).read_text() - # Fetch an offset of 80 characters around the error line - subtext = text[offset-min(80, offset):offset+80] - segments = subtext.split('\n') - if len(segments) > 3: - # Remove first and last lines, to avoid odd truncations - return '\n'.join(segments[1:-1]) - else: - return subtext + + def __init__(self, key: str, offset: int): + self.offset = offset + self.err = f'Environment variable {key} was requested for substitution, but is not set.' + super().__init__(self.err) + + +def log_config_error_range(path: str, offset: int) -> str: + """ + Parses configuration file and prints range around the specified offset + """ + if path != '-' and offset != -1: + text = Path(path).read_text() + # Fetch an offset of 80 characters around the error line + subtext = text[offset - min(80, offset):offset + 80] + segments = subtext.split('\n') + if len(segments) > 3: + # Remove first and last lines, to avoid odd truncations + return '\n'.join(segments[1:-1]) + else: + return subtext + return '' +def substitute_environment_variable(match: re.Match) -> str: + """ + Substitutes a matched environment variable with its value + """ + key = match.group(1).strip() + if key not in environ: + raise SubstitutionException(key, match.start(0)) + + return environ[key] + + +def extract_error_offset(errmsg: str) -> int: + offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg) + if offsetlist: + return int(offsetlist[0]) + + return -1 + + def load_config_file(path: str) -> Dict[str, Any]: """ Loads a config file from the given path @@ -47,13 +75,22 @@ def load_config_file(path: str) -> Dict[str, Any]: try: # Read config from stdin if requested in the options with open(path) if path != '-' else sys.stdin as file: - config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE) + content = re.sub(r'\${(.*?)}', substitute_environment_variable, file.read()) + config = rapidjson.loads(content, parse_mode=CONFIG_PARSE_MODE) except FileNotFoundError: raise OperationalException( f'Config file "{path}" not found!' ' Please create a config file or check whether it exists.') except rapidjson.JSONDecodeError as e: - err_range = log_config_error_range(path, str(e)) + err_offset = extract_error_offset(str(e)) + err_range = log_config_error_range(path, err_offset) + raise OperationalException( + f'{e}\n' + f'Please verify the following segment of your configuration:\n{err_range}' + if err_range else 'Please verify your configuration file for syntax errors.' + ) + except SubstitutionException as e: + err_range = log_config_error_range(path, e.offset) raise OperationalException( f'{e}\n' f'Please verify the following segment of your configuration:\n{err_range}' diff --git a/tests/config_test_environment.json b/tests/config_test_environment.json new file mode 100644 index 000000000..ac3a64390 --- /dev/null +++ b/tests/config_test_environment.json @@ -0,0 +1,127 @@ +{ + /* Single-line C-style comment */ + "max_open_trades": 3, + /* + * Multi-line C-style comment + */ + "stake_currency": "BTC", + "stake_amount": 0.05, + "fiat_display_currency": "USD", // C++-style comment + "amount_reserve_percent" : 0.05, // And more, tabs before this comment + "dry_run": false, + "timeframe": "5m", + "trailing_stop": false, + "trailing_stop_positive": 0.005, + "trailing_stop_positive_offset": 0.0051, + "trailing_only_offset_is_reached": false, + "minimal_roi": { + "40": 0.0, + "30": 0.01, + "20": 0.02, + "0": 0.04 + }, + "stoploss": -0.10, + "unfilledtimeout": { + "buy": 10, + "sell": 30, // Trailing comma should also be accepted now + }, + "bid_strategy": { + "use_order_book": false, + "ask_last_balance": 0.0, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy":{ + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 9 + }, + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market", + "stoploss_on_exchange": false, + "stoploss_on_exchange_interval": 60 + }, + "order_time_in_force": { + "buy": "gtc", + "sell": "gtc" + }, + "pairlist": { + "method": "VolumePairList", + "config": { + "number_assets": 20, + "sort_key": "quoteVolume", + "precision_filter": false + } + }, + "exchange": { + "name": "bittrex", + "sandbox": false, + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "password": "", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": false, + "rateLimit": 500, + "aiohttp_trust_env": false + }, + "pair_whitelist": [ + "ETH/BTC", + "LTC/BTC", + "ETC/BTC", + "DASH/BTC", + "ZEC/BTC", + "XLM/BTC", + "NXT/BTC", + "TRX/BTC", + "ADA/BTC", + "XMR/BTC" + ], + "pair_blacklist": [ + "DOGE/BTC" + ], + "outdated_offset": 5, + "markets_refresh_interval": 60 + }, + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { +// We can now comment out some settings +// "enabled": true, + "enabled": false, + "token": "${TELEGRAM_TOKEN}", + "chat_id": "${TELEGRAM_CHAT}" + }, + "api_server": { + "enabled": false, + "listen_ip_address": "127.0.0.1", + "listen_port": 8080, + "username": "freqtrader", + "password": "SuperSecurePassword" + }, + "db_url": "sqlite:///tradesv3.sqlite", + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + }, + "strategy": "DefaultStrategy", + "strategy_path": "user_data/strategies/" +} diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 94c3e24f6..b25067bd5 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name import json import logging +import os import sys import warnings from copy import deepcopy @@ -18,7 +19,8 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti process_deprecated_setting, process_removed_setting, process_temporary_deprecated_settings) -from freqtrade.configuration.load_config import load_config_file, log_config_error_range +from freqtrade.configuration.load_config import (extract_error_offset, load_config_file, + log_config_error_range) from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.exceptions import OperationalException from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre @@ -82,7 +84,8 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None: '"stake_amount": 0.001,', '"stake_amount": .001,') mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata)) - x = log_config_error_range('somefile', 'Parse error at offset 64: Invalid value.') + offset = extract_error_offset('Parse error at offset 64: Invalid value.') + x = log_config_error_range('somefile', offset) assert isinstance(x, str) assert (x == '{"max_open_trades": 1, "stake_currency": "BTC", ' '"stake_amount": .001, "fiat_display_currency": "USD", ' @@ -909,6 +912,32 @@ def test_load_config_test_comments() -> None: assert conf +def test_load_config_test_env_variables(mocker) -> None: + """ + Load config with environment variables + """ + token = "17264728:eW91dHUuYmUvMDAwYWw3cnUzbXMg" + chat_id = "17263827" + + mocker.patch.dict(os.environ, {'TELEGRAM_TOKEN': token, 'TELEGRAM_CHAT': chat_id}) + config_file = Path(__file__).parents[0] / "config_test_environment.json" + conf = load_config_file(str(config_file)) + + assert conf + assert conf['telegram']['token'] == token + assert conf['telegram']['chat_id'] == chat_id + + +def test_load_config_test_substitution_error() -> None: + """ + Load config with environment variables without setting them + """ + + config_file = Path(__file__).parents[0] / "config_test_environment.json" + with pytest.raises(OperationalException, match=r'.*Environment variable TELEGRAM_TOKEN*'): + load_config_file(str(config_file)) + + def test_load_config_default_exchange(all_conf) -> None: """ config['exchange'] subtree has required options in it