adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement environment variable substitution within configuration

Browse Source
This commit is contained in:
Filip Krakowski 2021-02-20 21:18:50 +01:00
parent 245e39e523
commit aa07a50468
No known key found for this signature in database
GPG Key ID: 773AE6E405AFE69A
3 changed files with 213 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
freqtrade/configuration/load_config.py
View File

@ -4,6 +4,7 @@ This module contain functions to load the configuration file
import logging
import re
import sys
from os import environ
from pathlib import Path
from typing import Any, Dict
@ -14,30 +15,57 @@ from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS
def log_config_error_range(path: str, errmsg: str) -> str:
class SubstitutionException(Exception):
"""
Parses configuration file and prints range around error
Indicates that a variable within the configuration couldn't be substituted.
"""
if path != '-':
offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
if offsetlist:
offset = int(offsetlist[0])
text = Path(path).read_text()
# Fetch an offset of 80 characters around the error line
subtext = text[offset-min(80, offset):offset+80]
segments = subtext.split('\n')
if len(segments) > 3:
# Remove first and last lines, to avoid odd truncations
return '\n'.join(segments[1:-1])
else:
return subtext
def __init__(self, key: str, offset: int):
self.offset = offset
self.err = f'Environment variable {key} was requested for substitution, but is not set.'
super().__init__(self.err)
def log_config_error_range(path: str, offset: int) -> str:
"""
Parses configuration file and prints range around the specified offset
"""
if path != '-' and offset != -1:
text = Path(path).read_text()
# Fetch an offset of 80 characters around the error line
subtext = text[offset - min(80, offset):offset + 80]
segments = subtext.split('\n')
if len(segments) > 3:
# Remove first and last lines, to avoid odd truncations
return '\n'.join(segments[1:-1])
else:
return subtext
return ''
def substitute_environment_variable(match: re.Match) -> str:
"""
Substitutes a matched environment variable with its value
"""
key = match.group(1).strip()
if key not in environ:
raise SubstitutionException(key, match.start(0))
return environ[key]
def extract_error_offset(errmsg: str) -> int:
offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg)
if offsetlist:
return int(offsetlist[0])
return -1
def load_config_file(path: str) -> Dict[str, Any]:
"""
Loads a config file from the given path
@ -47,13 +75,22 @@ def load_config_file(path: str) -> Dict[str, Any]:
try:
# Read config from stdin if requested in the options
with open(path) if path != '-' else sys.stdin as file:
config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
content = re.sub(r'\${(.*?)}', substitute_environment_variable, file.read())
config = rapidjson.loads(content, parse_mode=CONFIG_PARSE_MODE)
except FileNotFoundError:
raise OperationalException(
f'Config file "{path}" not found!'
' Please create a config file or check whether it exists.')
except rapidjson.JSONDecodeError as e:
err_range = log_config_error_range(path, str(e))
err_offset = extract_error_offset(str(e))
err_range = log_config_error_range(path, err_offset)
raise OperationalException(
f'{e}\n'
f'Please verify the following segment of your configuration:\n{err_range}'
if err_range else 'Please verify your configuration file for syntax errors.'
)
except SubstitutionException as e:
err_range = log_config_error_range(path, e.offset)
raise OperationalException(
f'{e}\n'
f'Please verify the following segment of your configuration:\n{err_range}'

127
tests/config_test_environment.json Normal file
View File

@ -0,0 +1,127 @@
{
/* Single-line C-style comment */
"max_open_trades": 3,
/*
* Multi-line C-style comment
*/
"stake_currency": "BTC",
"stake_amount": 0.05,
"fiat_display_currency": "USD", // C++-style comment
"amount_reserve_percent" : 0.05, // And more, tabs before this comment
"dry_run": false,
"timeframe": "5m",
"trailing_stop": false,
"trailing_stop_positive": 0.005,
"trailing_stop_positive_offset": 0.0051,
"trailing_only_offset_is_reached": false,
"minimal_roi": {
"40": 0.0,
"30": 0.01,
"20": 0.02,
"0": 0.04
},
"stoploss": -0.10,
"unfilledtimeout": {
"buy": 10,
"sell": 30, // Trailing comma should also be accepted now
},
"bid_strategy": {
"use_order_book": false,
"ask_last_balance": 0.0,
"order_book_top": 1,
"check_depth_of_market": {
"enabled": false,
"bids_to_ask_delta": 1
}
},
"ask_strategy":{
"use_order_book": false,
"order_book_min": 1,
"order_book_max": 9
},
"order_types": {
"buy": "limit",
"sell": "limit",
"stoploss": "market",
"stoploss_on_exchange": false,
"stoploss_on_exchange_interval": 60
},
"order_time_in_force": {
"buy": "gtc",
"sell": "gtc"
},
"pairlist": {
"method": "VolumePairList",
"config": {
"number_assets": 20,
"sort_key": "quoteVolume",
"precision_filter": false
}
},
"exchange": {
"name": "bittrex",
"sandbox": false,
"key": "your_exchange_key",
"secret": "your_exchange_secret",
"password": "",
"ccxt_config": {"enableRateLimit": true},
"ccxt_async_config": {
"enableRateLimit": false,
"rateLimit": 500,
"aiohttp_trust_env": false
},
"pair_whitelist": [
"ETH/BTC",
"LTC/BTC",
"ETC/BTC",
"DASH/BTC",
"ZEC/BTC",
"XLM/BTC",
"NXT/BTC",
"TRX/BTC",
"ADA/BTC",
"XMR/BTC"
],
"pair_blacklist": [
"DOGE/BTC"
],
"outdated_offset": 5,
"markets_refresh_interval": 60
},
"edge": {
"enabled": false,
"process_throttle_secs": 3600,
"calculate_since_number_of_days": 7,
"allowed_risk": 0.01,
"stoploss_range_min": -0.01,
"stoploss_range_max": -0.1,
"stoploss_range_step": -0.01,
"minimum_winrate": 0.60,
"minimum_expectancy": 0.20,
"min_trade_number": 10,
"max_trade_duration_minute": 1440,
"remove_pumps": false
},
"telegram": {
// We can now comment out some settings
// "enabled": true,
"enabled": false,
"token": "${TELEGRAM_TOKEN}",
"chat_id": "${TELEGRAM_CHAT}"
},
"api_server": {
"enabled": false,
"listen_ip_address": "127.0.0.1",
"listen_port": 8080,
"username": "freqtrader",
"password": "SuperSecurePassword"
},
"db_url": "sqlite:///tradesv3.sqlite",
"initial_state": "running",
"forcebuy_enable": false,
"internals": {
"process_throttle_secs": 5
},
"strategy": "DefaultStrategy",
"strategy_path": "user_data/strategies/"
}

33
tests/test_configuration.py
View File

@ -1,6 +1,7 @@
# pragma pylint: disable=missing-docstring, protected-access, invalid-name
import json
import logging
import os
import sys
import warnings
from copy import deepcopy
@ -18,7 +19,8 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti
process_deprecated_setting,
process_removed_setting,
process_temporary_deprecated_settings)
from freqtrade.configuration.load_config import load_config_file, log_config_error_range
from freqtrade.configuration.load_config import (extract_error_offset, load_config_file,
log_config_error_range)
from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre
@ -82,7 +84,8 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None:
'"stake_amount": 0.001,', '"stake_amount": .001,')
mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata))
x = log_config_error_range('somefile', 'Parse error at offset 64: Invalid value.')
offset = extract_error_offset('Parse error at offset 64: Invalid value.')
x = log_config_error_range('somefile', offset)
assert isinstance(x, str)
assert (x == '{"max_open_trades": 1, "stake_currency": "BTC", '
'"stake_amount": .001, "fiat_display_currency": "USD", '
@ -909,6 +912,32 @@ def test_load_config_test_comments() -> None:
assert conf
def test_load_config_test_env_variables(mocker) -> None:
"""
Load config with environment variables
"""
token = "17264728:eW91dHUuYmUvMDAwYWw3cnUzbXMg"
chat_id = "17263827"
mocker.patch.dict(os.environ, {'TELEGRAM_TOKEN': token, 'TELEGRAM_CHAT': chat_id})
config_file = Path(__file__).parents[0] / "config_test_environment.json"
conf = load_config_file(str(config_file))
assert conf
assert conf['telegram']['token'] == token
assert conf['telegram']['chat_id'] == chat_id
def test_load_config_test_substitution_error() -> None:
"""
Load config with environment variables without setting them
"""
config_file = Path(__file__).parents[0] / "config_test_environment.json"
with pytest.raises(OperationalException, match=r'.*Environment variable TELEGRAM_TOKEN*'):
load_config_file(str(config_file))
def test_load_config_default_exchange(all_conf) -> None:
"""
config['exchange'] subtree has required options in it