Merge pull request #6665 from freqtrade/config_from_config

Allow recursive loading of configuration files

config_examples/config_full.example.json

@@ -182,6 +182,7 @@
     "disable_dataframe_checks": false,
     "strategy": "SampleStrategy",
     "strategy_path": "user_data/strategies/",
+    "add_config_files": [],
     "dataformat_ohlcv": "json",
     "dataformat_trades": "jsongz"
 }

docs/configuration.md

@@ -53,14 +53,33 @@ FREQTRADE__EXCHANGE__SECRET=<yourExchangeSecret>
 
 Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
 
+You can specify additional configuration files in `add_config_files`. Files specified in this parameter will be loaded and merged with the initial config file. The files are resolved relative to the initial configuration file.
+This is similar to using multiple `--config` parameters, but simpler in usage as you don't have to specify all files for all commands.
+
 !!! Tip "Use multiple configuration files to keep secrets secret"
     You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
 
+    ``` json title="user_data/config.json"
+    "add_config_files": [
+        "config-private.json"
+    ]
+    ```
+    
+    ``` bash
+    freqtrade trade --config user_data/config.json <...>
+    ```
+
+    The 2nd file should only specify what you intend to override.
+    If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
+
+    For one-off commands, you can also use the below syntax by specifying multiple "--config" parameters.
+
     ``` bash
     freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>
     ```
-    The 2nd file should only specify what you intend to override.
-    If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
+
+    This is equivalent to the example above - but `config-private.json` is specified as cli argument.
+
 
 ## Configuration parameters
 
@@ -175,6 +194,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi
 | `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details. <br> **Datatype:** Boolean
 | `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file. <br> **Datatype:** String
 | `user_data_dir` | Directory containing user data. <br> *Defaults to `./user_data/`*. <br> **Datatype:** String
+| `add_config_files` | Additional config files. These files will be loaded and merged with the current config file. The files are resolved relative to the initial file.<br> *Defaults to `[]`*. <br> **Datatype:** List of strings
 | `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data. <br> *Defaults to `json`*. <br> **Datatype:** String
 | `dataformat_trades` | Data format to use to store historical trades data. <br> *Defaults to `jsongz`*. <br> **Datatype:** String
 | `position_adjustment_enable` | Enables the strategy to use position adjustments (additional buys or sells). [More information here](strategy-callbacks.md#adjust-trade-position). <br> [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.*<br> **Datatype:** Boolean

freqtrade/configuration/load_config.py

@@ -75,19 +75,41 @@ def load_config_file(path: str) -> Dict[str, Any]:
     return config
 
 
-def load_from_files(files: List[str]) -> Dict[str, Any]:
-
+def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> Dict[str, Any]:
+    """
+    Recursively load configuration files if specified.
+    Sub-files are assumed to be relative to the initial config.
+    """
     config: Dict[str, Any] = {}
+    if level > 5:
+        raise OperationalException("Config loop detected.")
 
     if not files:
         return deepcopy(MINIMAL_CONFIG)
-
+    files_loaded = []
     # We expect here a list of config filenames
-    for path in files:
-        logger.info(f'Using config: {path} ...')
-        # Merge config options, overwriting old values
-        config = deep_merge_dicts(load_config_file(path), config)
+    for filename in files:
+        logger.info(f'Using config: {filename} ...')
+        if filename == '-':
+            # Immediately load stdin and return
+            return load_config_file(filename)
+        file = Path(filename)
+        if base_path:
+            # Prepend basepath to allow for relative assignments
+            file = base_path / file
 
-    config['config_files'] = files
+        config_tmp = load_config_file(str(file))
+        if 'add_config_files' in config_tmp:
+            config_sub = load_from_files(
+                config_tmp['add_config_files'], file.resolve().parent, level + 1)
+            files_loaded.extend(config_sub.get('config_files', []))
+            deep_merge_dicts(config_sub, config_tmp)
+
+        files_loaded.insert(0, str(file))
+
+        # Merge config options, overwriting prior values
+        config = deep_merge_dicts(config_tmp, config)
+
+    config['config_files'] = files_loaded
 
     return config

freqtrade/constants.py

@@ -91,15 +91,14 @@ SUPPORTED_FIAT = [
 ]
 
 MINIMAL_CONFIG = {
-    'stake_currency': '',
-    'dry_run': True,
-    'exchange': {
-        'name': '',
-        'key': '',
-        'secret': '',
-        'pair_whitelist': [],
-        'ccxt_async_config': {
-            'enableRateLimit': True,
+    "stake_currency": "",
+    "dry_run": True,
+    "exchange": {
+        "name": "",
+        "key": "",
+        "secret": "",
+        "pair_whitelist": [],
+        "ccxt_async_config": {
         }
     }
 }

tests/test_configuration.py

@@ -18,7 +18,8 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti
                                                          process_removed_setting,
                                                          process_temporary_deprecated_settings)
 from freqtrade.configuration.environment_vars import flat_vars_to_nested_dict
-from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range
+from freqtrade.configuration.load_config import (load_config_file, load_file, load_from_files,
+                                                 log_config_error_range)
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
@@ -206,6 +207,32 @@ def test_from_config(default_conf, mocker, caplog) -> None:
     assert isinstance(validated_conf['user_data_dir'], Path)
 
 
+def test_from_recursive_files(testdatadir) -> None:
+    files = testdatadir / "testconfigs/testconfig.json"
+
+    conf = Configuration.from_files([files])
+
+    assert conf
+    # Exchange comes from "the first config"
+    assert conf['exchange']
+    # Pricing comes from the 2nd config
+    assert conf['entry_pricing']
+    assert conf['entry_pricing']['price_side'] == "same"
+    assert conf['exit_pricing']
+    # The other key comes from pricing2, which is imported by pricing.json
+    assert conf['exit_pricing']['price_side'] == "other"
+
+    assert len(conf['config_files']) == 4
+    assert 'testconfig.json' in conf['config_files'][0]
+    assert 'test_pricing_conf.json' in conf['config_files'][1]
+    assert 'test_base_config.json' in conf['config_files'][2]
+    assert 'test_pricing2_conf.json' in conf['config_files'][3]
+
+    files = testdatadir / "testconfigs/recursive.json"
+    with pytest.raises(OperationalException, match="Config loop detected."):
+        load_from_files([files])
+
+
 def test_print_config(default_conf, mocker, caplog) -> None:
     conf1 = deepcopy(default_conf)
     # Delete non-json elements from default_conf

tests/testdata/testconfigs/recursive.json

@@ -0,0 +1,6 @@
+{
+    // This file fails as it's loading itself over and over
+    "add_config_files": [
+        "./recursive.json"
+    ]
+}

tests/testdata/testconfigs/test_base_config.json

@@ -0,0 +1,12 @@
+{
+    "stake_currency": "",
+    "dry_run": true,
+    "exchange": {
+        "name": "",
+        "key": "",
+        "secret": "",
+        "pair_whitelist": [],
+        "ccxt_async_config": {
+        }
+    }
+}

tests/testdata/testconfigs/test_pricing2_conf.json

@@ -0,0 +1,18 @@
+{
+    "entry_pricing": {
+        "price_side": "same",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "exit_pricing":{
+        "price_side": "other",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0
+    }
+}

tests/testdata/testconfigs/test_pricing_conf.json

@@ -0,0 +1,21 @@
+{
+    "entry_pricing": {
+        "price_side": "same",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "exit_pricing":{
+        "price_side": "same",
+        "use_order_book": true,
+        "order_book_top": 1,
+        "price_last_balance": 0.0
+    },
+    "add_config_files": [
+        "./test_pricing2_conf.json"
+    ]
+}

tests/testdata/testconfigs/testconfig.json

@@ -0,0 +1,6 @@
+{
+    "add_config_files": [
+        "test_base_config.json",
+        "test_pricing_conf.json"
+    ]
+}