Merge branch 'develop' into log_has_ref

docs/bot-usage.md

@@ -56,8 +56,8 @@ freqtrade -c path/far/far/away/config.json
 
 The bot allows you to use multiple configuration files by specifying multiple
 `-c/--config` configuration options in the command line. Configuration parameters
-defined in the last configuration file override parameters with the same name
+defined in latter configuration files override parameters with the same name
-defined in the previous configuration file specified in the command line.
+defined in the previous configuration files specified in the command line earlier.
 
 For example, you can make a separate configuration file with your key and secrete
 for the Exchange you use for trading, specify default configuration file with

docs/data-analysis.md

@@ -31,6 +31,16 @@ df = load_trades_from_db("sqlite:///tradesv3.sqlite")
 df.groupby("pair")["sell_reason"].value_counts()
 ```
 
+### Load multiple configuration files
+
+This option can be usefull to inspect the results of passing in multiple configs in case of problems
+
+``` python
+from freqtrade.configuration import Configuration
+config = Configuration.from_files(["config1.json", "config2.json"])
+print(config)
+```
+
 ## Strategy debugging example
 
 Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data.

freqtrade/configuration/configuration.py

@@ -4,7 +4,7 @@ This module contains the configuration class
 import logging
 import warnings
 from argparse import Namespace
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from freqtrade import OperationalException, constants
 from freqtrade.configuration.check_exchange import check_exchange
@@ -39,43 +39,43 @@ class Configuration(object):
 
         return self.config
 
-    def _load_config_files(self) -> Dict[str, Any]:
+    @staticmethod
+    def from_files(files: List[str]) -> Dict[str, Any]:
         """
-        Iterate through the config files passed in the args,
+        Iterate through the config files passed in, loading all of them
-        loading all of them and merging their contents.
+        and merging their contents.
+        Files are loaded in sequence, parameters in later configuration files
+        override the same parameter from an earlier file (last definition wins).
+        :param files: List of file paths
+        :return: configuration dictionary
         """
+        # Keep this method as staticmethod, so it can be used from interactive environments
         config: Dict[str, Any] = {}
 
         # We expect here a list of config filenames
-        for path in self.args.config:
+        for path in files:
-            logger.info('Using config: %s ...', path)
+            logger.info(f'Using config: {path} ...')
 
             # Merge config options, overwriting old values
             config = deep_merge_dicts(load_config_file(path), config)
 
-        return config
+        # Normalize config
-
-    def _normalize_config(self, config: Dict[str, Any]) -> None:
-        """
-        Make config more canonical -- i.e. for example add missing parts that we expect
-        to be normally in it...
-        """
         if 'internals' not in config:
             config['internals'] = {}
 
+        # validate configuration before returning
+        logger.info('Validating configuration ...')
+        validate_config_schema(config)
+
+        return config
+
     def load_config(self) -> Dict[str, Any]:
         """
         Extract information for sys.argv and load the bot configuration
         :return: Configuration dictionary
         """
         # Load all configs
-        config: Dict[str, Any] = self._load_config_files()
+        config: Dict[str, Any] = Configuration.from_files(self.args.config)
-
-        # Make resulting config more canonical
-        self._normalize_config(config)
-
-        logger.info('Validating configuration ...')
-        validate_config_schema(config)
 
         self._validate_config_consistency(config)
 

freqtrade/optimize/default_hyperopt.py

@@ -14,36 +14,48 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt
 class DefaultHyperOpts(IHyperOpt):
     """
     Default hyperopt provided by the Freqtrade bot.
-    You can override it with your own hyperopt
+    You can override it with your own Hyperopt
     """
     @staticmethod
     def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """
+        Add several indicators needed for buy and sell strategies defined below.
+        """
+        # ADX
         dataframe['adx'] = ta.ADX(dataframe)
+        # MACD
         macd = ta.MACD(dataframe)
         dataframe['macd'] = macd['macd']
         dataframe['macdsignal'] = macd['macdsignal']
+        # MFI
         dataframe['mfi'] = ta.MFI(dataframe)
+        # RSI
         dataframe['rsi'] = ta.RSI(dataframe)
+        # Stochastic Fast
         stoch_fast = ta.STOCHF(dataframe)
         dataframe['fastd'] = stoch_fast['fastd']
+        # Minus-DI
         dataframe['minus_di'] = ta.MINUS_DI(dataframe)
         # Bollinger bands
         bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
         dataframe['bb_lowerband'] = bollinger['lower']
         dataframe['bb_upperband'] = bollinger['upper']
+        # SAR
         dataframe['sar'] = ta.SAR(dataframe)
+
         return dataframe
 
     @staticmethod
     def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
         """
-        Define the buy strategy parameters to be used by hyperopt
+        Define the buy strategy parameters to be used by Hyperopt.
         """
         def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
             """
-            Buy strategy Hyperopt will build and use
+            Buy strategy Hyperopt will build and use.
             """
             conditions = []
+
             # GUARDS AND TRENDS
             if 'mfi-enabled' in params and params['mfi-enabled']:
                 conditions.append(dataframe['mfi'] < params['mfi-value'])
@@ -79,7 +91,7 @@ class DefaultHyperOpts(IHyperOpt):
     @staticmethod
     def indicator_space() -> List[Dimension]:
         """
-        Define your Hyperopt space for searching strategy parameters
+        Define your Hyperopt space for searching buy strategy parameters.
         """
         return [
             Integer(10, 25, name='mfi-value'),
@@ -96,14 +108,14 @@ class DefaultHyperOpts(IHyperOpt):
     @staticmethod
     def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
         """
-        Define the sell strategy parameters to be used by hyperopt
+        Define the sell strategy parameters to be used by Hyperopt.
         """
         def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
             """
-            Sell strategy Hyperopt will build and use
+            Sell strategy Hyperopt will build and use.
             """
-            # print(params)
             conditions = []
+
             # GUARDS AND TRENDS
             if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']:
                 conditions.append(dataframe['mfi'] > params['sell-mfi-value'])
@@ -139,7 +151,7 @@ class DefaultHyperOpts(IHyperOpt):
     @staticmethod
     def sell_indicator_space() -> List[Dimension]:
         """
-        Define your Hyperopt space for searching sell strategy parameters
+        Define your Hyperopt space for searching sell strategy parameters.
         """
         return [
             Integer(75, 100, name='sell-mfi-value'),
@@ -157,9 +169,9 @@ class DefaultHyperOpts(IHyperOpt):
 
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Based on TA indicators. Should be a copy of from strategy
+        Based on TA indicators. Should be a copy of same method from strategy.
-        must align to populate_indicators in this file
+        Must align to populate_indicators in this file.
-        Only used when --spaces does not include buy
+        Only used when --spaces does not include buy space.
         """
         dataframe.loc[
             (
@@ -174,9 +186,9 @@ class DefaultHyperOpts(IHyperOpt):
 
     def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Based on TA indicators. Should be a copy of from strategy
+        Based on TA indicators. Should be a copy of same method from strategy.
-        must align to populate_indicators in this file
+        Must align to populate_indicators in this file.
-        Only used when --spaces does not include sell
+        Only used when --spaces does not include sell space.
         """
         dataframe.loc[
             (
@@ -186,4 +198,5 @@ class DefaultHyperOpts(IHyperOpt):
                 (dataframe['fastd'] > 54)
             ),
             'sell'] = 1
+
         return dataframe

freqtrade/tests/test_configuration.py

@@ -133,6 +133,35 @@ def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None:
     assert log_has('Validating configuration ...', caplog)
 
 
+def test_from_config(default_conf, mocker, caplog) -> None:
+    conf1 = deepcopy(default_conf)
+    conf2 = deepcopy(default_conf)
+    del conf1['exchange']['key']
+    del conf1['exchange']['secret']
+    del conf2['exchange']['name']
+    conf2['exchange']['pair_whitelist'] += ['NANO/BTC']
+    conf2['fiat_display_currency'] = "EUR"
+    config_files = [conf1, conf2]
+
+    configsmock = MagicMock(side_effect=config_files)
+    mocker.patch(
+        'freqtrade.configuration.configuration.load_config_file',
+        configsmock
+    )
+
+    validated_conf = Configuration.from_files(['test_conf.json', 'test2_conf.json'])
+
+    exchange_conf = default_conf['exchange']
+    assert validated_conf['exchange']['name'] == exchange_conf['name']
+    assert validated_conf['exchange']['key'] == exchange_conf['key']
+    assert validated_conf['exchange']['secret'] == exchange_conf['secret']
+    assert validated_conf['exchange']['pair_whitelist'] != conf1['exchange']['pair_whitelist']
+    assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist']
+    assert validated_conf['fiat_display_currency'] == "EUR"
+    assert 'internals' in validated_conf
+    assert log_has('Validating configuration ...', caplog)
+
+
 def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> None:
     default_conf['max_open_trades'] = -1
     patched_configuration_load_config_file(mocker, default_conf)

user_data/hyperopts/sample_hyperopt.py

@@ -1,11 +1,10 @@
 # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
 
 from functools import reduce
-from math import exp
 from typing import Any, Callable, Dict, List
 from datetime import datetime
 
-import numpy as np# noqa F401
+import numpy as np
 import talib.abstract as ta
 from pandas import DataFrame
 from skopt.space import Categorical, Dimension, Integer, Real
@@ -16,7 +15,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt
 
 class SampleHyperOpts(IHyperOpt):
     """
-    This is a sample hyperopt to inspire you.
+    This is a sample Hyperopt to inspire you.
     Feel free to customize it.
 
     More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md
@@ -37,32 +36,44 @@ class SampleHyperOpts(IHyperOpt):
     """
     @staticmethod
     def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """
+        Add several indicators needed for buy and sell strategies defined below.
+        """
+        # ADX
         dataframe['adx'] = ta.ADX(dataframe)
+        # MACD
         macd = ta.MACD(dataframe)
         dataframe['macd'] = macd['macd']
         dataframe['macdsignal'] = macd['macdsignal']
+        # MFI
         dataframe['mfi'] = ta.MFI(dataframe)
+        # RSI
         dataframe['rsi'] = ta.RSI(dataframe)
+        # Stochastic Fast
         stoch_fast = ta.STOCHF(dataframe)
         dataframe['fastd'] = stoch_fast['fastd']
+        # Minus-DI
         dataframe['minus_di'] = ta.MINUS_DI(dataframe)
         # Bollinger bands
         bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
         dataframe['bb_lowerband'] = bollinger['lower']
         dataframe['bb_upperband'] = bollinger['upper']
+        # SAR
         dataframe['sar'] = ta.SAR(dataframe)
+
         return dataframe
 
     @staticmethod
     def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
         """
-        Define the buy strategy parameters to be used by hyperopt
+        Define the buy strategy parameters to be used by Hyperopt.
         """
         def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
             """
-            Buy strategy Hyperopt will build and use
+            Buy strategy Hyperopt will build and use.
             """
             conditions = []
+
             # GUARDS AND TRENDS
             if 'mfi-enabled' in params and params['mfi-enabled']:
                 conditions.append(dataframe['mfi'] < params['mfi-value'])
@@ -98,7 +109,7 @@ class SampleHyperOpts(IHyperOpt):
     @staticmethod
     def indicator_space() -> List[Dimension]:
         """
-        Define your Hyperopt space for searching strategy parameters
+        Define your Hyperopt space for searching buy strategy parameters.
         """
         return [
             Integer(10, 25, name='mfi-value'),
@@ -115,14 +126,14 @@ class SampleHyperOpts(IHyperOpt):
     @staticmethod
     def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
         """
-        Define the sell strategy parameters to be used by hyperopt
+        Define the sell strategy parameters to be used by Hyperopt.
         """
         def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
             """
-            Sell strategy Hyperopt will build and use
+            Sell strategy Hyperopt will build and use.
             """
-            # print(params)
             conditions = []
+
             # GUARDS AND TRENDS
             if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']:
                 conditions.append(dataframe['mfi'] > params['sell-mfi-value'])
@@ -158,7 +169,7 @@ class SampleHyperOpts(IHyperOpt):
     @staticmethod
     def sell_indicator_space() -> List[Dimension]:
         """
-        Define your Hyperopt space for searching sell strategy parameters
+        Define your Hyperopt space for searching sell strategy parameters.
         """
         return [
             Integer(75, 100, name='sell-mfi-value'),
@@ -176,9 +187,9 @@ class SampleHyperOpts(IHyperOpt):
 
     def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Based on TA indicators. Should be a copy of from strategy
+        Based on TA indicators. Should be a copy of same method from strategy.
-        must align to populate_indicators in this file
+        Must align to populate_indicators in this file.
-        Only used when --spaces does not include buy
+        Only used when --spaces does not include buy space.
         """
         dataframe.loc[
             (
@@ -193,9 +204,9 @@ class SampleHyperOpts(IHyperOpt):
 
     def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Based on TA indicators. Should be a copy of from strategy
+        Based on TA indicators. Should be a copy of same method from strategy.
-        must align to populate_indicators in this file
+        Must align to populate_indicators in this file.
-        Only used when --spaces does not include sell
+        Only used when --spaces does not include sell space.
         """
         dataframe.loc[
             (
@@ -205,4 +216,5 @@ class SampleHyperOpts(IHyperOpt):
                 (dataframe['fastd'] > 54)
             ),
             'sell'] = 1
+
         return dataframe