From c4cbe79b48795fd7f045f330e76f17d37ab18cd1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Aug 2019 19:55:33 +0200 Subject: [PATCH 1/6] Adjust documentation --- docs/bot-usage.md | 4 ++-- docs/data-analysis.md | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 0ca2f3cc5..4e3f55d91 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -56,8 +56,8 @@ freqtrade -c path/far/far/away/config.json The bot allows you to use multiple configuration files by specifying multiple `-c/--config` configuration options in the command line. Configuration parameters -defined in the last configuration file override parameters with the same name -defined in the previous configuration file specified in the command line. +defined in latter configuration files override parameters with the same name +defined in the earlier configuration files specified in the command line. For example, you can make a separate configuration file with your key and secrete for the Exchange you use for trading, specify default configuration file with diff --git a/docs/data-analysis.md b/docs/data-analysis.md index ecd94445b..c89353cc8 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -31,6 +31,16 @@ df = load_trades_from_db("sqlite:///tradesv3.sqlite") df.groupby("pair")["sell_reason"].value_counts() ``` +### Load multiple configuration files + +This option can be usefull to inspect the results of passing in multiple configs in case of problems + +``` python +from freqtrade.configuration import Configuration +config = Configuration.from_files(["config1.json", "config2.json"]) +print(config) +``` + ## Strategy debugging example Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data. From afba31c3f996c5885fcddf9d572e3a645f3aa026 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Aug 2019 19:57:49 +0200 Subject: [PATCH 2/6] change method from _load_config_Files to from_files() --- freqtrade/configuration/configuration.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e564c79ce..85a156fab 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,7 +4,7 @@ This module contains the configuration class import logging import warnings from argparse import Namespace -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, Optional, List from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange @@ -39,16 +39,22 @@ class Configuration(object): return self.config - def _load_config_files(self) -> Dict[str, Any]: + @staticmethod + def from_files(files: List[str]) -> Dict[str, Any]: """ - Iterate through the config files passed in the args, - loading all of them and merging their contents. + Iterate through the config files passed in, loading all of them + and merging their contents. + Files are loaded in sequence, parameters in later configuration files + override the same parameter from an earlier file (last definition wins). + :param files: List of file paths + :return: configuration dictionary """ + # Keep this method as staticmethod, so it can be used from interactive environments config: Dict[str, Any] = {} # We expect here a list of config filenames - for path in self.args.config: - logger.info('Using config: %s ...', path) + for path in files: + logger.info(f'Using config: {path} ...') # Merge config options, overwriting old values config = deep_merge_dicts(load_config_file(path), config) @@ -69,7 +75,7 @@ class Configuration(object): :return: Configuration dictionary """ # Load all configs - config: Dict[str, Any] = self._load_config_files() + config: Dict[str, Any] = Configuration.from_files(self.args.config) # Make resulting config more canonical self._normalize_config(config) From eb328037b74847c5e911886249191be1a4c30d8b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Aug 2019 19:58:04 +0200 Subject: [PATCH 3/6] combine normalize method and config validation to in_files --- freqtrade/configuration/configuration.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 85a156fab..237346e37 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -4,7 +4,7 @@ This module contains the configuration class import logging import warnings from argparse import Namespace -from typing import Any, Callable, Dict, Optional, List +from typing import Any, Callable, Dict, List, Optional from freqtrade import OperationalException, constants from freqtrade.configuration.check_exchange import check_exchange @@ -59,16 +59,16 @@ class Configuration(object): # Merge config options, overwriting old values config = deep_merge_dicts(load_config_file(path), config) - return config - - def _normalize_config(self, config: Dict[str, Any]) -> None: - """ - Make config more canonical -- i.e. for example add missing parts that we expect - to be normally in it... - """ + # Normalize config if 'internals' not in config: config['internals'] = {} + # validate configuration before returning + logger.info('Validating configuration ...') + validate_config_schema(config) + + return config + def load_config(self) -> Dict[str, Any]: """ Extract information for sys.argv and load the bot configuration @@ -77,12 +77,6 @@ class Configuration(object): # Load all configs config: Dict[str, Any] = Configuration.from_files(self.args.config) - # Make resulting config more canonical - self._normalize_config(config) - - logger.info('Validating configuration ...') - validate_config_schema(config) - self._validate_config_consistency(config) self._process_common_options(config) From 6d89da45b0fe2c4db46ebf3e229ebc8b056527f1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 10 Aug 2019 20:02:11 +0200 Subject: [PATCH 4/6] Add test for from_config --- freqtrade/tests/test_configuration.py | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/freqtrade/tests/test_configuration.py b/freqtrade/tests/test_configuration.py index e325a0de2..667bd042e 100644 --- a/freqtrade/tests/test_configuration.py +++ b/freqtrade/tests/test_configuration.py @@ -133,6 +133,35 @@ def test_load_config_combine_dicts(default_conf, mocker, caplog) -> None: assert log_has('Validating configuration ...', caplog.record_tuples) +def test_from_config(default_conf, mocker, caplog) -> None: + conf1 = deepcopy(default_conf) + conf2 = deepcopy(default_conf) + del conf1['exchange']['key'] + del conf1['exchange']['secret'] + del conf2['exchange']['name'] + conf2['exchange']['pair_whitelist'] += ['NANO/BTC'] + conf2['fiat_display_currency'] = "EUR" + config_files = [conf1, conf2] + + configsmock = MagicMock(side_effect=config_files) + mocker.patch( + 'freqtrade.configuration.configuration.load_config_file', + configsmock + ) + + validated_conf = Configuration.from_files(['test_conf.json', 'test2_conf.json']) + + exchange_conf = default_conf['exchange'] + assert validated_conf['exchange']['name'] == exchange_conf['name'] + assert validated_conf['exchange']['key'] == exchange_conf['key'] + assert validated_conf['exchange']['secret'] == exchange_conf['secret'] + assert validated_conf['exchange']['pair_whitelist'] != conf1['exchange']['pair_whitelist'] + assert validated_conf['exchange']['pair_whitelist'] == conf2['exchange']['pair_whitelist'] + assert validated_conf['fiat_display_currency'] == "EUR" + assert 'internals' in validated_conf + assert log_has('Validating configuration ...', caplog.record_tuples) + + def test_load_config_max_open_trades_minus_one(default_conf, mocker, caplog) -> None: default_conf['max_open_trades'] = -1 patched_configuration_load_config_file(mocker, default_conf) From e5dcd520bae7516c37eba0cd3c12484e1a0210f4 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 12 Aug 2019 02:19:50 +0300 Subject: [PATCH 5/6] cosmetics in sample_hyperopt and default_hyperopt --- freqtrade/optimize/default_hyperopt.py | 41 ++++++++++++++++-------- user_data/hyperopts/sample_hyperopt.py | 44 ++++++++++++++++---------- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index e05dfc95c..2554982ad 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -14,36 +14,48 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt class DefaultHyperOpts(IHyperOpt): """ Default hyperopt provided by the Freqtrade bot. - You can override it with your own hyperopt + You can override it with your own Hyperopt """ @staticmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Add several indicators needed for buy and sell strategies defined below. + """ + # ADX dataframe['adx'] = ta.ADX(dataframe) + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] + # MFI dataframe['mfi'] = ta.MFI(dataframe) + # RSI dataframe['rsi'] = ta.RSI(dataframe) + # Stochastic Fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] + # Minus-DI dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_upperband'] = bollinger['upper'] + # SAR dataframe['sar'] = ta.SAR(dataframe) + return dataframe @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the buy strategy parameters to be used by hyperopt + Define the buy strategy parameters to be used by Hyperopt. """ def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Buy strategy Hyperopt will build and use + Buy strategy Hyperopt will build and use. """ conditions = [] + # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) @@ -79,7 +91,7 @@ class DefaultHyperOpts(IHyperOpt): @staticmethod def indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching strategy parameters + Define your Hyperopt space for searching buy strategy parameters. """ return [ Integer(10, 25, name='mfi-value'), @@ -96,14 +108,14 @@ class DefaultHyperOpts(IHyperOpt): @staticmethod def sell_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the sell strategy parameters to be used by hyperopt + Define the sell strategy parameters to be used by Hyperopt. """ def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Sell strategy Hyperopt will build and use + Sell strategy Hyperopt will build and use. """ - # print(params) conditions = [] + # GUARDS AND TRENDS if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: conditions.append(dataframe['mfi'] > params['sell-mfi-value']) @@ -139,7 +151,7 @@ class DefaultHyperOpts(IHyperOpt): @staticmethod def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching sell strategy parameters + Define your Hyperopt space for searching sell strategy parameters. """ return [ Integer(75, 100, name='sell-mfi-value'), @@ -157,9 +169,9 @@ class DefaultHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include buy + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include buy space. """ dataframe.loc[ ( @@ -174,9 +186,9 @@ class DefaultHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include sell + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include sell space. """ dataframe.loc[ ( @@ -186,4 +198,5 @@ class DefaultHyperOpts(IHyperOpt): (dataframe['fastd'] > 54) ), 'sell'] = 1 + return dataframe diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 1a3823afa..fabfdb23e 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -1,11 +1,10 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement from functools import reduce -from math import exp from typing import Any, Callable, Dict, List from datetime import datetime -import numpy as np# noqa F401 +import numpy as np import talib.abstract as ta from pandas import DataFrame from skopt.space import Categorical, Dimension, Integer, Real @@ -16,7 +15,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt class SampleHyperOpts(IHyperOpt): """ - This is a sample hyperopt to inspire you. + This is a sample Hyperopt to inspire you. Feel free to customize it. More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md @@ -37,32 +36,44 @@ class SampleHyperOpts(IHyperOpt): """ @staticmethod def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Add several indicators needed for buy and sell strategies defined below. + """ + # ADX dataframe['adx'] = ta.ADX(dataframe) + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] + # MFI dataframe['mfi'] = ta.MFI(dataframe) + # RSI dataframe['rsi'] = ta.RSI(dataframe) + # Stochastic Fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] + # Minus-DI dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_upperband'] = bollinger['upper'] + # SAR dataframe['sar'] = ta.SAR(dataframe) + return dataframe @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the buy strategy parameters to be used by hyperopt + Define the buy strategy parameters to be used by Hyperopt. """ def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Buy strategy Hyperopt will build and use + Buy strategy Hyperopt will build and use. """ conditions = [] + # GUARDS AND TRENDS if 'mfi-enabled' in params and params['mfi-enabled']: conditions.append(dataframe['mfi'] < params['mfi-value']) @@ -98,7 +109,7 @@ class SampleHyperOpts(IHyperOpt): @staticmethod def indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching strategy parameters + Define your Hyperopt space for searching buy strategy parameters. """ return [ Integer(10, 25, name='mfi-value'), @@ -115,14 +126,14 @@ class SampleHyperOpts(IHyperOpt): @staticmethod def sell_strategy_generator(params: Dict[str, Any]) -> Callable: """ - Define the sell strategy parameters to be used by hyperopt + Define the sell strategy parameters to be used by Hyperopt. """ def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Sell strategy Hyperopt will build and use + Sell strategy Hyperopt will build and use. """ - # print(params) conditions = [] + # GUARDS AND TRENDS if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: conditions.append(dataframe['mfi'] > params['sell-mfi-value']) @@ -158,7 +169,7 @@ class SampleHyperOpts(IHyperOpt): @staticmethod def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching sell strategy parameters + Define your Hyperopt space for searching sell strategy parameters. """ return [ Integer(75, 100, name='sell-mfi-value'), @@ -176,9 +187,9 @@ class SampleHyperOpts(IHyperOpt): def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include buy + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include buy space. """ dataframe.loc[ ( @@ -193,9 +204,9 @@ class SampleHyperOpts(IHyperOpt): def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators. Should be a copy of from strategy - must align to populate_indicators in this file - Only used when --spaces does not include sell + Based on TA indicators. Should be a copy of same method from strategy. + Must align to populate_indicators in this file. + Only used when --spaces does not include sell space. """ dataframe.loc[ ( @@ -205,4 +216,5 @@ class SampleHyperOpts(IHyperOpt): (dataframe['fastd'] > 54) ), 'sell'] = 1 + return dataframe From 197ce0b67002161557ea4e8bd0ea62f44655f32c Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Aug 2019 06:35:47 +0200 Subject: [PATCH 6/6] Improve documentation wording for multiconfig files --- docs/bot-usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4e3f55d91..988e08029 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -57,7 +57,7 @@ freqtrade -c path/far/far/away/config.json The bot allows you to use multiple configuration files by specifying multiple `-c/--config` configuration options in the command line. Configuration parameters defined in latter configuration files override parameters with the same name -defined in the earlier configuration files specified in the command line. +defined in the previous configuration files specified in the command line earlier. For example, you can make a separate configuration file with your key and secrete for the Exchange you use for trading, specify default configuration file with