From 68ba1e1f3741d248ba21ff05705000d47ca8f9b1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:16:30 +0100 Subject: [PATCH 01/42] Add sell signal hyperopt --- freqtrade/arguments.py | 2 +- freqtrade/optimize/default_hyperopt.py | 61 ++++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 8 +++- freqtrade/optimize/hyperopt_interface.py | 14 ++++++ user_data/hyperopts/sample_hyperopt.py | 61 ++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 2 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 19cccfe8b..b86d3502a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -272,7 +272,7 @@ class Arguments(object): '-s', '--spaces', help='Specify which parameters to hyperopt. Space separate list. \ Default: %(default)s', - choices=['all', 'buy', 'roi', 'stoploss'], + choices=['all', 'buy', 'sell', 'roi', 'stoploss'], default='all', nargs='+', dest='spaces', diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 6139f8140..1c2d92950 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -93,6 +93,67 @@ class DefaultHyperOpts(IHyperOpt): Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] + @staticmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the sell strategy parameters to be used by hyperopt + """ + def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Sell strategy Hyperopt will build and use + """ + # print(params) + conditions = [] + # GUARDS AND TRENDS + if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: + conditions.append(dataframe['mfi'] > params['sell-mfi-value']) + if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: + conditions.append(dataframe['fastd'] > params['sell-fastd-value']) + if 'sell-adx-enabled' in params and params['sell-adx-enabled']: + conditions.append(dataframe['adx'] > params['sell-adx-value']) + if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: + conditions.append(dataframe['rsi'] > params['sell-rsi-value']) + + # TRIGGERS + if 'sell-trigger' in params: + if params['sell-trigger'] == 'sell-bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['sell-trigger'] == 'sell-sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + + return dataframe + + return populate_sell_trend + + @staticmethod + def sell_indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching sell strategy parameters + """ + return [ + Integer(75, 100, name='sell-mfi-value'), + Integer(50, 100, name='sell-fastd-value'), + Integer(50, 100, name='sell-adx-value'), + Integer(60, 100, name='sell-rsi-value'), + Categorical([True, False], name='sell-mfi-enabled'), + Categorical([True, False], name='sell-fastd-enabled'), + Categorical([True, False], name='sell-adx-enabled'), + Categorical([True, False], name='sell-rsi-enabled'), + Categorical(['sell-bb_lower', + 'sell-macd_cross_signal', + 'sell-sar_reversal'], name='sell-trigger') + ] + @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: """ diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2d08fec81..674f7d3d1 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -151,6 +151,9 @@ class Hyperopt(Backtesting): spaces: List[Dimension] = [] if self.has_space('buy'): spaces += self.custom_hyperopt.indicator_space() + if self.has_space('sell'): + spaces += self.custom_hyperopt.sell_indicator_space() + self.config['experimental']['use_sell_signal'] = True if self.has_space('roi'): spaces += self.custom_hyperopt.roi_space() if self.has_space('stoploss'): @@ -165,6 +168,9 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) + if self.has_space('sell'): + self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) + if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -247,7 +253,7 @@ class Hyperopt(Backtesting): timerange=timerange ) - if self.has_space('buy'): + if self.has_space('buy') or self.has_space('sell'): self.strategy.advise_indicators = \ self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index d42206658..622de3015 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -37,6 +37,13 @@ class IHyperOpt(ABC): Create a buy strategy generator """ + @staticmethod + @abstractmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Create a sell strategy generator + """ + @staticmethod @abstractmethod def indicator_space() -> List[Dimension]: @@ -44,6 +51,13 @@ class IHyperOpt(ABC): Create an indicator space """ + @staticmethod + @abstractmethod + def sell_indicator_space() -> List[Dimension]: + """ + Create a sell indicator space + """ + @staticmethod @abstractmethod def generate_roi_table(params: Dict) -> Dict[int, float]: diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index f11236a82..ee4e4dca7 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -102,6 +102,67 @@ class SampleHyperOpts(IHyperOpt): Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') ] + @staticmethod + def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the sell strategy parameters to be used by hyperopt + """ + def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Sell strategy Hyperopt will build and use + """ + # print(params) + conditions = [] + # GUARDS AND TRENDS + if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: + conditions.append(dataframe['mfi'] > params['sell-mfi-value']) + if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: + conditions.append(dataframe['fastd'] > params['sell-fastd-value']) + if 'sell-adx-enabled' in params and params['sell-adx-enabled']: + conditions.append(dataframe['adx'] > params['sell-adx-value']) + if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: + conditions.append(dataframe['rsi'] > params['sell-rsi-value']) + + # TRIGGERS + if 'sell-trigger' in params: + if params['sell-trigger'] == 'sell-bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['sell-trigger'] == 'sell-sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + + return dataframe + + return populate_sell_trend + + @staticmethod + def sell_indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching sell strategy parameters + """ + return [ + Integer(75, 100, name='sell-mfi-value'), + Integer(50, 100, name='sell-fastd-value'), + Integer(50, 100, name='sell-adx-value'), + Integer(60, 100, name='sell-rsi-value'), + Categorical([True, False], name='sell-mfi-enabled'), + Categorical([True, False], name='sell-fastd-enabled'), + Categorical([True, False], name='sell-adx-enabled'), + Categorical([True, False], name='sell-rsi-enabled'), + Categorical(['sell-bb_lower', + 'sell-macd_cross_signal', + 'sell-sar_reversal'], name='sell-trigger') + ] + @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: """ From 5e08769366a298517a56249806dc3c61af7ab930 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:27:23 +0100 Subject: [PATCH 02/42] Update hyperopt documentation with sell-stuff --- docs/hyperopt.md | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 58dc91e3a..06b90cff9 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -11,30 +11,29 @@ and still take a long time. ## Prepare Hyperopting -Before we start digging in Hyperopt, we recommend you to take a look at +Before we start digging into Hyperopt, we recommend you to take a look at an example hyperopt file located into [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) -### 1. Install a Custom Hyperopt File -This is very simple. Put your hyperopt file into the folder -`user_data/hyperopts`. +Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar and a lot of code can be copied across from the strategy. -Let assume you want a hyperopt file `awesome_hyperopt.py`:
+### 1. Install a Custom Hyperopt File + +Put your hyperopt file into the folder`user_data/hyperopts`. + +Let assume you want a hyperopt file `awesome_hyperopt.py`: Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` - ### 2. Configure your Guards and Triggers -There are two places you need to change in your hyperopt file to add a -new buy hyperopt for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). + +There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: + +- Inside `indicator_space()` - the parameters hyperopt shall be optimizing. +- Inside `populate_buy_trend()` - applying the parameters. There you have two different types of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or never buy if -current price is over EMA10. -2. Triggers are ones that actually trigger buy in specific moment, like -"buy when EMA5 crosses over EMA10" or "buy when close price touches lower -bollinger band". +1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. +2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like @@ -45,6 +44,17 @@ If you have updated the buy strategy, ie. changed the contents of `populate_buy_trend()` method you have to update the `guards` and `triggers` hyperopts must use. +#### Sell optimization + +Similar to the buy-signal above, sell-signals can also be optimized. +Place the corresponding settings into the following methods + +- Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. +- Inside `populate_sell_trend()` - applying the parameters. + +The configuration and rules are the same than for buy signals. +To avoid naming collisions in the search-space, please prefix all sell-spaces with sell-. + ## Solving a Mystery Let's say you are curious: should you use MACD crossings or lower Bollinger @@ -125,11 +135,10 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py --hyperopt -c config.json hyperopt -e 5000 ``` -Use `` and `` as the names of the custom strategy -(only required for generating sells) and the custom hyperopt used. +Use `` as the name of the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. @@ -162,6 +171,7 @@ Legal values are: - `all`: optimize everything - `buy`: just search for a new buy strategy +- `sell`: just search for a new sell strategy - `roi`: just optimize the minimal profit table for your strategy - `stoploss`: search for the best stoploss value - space-separated list of any of the above values for example `--spaces roi stoploss` @@ -237,6 +247,7 @@ Once the optimized strategy has been implemented into your strategy, you should To archive the same results (number of trades, ...) than during hyperopt, please use the command line flag `--disable-max-market-positions`. This setting is the default for hyperopt for speed reasons. You can overwrite this in the configuration by setting `"position_stacking"=false` or by changing the relevant line in your hyperopt file [here](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L283). +!!! Note: Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. ## Next Step From 798ae460d8f8c91435f4e0716c50365cdf11b479 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:30:58 +0100 Subject: [PATCH 03/42] Add check if trigger is in parameters --- docs/hyperopt.md | 13 +++++++------ freqtrade/optimize/default_hyperopt.py | 21 +++++++++++---------- user_data/hyperopts/sample_hyperopt.py | 21 +++++++++++---------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 06b90cff9..c4dd76009 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -98,12 +98,13 @@ So let's write the buy strategy using these values: conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 1c2d92950..315a1e8a5 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -57,16 +57,17 @@ class DefaultHyperOpts(IHyperOpt): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index ee4e4dca7..1700af932 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -66,16 +66,17 @@ class SampleHyperOpts(IHyperOpt): conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), From 2147bd88470dba08b047f76ef6c50f0281a11000 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 10:35:18 +0100 Subject: [PATCH 04/42] Fix problem when no experimental dict is available --- freqtrade/optimize/hyperopt.py | 3 +++ freqtrade/tests/optimize/test_hyperopt.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 674f7d3d1..ba9c186da 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -153,6 +153,9 @@ class Hyperopt(Backtesting): spaces += self.custom_hyperopt.indicator_space() if self.has_space('sell'): spaces += self.custom_hyperopt.sell_indicator_space() + # Make sure experimental is enabled + if 'experimental' not in self.config: + self.config['experimental'] = {} self.config['experimental']['use_sell_signal'] = True if self.has_space('roi'): spaces += self.custom_hyperopt.roi_space() diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 53d991c09..5c6d3981a 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -312,6 +312,15 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'mfi-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', + 'sell-adx-value': 0, + 'sell-fastd-value': 75, + 'sell-mfi-value': 0, + 'sell-rsi-value': 0, + 'sell-adx-enabled': False, + 'sell-fastd-enabled': True, + 'sell-mfi-enabled': False, + 'sell-rsi-enabled': False, + 'sell-trigger': 'macd_cross_signal', 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, From a0df7b9d7cfa6e5e8d5c841724e5b1a0fcc661e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:12:55 +0100 Subject: [PATCH 05/42] Use sell/buy trends from hyperopt file if available --- freqtrade/optimize/hyperopt.py | 4 ++++ freqtrade/resolvers/hyperopt_resolver.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ba9c186da..b8b8aaf56 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -170,9 +170,13 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) + elif hasattr(self.custom_hyperopt, 'populate_buy_trend'): + self.advise_buy = self.custom_hyperopt.populate_buy_trend if self.has_space('sell'): self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) + elif hasattr(self.custom_hyperopt, 'populate_sell_trend'): + self.advise_sell = self.custom_hyperopt.populate_sell_trend if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index eb91c0e89..6bf7fa17d 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -32,6 +32,13 @@ class HyperOptResolver(IResolver): hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + if not hasattr(self.hyperopt, 'populate_buy_trend'): + logger.warning("Custom Hyperopt does not provide populate_buy_trend. " + "Using populate_buy_trend from DefaultStrategy.") + if not hasattr(self.hyperopt, 'populate_sell_trend'): + logger.warning("Custom Hyperopt does not provide populate_sell_trend. " + "Using populate_sell_trend from DefaultStrategy.") + def _load_hyperopt( self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: """ From 167088827adbd03f7799afebe62d6f80f3c7207c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:13:15 +0100 Subject: [PATCH 06/42] include default buy/sell trends for the hyperopt strategy --- freqtrade/optimize/default_hyperopt.py | 46 ++++++++++++++++++++++---- user_data/hyperopts/sample_hyperopt.py | 46 ++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 315a1e8a5..721848d2e 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -33,6 +33,7 @@ class DefaultHyperOpts(IHyperOpt): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_upperband'] = bollinger['upper'] dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -111,21 +112,21 @@ class DefaultHyperOpts(IHyperOpt): if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: conditions.append(dataframe['fastd'] > params['sell-fastd-value']) if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] > params['sell-adx-value']) + conditions.append(dataframe['adx'] < params['sell-adx-value']) if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: conditions.append(dataframe['rsi'] > params['sell-rsi-value']) # TRIGGERS if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-bb_upper': + conditions.append(dataframe['close'] > dataframe['bb_upperband']) if params['sell-trigger'] == 'sell-macd_cross_signal': conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] + dataframe['macdsignal'], dataframe['macd'] )) if params['sell-trigger'] == 'sell-sar_reversal': conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] + dataframe['sar'], dataframe['close'] )) dataframe.loc[ @@ -150,7 +151,7 @@ class DefaultHyperOpts(IHyperOpt): Categorical([True, False], name='sell-fastd-enabled'), Categorical([True, False], name='sell-adx-enabled'), Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_lower', + Categorical(['sell-bb_upper', 'sell-macd_cross_signal', 'sell-sar_reversal'], name='sell-trigger') ] @@ -190,3 +191,36 @@ class DefaultHyperOpts(IHyperOpt): Real(0.01, 0.07, name='roi_p2'), Real(0.01, 0.20, name='roi_p3'), ] + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include buy + """ + dataframe.loc[ + ( + (dataframe['close'] < dataframe['bb_lowerband']) & + (dataframe['mfi'] < 16) & + (dataframe['adx'] > 25) & + (dataframe['rsi'] < 21) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include sell + """ + dataframe.loc[ + ( + (qtpylib.crossed_above( + dataframe['macdsignal'], dataframe['macd'] + )) & + (dataframe['fastd'] > 54) + ), + 'sell'] = 1 + return dataframe diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 1700af932..54f65a7e6 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -42,6 +42,7 @@ class SampleHyperOpts(IHyperOpt): # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_upperband'] = bollinger['upper'] dataframe['sar'] = ta.SAR(dataframe) return dataframe @@ -120,21 +121,21 @@ class SampleHyperOpts(IHyperOpt): if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: conditions.append(dataframe['fastd'] > params['sell-fastd-value']) if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] > params['sell-adx-value']) + conditions.append(dataframe['adx'] < params['sell-adx-value']) if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: conditions.append(dataframe['rsi'] > params['sell-rsi-value']) # TRIGGERS if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['sell-trigger'] == 'sell-bb_upper': + conditions.append(dataframe['close'] > dataframe['bb_upperband']) if params['sell-trigger'] == 'sell-macd_cross_signal': conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] + dataframe['macdsignal'], dataframe['macd'] )) if params['sell-trigger'] == 'sell-sar_reversal': conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] + dataframe['sar'], dataframe['close'] )) dataframe.loc[ @@ -159,7 +160,7 @@ class SampleHyperOpts(IHyperOpt): Categorical([True, False], name='sell-fastd-enabled'), Categorical([True, False], name='sell-adx-enabled'), Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_lower', + Categorical(['sell-bb_upper', 'sell-macd_cross_signal', 'sell-sar_reversal'], name='sell-trigger') ] @@ -199,3 +200,36 @@ class SampleHyperOpts(IHyperOpt): Real(0.01, 0.07, name='roi_p2'), Real(0.01, 0.20, name='roi_p3'), ] + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include buy + """ + dataframe.loc[ + ( + (dataframe['close'] < dataframe['bb_lowerband']) & + (dataframe['mfi'] < 16) & + (dataframe['adx'] > 25) & + (dataframe['rsi'] < 21) + ), + 'buy'] = 1 + + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Based on TA indicators. Should be a copy of from strategy + must align to populate_indicators in this file + Only used when --spaces does not include sell + """ + dataframe.loc[ + ( + (qtpylib.crossed_above( + dataframe['macdsignal'], dataframe['macd'] + )) & + (dataframe['fastd'] > 54) + ), + 'sell'] = 1 + return dataframe From dd2af86a4187afc389f26c2602d0190b948ecef8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:47:38 +0100 Subject: [PATCH 07/42] pprint results --- freqtrade/optimize/hyperopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b8b8aaf56..7d6a991d2 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -5,17 +5,18 @@ This module contains the hyperopt logic """ import logging -from argparse import Namespace +import multiprocessing import os import sys -from pathlib import Path +from argparse import Namespace from math import exp -import multiprocessing from operator import itemgetter +from pathlib import Path +from pprint import pprint from typing import Any, Dict, List -from pandas import DataFrame from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects +from pandas import DataFrame from skopt import Optimizer from skopt.space import Dimension @@ -26,7 +27,6 @@ from freqtrade.optimize import get_timeframe from freqtrade.optimize.backtesting import Backtesting from freqtrade.resolvers import HyperOptResolver - logger = logging.getLogger(__name__) MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization @@ -104,11 +104,11 @@ class Hyperopt(Backtesting): logger.info( 'Best result:\n%s\nwith values:\n%s', best_result['result'], - best_result['params'] + pprint(best_result['params'], indent=4) ) if 'roi_t1' in best_result['params']: logger.info('ROI table:\n%s', - self.custom_hyperopt.generate_roi_table(best_result['params'])) + pprint(self.custom_hyperopt.generate_roi_table(best_result['params']), indent=4)) def log_results(self, results) -> None: """ From 5dd1f9b38a8f9a2ee5feecfe25e7828f3305a921 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:47:53 +0100 Subject: [PATCH 08/42] improve hyperopt docs --- docs/hyperopt.md | 58 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index c4dd76009..d17464d32 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -16,6 +16,22 @@ an example hyperopt file located into [user_data/hyperopts/](https://github.com/ Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar and a lot of code can be copied across from the strategy. +### Checklist on all tasks / possibilities in hyperopt + +Depending on the space you want to optimize, only some of the below are required. + +* fill `populate_indicators` - probably a copy from your strategy +* fill `buy_strategy_generator` - for buy signal optimization +* fill `indicator_space` - for buy signal optimzation +* fill `sell_strategy_generator` - for sell signal optimization +* fill `sell_indicator_space` - for sell signal optimzation +* fill `roi_space` - for ROI optimization +* fill `generate_roi_table` - for ROI optimization (if you need more than 3 entries) +* fill `stoploss_space` - stoploss optimization +* Optional but reccomended + * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used + * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used + ### 1. Install a Custom Hyperopt File Put your hyperopt file into the folder`user_data/hyperopts`. @@ -49,11 +65,11 @@ If you have updated the buy strategy, ie. changed the contents of Similar to the buy-signal above, sell-signals can also be optimized. Place the corresponding settings into the following methods -- Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -- Inside `populate_sell_trend()` - applying the parameters. +* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. +* Inside `populate_sell_trend()` - applying the parameters. The configuration and rules are the same than for buy signals. -To avoid naming collisions in the search-space, please prefix all sell-spaces with sell-. +To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. ## Solving a Mystery @@ -65,7 +81,7 @@ mystery. We will start by defining a search space: -``` +```python def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters @@ -88,7 +104,7 @@ one we call `trigger` and use it to decide which buy trigger we want to use. So let's write the buy strategy using these values: -``` +``` python def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS @@ -136,7 +152,7 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py --hyperopt -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py --hyperopt -c config.json hyperopt -e 5000 --spaces all ``` Use `` as the name of the custom hyperopt used. @@ -144,6 +160,11 @@ Use `` as the name of the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. +The `--spaces all` flag determines that all possible parameters should be optimized. Possibilities are listed below. + +!!! Warning +When switching parameters or changing configuration options, the file `user_data/hyperopt_results.pickle` should be removed. It's used to be able to continue interrupted calculations, but does not detect changes to settings or the hyperopt file. + ### Execute Hyperopt with Different Ticker-Data Source If you would like to hyperopt parameters using an alternate ticker data that @@ -186,7 +207,11 @@ Given the following result from hyperopt: Best result: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. with values: -{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower'} +{ 'adx-value': 44, + 'rsi-value': 29, + 'adx-enabled': False, + 'rsi-enabled': True, + 'trigger': 'bb_lower'} ``` You should understand this result like: @@ -226,9 +251,24 @@ If you are optimizing ROI, you're result will look as follows and include a ROI Best result: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. with values: -{'adx-value': 44, 'rsi-value': 29, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'bb_lower', 'roi_t1': 40, 'roi_t2': 57, 'roi_t3': 21, 'roi_p1': 0.03634636907306948, 'roi_p2': 0.055237357937802885, 'roi_p3': 0.015163796015548354, 'stoploss': -0.37996664668703606} +{ 'adx-value': 44, + 'rsi-value': 29, + 'adx-enabled': false, + 'rsi-enabled': True, + 'trigger': 'bb_lower', + 'roi_t1': 40, + 'roi_t2': 57, + 'roi_t3': 21, + 'roi_p1': 0.03634636907306948, + 'roi_p2': 0.055237357937802885, + 'roi_p3': 0.015163796015548354, + 'stoploss': -0.37996664668703606 +} ROI table: -{0: 0.10674752302642071, 21: 0.09158372701087236, 78: 0.03634636907306948, 118: 0} +{ 0: 0.10674752302642071, + 21: 0.09158372701087236, + 78: 0.03634636907306948, + 118: 0} ``` This would translate to the following ROI table: From 40b1d8f0678ce320f44ed25cbb645233be8e6336 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 14:57:14 +0100 Subject: [PATCH 09/42] Fix CI problems --- freqtrade/optimize/hyperopt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7d6a991d2..6930bed04 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -102,13 +102,13 @@ class Hyperopt(Backtesting): results = sorted(self.trials, key=itemgetter('loss')) best_result = results[0] logger.info( - 'Best result:\n%s\nwith values:\n%s', - best_result['result'], - pprint(best_result['params'], indent=4) + 'Best result:\n%s\nwith values:\n', + best_result['result'] ) + pprint(best_result['params'], indent=4) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', - pprint(self.custom_hyperopt.generate_roi_table(best_result['params']), indent=4)) + logger.info('ROI table:') + pprint(self.custom_hyperopt.generate_roi_table(best_result['params']), indent=4) def log_results(self, results) -> None: """ @@ -171,12 +171,12 @@ class Hyperopt(Backtesting): if self.has_space('buy'): self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) elif hasattr(self.custom_hyperopt, 'populate_buy_trend'): - self.advise_buy = self.custom_hyperopt.populate_buy_trend + self.advise_buy = self.custom_hyperopt.populate_buy_trend # type: ignore if self.has_space('sell'): self.advise_sell = self.custom_hyperopt.sell_strategy_generator(params) elif hasattr(self.custom_hyperopt, 'populate_sell_trend'): - self.advise_sell = self.custom_hyperopt.populate_sell_trend + self.advise_sell = self.custom_hyperopt.populate_sell_trend # type: ignore if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] From 440a7ec9c2d287abf58168c5affcf88e6eda1569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 19:31:25 +0100 Subject: [PATCH 10/42] fix pytest --- freqtrade/tests/optimize/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index 5c6d3981a..d816988a8 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -201,7 +201,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: hyperopt.start() parallel.assert_called_once() - assert 'Best result:\nfoo result\nwith values:\n{}' in caplog.text + assert 'Best result:\nfoo result\nwith values:\n\n' in caplog.text assert dumper.called From f620449bec59b4cb1df8eb5f50f458ab0a922a75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jan 2019 19:38:32 +0100 Subject: [PATCH 11/42] Add test for hyperoptresolver --- freqtrade/tests/optimize/test_hyperopt.py | 25 ++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index d816988a8..20baee99e 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -9,7 +9,8 @@ import pytest from freqtrade.data.converter import parse_ticker_dataframe from freqtrade.data.history import load_tickerdata_file from freqtrade.optimize.hyperopt import Hyperopt, start -from freqtrade.resolvers import StrategyResolver +from freqtrade.optimize.default_hyperopt import DefaultHyperOpts +from freqtrade.resolvers import StrategyResolver, HyperOptResolver from freqtrade.tests.conftest import log_has, patch_exchange from freqtrade.tests.optimize.test_backtesting import get_args @@ -38,6 +39,28 @@ def create_trials(mocker, hyperopt) -> None: return [{'loss': 1, 'result': 'foo', 'params': {}}] +def test_hyperoptresolver(mocker, default_conf, caplog) -> None: + + mocker.patch( + 'freqtrade.configuration.Configuration._load_config_file', + lambda *args, **kwargs: default_conf + ) + hyperopts = DefaultHyperOpts + delattr(hyperopts, 'populate_buy_trend') + delattr(hyperopts, 'populate_sell_trend') + mocker.patch( + 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt', + MagicMock(return_value=hyperopts) + ) + x = HyperOptResolver(default_conf, ).hyperopt + assert not hasattr(x, 'populate_buy_trend') + assert not hasattr(x, 'populate_sell_trend') + assert log_has("Custom Hyperopt does not provide populate_sell_trend. " + "Using populate_sell_trend from DefaultStrategy.", caplog.record_tuples) + assert log_has("Custom Hyperopt does not provide populate_buy_trend. " + "Using populate_buy_trend from DefaultStrategy.", caplog.record_tuples) + + def test_start(mocker, default_conf, caplog) -> None: start_mock = MagicMock() mocker.patch( From b98526d32c47d71b1dc1aa5c6f6e700012cbfee6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 8 Jan 2019 13:33:06 +0100 Subject: [PATCH 12/42] Update ccxt from 1.18.102 to 1.18.114 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b07c450f3..87ecc53f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.102 +ccxt==1.18.114 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.12.1 From c3107272d34ee7893e8611948733e7c2961d3e7f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 8 Jan 2019 13:33:08 +0100 Subject: [PATCH 13/42] Update arrow from 0.12.1 to 0.13.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87ecc53f6..b0d8e2563 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ ccxt==1.18.114 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 -arrow==0.12.1 +arrow==0.13.0 cachetools==3.0.0 requests==2.21.0 urllib3==1.24.1 From ec22512fd95a95473a661f260b960dfe26809b9c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 8 Jan 2019 13:33:10 +0100 Subject: [PATCH 14/42] Update pytest from 4.0.2 to 4.1.0 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4a7416ca1..b2248c9c6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ -r requirements.txt flake8==3.6.0 -pytest==4.0.2 +pytest==4.1.0 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From 4069e2fdfb600d45901054f0ba4cb54e2a062d26 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 8 Jan 2019 13:33:11 +0100 Subject: [PATCH 15/42] Update pytest-asyncio from 0.9.0 to 0.10.0 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b2248c9c6..87be5d195 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,5 +4,5 @@ flake8==3.6.0 pytest==4.1.0 pytest-mock==1.10.0 -pytest-asyncio==0.9.0 +pytest-asyncio==0.10.0 pytest-cov==2.6.0 From f4e0e044622c0a5d77069fbb854a8bec5fbcd7be Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 8 Jan 2019 13:33:13 +0100 Subject: [PATCH 16/42] Update pytest-cov from 2.6.0 to 2.6.1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 87be5d195..2eeae1fbc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ flake8==3.6.0 pytest==4.1.0 pytest-mock==1.10.0 pytest-asyncio==0.10.0 -pytest-cov==2.6.0 +pytest-cov==2.6.1 From b5adfcf51a49da8ac65979f4af32e5f94a16a715 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 17:07:32 +0100 Subject: [PATCH 17/42] Fix documentation typo --- docs/hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index d17464d32..0c18110bd 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -28,7 +28,7 @@ Depending on the space you want to optimize, only some of the below are required * fill `roi_space` - for ROI optimization * fill `generate_roi_table` - for ROI optimization (if you need more than 3 entries) * fill `stoploss_space` - stoploss optimization -* Optional but reccomended +* Optional but recommended * copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used * copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used From b3f67bb8c6289c43a38865ff3abf36f5eef80c4c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 20:09:32 +0100 Subject: [PATCH 18/42] Fix git pull in docker-image --- build_helpers/publish_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 9d82fc2d5..7a8127c44 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -13,7 +13,7 @@ if [ "${TRAVIS_EVENT_TYPE}" = "cron" ]; then else echo "event ${TRAVIS_EVENT_TYPE}: building with cache" # Pull last build to avoid rebuilding the whole image - docker pull ${REPO}:${TAG} + docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . fi From 64372ea6fb1b02f6ec2e709b2b21652c64081967 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 20:12:39 +0100 Subject: [PATCH 19/42] Fix ta-lib installation build helpers --- build_helpers/install_ta-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 4d4f37c17..95119a51f 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -1,5 +1,5 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then - tar zxvf ta-lib-0.4.0-src.tar.gz + tar zxvf build_helpers/ta-lib-0.4.0-src.tar.gz cd ta-lib \ && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ && ./configure \ From df97652f6e07ddb24b3cda697a469acf0797224d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 20:35:08 +0100 Subject: [PATCH 20/42] Try fix ta-lib install --- .travis.yml | 2 +- build_helpers/install_ta-lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4398a1386..f448c72d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ addons: - libdw-dev - binutils-dev install: -- ./build_helpers/install_ta-lib.sh +- cd build_helpers && ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements-dev.txt diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 95119a51f..4d4f37c17 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -1,5 +1,5 @@ if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then - tar zxvf build_helpers/ta-lib-0.4.0-src.tar.gz + tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib \ && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ && ./configure \ From 69eed95a54f6dc2aaa9595f2e963ed6e6423d578 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 20:39:50 +0100 Subject: [PATCH 21/42] cd out of the build-helpers dir --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f448c72d8..503295e4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ addons: - libdw-dev - binutils-dev install: -- cd build_helpers && ./install_ta-lib.sh +- cd build_helpers && ./install_ta-lib.sh; cd .. - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy - pip install -r requirements-dev.txt From da436c920f6499f43085c6a8658353b633616c17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 20:58:14 +0100 Subject: [PATCH 22/42] switch travis to xenial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 503295e4a..c389189b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: true os: - linux -dist: trusty +dist: xenial language: python python: - 3.6 From 356a17cdaaf8753875527b5d8d609f3619ddd89c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 21:07:08 +0100 Subject: [PATCH 23/42] Build python 3.7 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c389189b9..de1c6a139 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ dist: xenial language: python python: - 3.6 +- 3.7 services: - docker env: From bb3d78757d763535673823268d24eabf957f2f43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 21:11:36 +0100 Subject: [PATCH 24/42] Test python 3.7 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index de1c6a139..495fe3726 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,9 @@ jobs: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls name: pytest + python: + - 3.6 + - 3.7 - script: - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting From 7a13565efb54618b0564d0c4dac4b00657efec86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 21:13:56 +0100 Subject: [PATCH 25/42] travis - python - test --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 495fe3726..4617aa19e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ dist: xenial language: python python: - 3.6 -- 3.7 services: - docker env: From f9a99f4ad33dc7b77a31d89812bfe4f3a42dae61 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 21:27:50 +0100 Subject: [PATCH 26/42] Change caching --- .travis.yml | 4 ++-- build_helpers/install_ta-lib.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4617aa19e..7e2e9ac34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,6 @@ jobs: - coveralls name: pytest python: - - 3.6 - 3.7 - script: - cp config.json.example config.json @@ -56,6 +55,7 @@ notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= cache: + pip: True directories: - - $HOME/.cache/pip - ta-lib + - /usr/local/lib/ diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 4d4f37c17..4f46a5784 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -1,4 +1,4 @@ -if [ ! -f "ta-lib/CHANGELOG.TXT" ]; then +if [ ! -f "/usr/local/lib/libta_lib.a" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib \ && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ From dd7d655a63abd90430f39f6859e626add15fbe89 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jan 2019 21:30:11 +0100 Subject: [PATCH 27/42] remove unwriteable cache-dir --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7e2e9ac34..c995ec700 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,4 +58,3 @@ cache: pip: True directories: - ta-lib - - /usr/local/lib/ From f4979e0e8a917ed7a41d33cd8222ba58e2909aae Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Jan 2019 06:27:58 +0100 Subject: [PATCH 28/42] Cache /usr/loca/lib --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c995ec700..4c453a873 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,6 @@ jobs: - pytest --cov=freqtrade --cov-config=.coveragerc freqtrade/tests/ - coveralls name: pytest - python: - - 3.7 - script: - cp config.json.example config.json - python freqtrade/main.py --datadir freqtrade/tests/testdata backtesting @@ -58,3 +56,4 @@ cache: pip: True directories: - ta-lib + - /usr/local/lib From e76ed31b0863b8521a9f9eb9c78a2099ff353913 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Jan 2019 08:15:36 +0100 Subject: [PATCH 29/42] fix ta-lib cache --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c453a873..c39817aa1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,5 +55,4 @@ notifications: cache: pip: True directories: - - ta-lib - - /usr/local/lib + - build_helpers/ta-lib From 2e530a3e0382e86ff41043f395cc0df51aca5709 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Jan 2019 08:16:33 +0100 Subject: [PATCH 30/42] Update install_ta-lib.sh --- build_helpers/install_ta-lib.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 4f46a5784..9fe341bba 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -7,7 +7,5 @@ if [ ! -f "/usr/local/lib/libta_lib.a" ]; then && which sudo && sudo make install || make install \ && cd .. else - echo "TA-lib already installed, skipping download and build." - cd ta-lib && sudo make install && cd .. - + echo "TA-lib already installed, skipping installation" fi From a4947554492949e2d9dc728593312b73edb4d309 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Jan 2019 08:17:03 +0100 Subject: [PATCH 31/42] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c39817aa1..57265fd40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,4 +55,4 @@ notifications: cache: pip: True directories: - - build_helpers/ta-lib + - /usr/local/lib From c1007f95b32cdd622c124e663081a797941f69a3 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 9 Jan 2019 13:32:07 +0100 Subject: [PATCH 32/42] Update ccxt from 1.18.114 to 1.18.117 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0d8e2563..25c441d73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.114 +ccxt==1.18.117 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.13.0 From 3f8092192ebc664102463894d56fd18dbe825c58 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 10 Jan 2019 13:32:07 +0100 Subject: [PATCH 33/42] Update ccxt from 1.18.117 to 1.18.119 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 25c441d73..b847fb1a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.117 +ccxt==1.18.119 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.13.0 From 4920ee34554512cfc56ba32bd0145929881dbfe4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Thu, 10 Jan 2019 13:32:09 +0100 Subject: [PATCH 34/42] Update wrapt from 1.10.11 to 1.11.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b847fb1a9..30feb1f6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ arrow==0.13.0 cachetools==3.0.0 requests==2.21.0 urllib3==1.24.1 -wrapt==1.10.11 +wrapt==1.11.0 pandas==0.23.4 scikit-learn==0.20.2 joblib==0.13.0 From 3867f73c8ca0f9d83aa5656e05cdc1e390dc08cd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 11 Jan 2019 13:32:07 +0100 Subject: [PATCH 35/42] Update ccxt from 1.18.119 to 1.18.120 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 30feb1f6c..380185348 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.119 +ccxt==1.18.120 SQLAlchemy==1.2.15 python-telegram-bot==11.1.0 arrow==0.13.0 From e95351fd0407ade72f475943178d8023b70ed5da Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 12 Jan 2019 13:32:06 +0100 Subject: [PATCH 36/42] Update sqlalchemy from 1.2.15 to 1.2.16 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 380185348..d7c4e2bcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.18.120 -SQLAlchemy==1.2.15 +SQLAlchemy==1.2.16 python-telegram-bot==11.1.0 arrow==0.13.0 cachetools==3.0.0 From 690fbeb9074b9779dcfe360311732c3efbbe16f9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 12 Jan 2019 13:32:08 +0100 Subject: [PATCH 37/42] Update joblib from 0.13.0 to 0.13.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d7c4e2bcf..505bebc7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ urllib3==1.24.1 wrapt==1.11.0 pandas==0.23.4 scikit-learn==0.20.2 -joblib==0.13.0 +joblib==0.13.1 scipy==1.2.0 jsonschema==2.6.0 numpy==1.15.4 From 04786f09f44d53253532e6395dbdb25bcb7d53d9 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 13 Jan 2019 13:32:06 +0100 Subject: [PATCH 38/42] Update pytest from 4.1.0 to 4.1.1 --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2eeae1fbc..50d181296 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ -r requirements.txt flake8==3.6.0 -pytest==4.1.0 +pytest==4.1.1 pytest-mock==1.10.0 pytest-asyncio==0.10.0 pytest-cov==2.6.1 From a3897c990d16b7482f2f3b6579a8c7902b4f32f7 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 14 Jan 2019 13:32:07 +0100 Subject: [PATCH 39/42] Update ccxt from 1.18.120 to 1.18.126 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 505bebc7a..0ec1f1f5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.120 +ccxt==1.18.126 SQLAlchemy==1.2.16 python-telegram-bot==11.1.0 arrow==0.13.0 From da04182287a648637b5133a77617187a67590d0b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 14 Jan 2019 13:32:08 +0100 Subject: [PATCH 40/42] Update numpy from 1.15.4 to 1.16.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ec1f1f5f..9baeacf08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ scikit-learn==0.20.2 joblib==0.13.1 scipy==1.2.0 jsonschema==2.6.0 -numpy==1.15.4 +numpy==1.16.0 TA-Lib==0.4.17 tabulate==0.8.2 coinmarketcap==5.0.3 From 494b905d1e0ddb1ae5e497763f605604aefcb4d5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 15 Jan 2019 13:32:08 +0100 Subject: [PATCH 41/42] Update ccxt from 1.18.126 to 1.18.131 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9baeacf08..92885045d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.18.126 +ccxt==1.18.131 SQLAlchemy==1.2.16 python-telegram-bot==11.1.0 arrow==0.13.0 From 399d2d89a3642601355309426cf6ce9cf272cc6e Mon Sep 17 00:00:00 2001 From: gautier pialat Date: Tue, 15 Jan 2019 15:02:01 +0100 Subject: [PATCH 42/42] Win rate formula was wrong (but not the definition) --- docs/edge.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 61abf354b..b208cb318 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -24,7 +24,7 @@ The answer comes to two factors: Means over X trades what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). -`W = (Number of winning trades) / (Number of losing trades)` +`W = (Number of winning trades) / (Total number of trades)` ### Risk Reward Ratio Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: @@ -209,4 +209,4 @@ The full timerange specification: * Use tickframes till 2018/01/31: --timerange=-20180131 * Use tickframes since 2018/01/31: --timerange=20180131- * Use tickframes since 2018/01/31 till 2018/03/01 : --timerange=20180131-20180301 -* Use tickframes between POSIX timestamps 1527595200 1527618600: --timerange=1527595200-1527618600 \ No newline at end of file +* Use tickframes between POSIX timestamps 1527595200 1527618600: --timerange=1527595200-1527618600