From 469db0d434e86965d2d85e7d37097740de3c4e24 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 19:27:13 +1100 Subject: [PATCH 01/61] Decoupled custom hyperopts from hyperopt.py --- freqtrade/arguments.py | 8 + freqtrade/configuration.py | 3 + freqtrade/constants.py | 1 + freqtrade/optimize/custom_hyperopt.py | 152 +++++++++++ freqtrade/optimize/default_hyperopt.py | 314 ++++++++++++++++++++++ freqtrade/optimize/hyperopt.py | 138 +--------- freqtrade/optimize/interface.py | 59 ++++ freqtrade/tests/optimize/test_hyperopt.py | 10 +- user_data/hyperopts/__init__.py | 0 user_data/hyperopts/test_hyperopt.py | 283 +++++++++++++++++++ 10 files changed, 839 insertions(+), 129 deletions(-) create mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/default_hyperopt.py create mode 100644 freqtrade/optimize/interface.py create mode 100644 user_data/hyperopts/__init__.py create mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index bb571b4ea..a36b3e66a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -104,6 +104,14 @@ class Arguments(object): type=str, metavar='PATH', ) + self.parser.add_argument( + '--hyperopt', + help='specify hyperopt file (default: %(default)s)', + dest='hyperopt', + default=Constants.DEFAULT_HYPEROPT, + type=str, + metavar='PATH', + ) self.parser.add_argument( '--dynamic-whitelist', help='dynamically generate and update whitelist' diff --git a/freqtrade/configuration.py b/freqtrade/configuration.py index e043525a7..7b43ff2f8 100644 --- a/freqtrade/configuration.py +++ b/freqtrade/configuration.py @@ -52,6 +52,9 @@ class Configuration(object): if self.args.strategy_path: config.update({'strategy_path': self.args.strategy_path}) + # Add the hyperopt file to use + config.update({'hyperopt': self.args.hyperopt}) + # Load Common configuration config = self._load_common_config(config) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2b09aa6c9..a03efdc4b 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,6 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' +DEFAULT_HYPEROPT = 'default_hyperopt' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py new file mode 100644 index 000000000..688d58df8 --- /dev/null +++ b/freqtrade/optimize/custom_hyperopt.py @@ -0,0 +1,152 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib +import os +import sys +from typing import Dict, Any, Callable + +from pandas import DataFrame + +from freqtrade.constants import Constants +from freqtrade.logger import Logger +from freqtrade.optimize.interface import IHyperOpt + +sys.path.insert(0, r'../../user_data/hyperopts') + + +class CustomHyperOpt(object): + """ + This class contains all the logic to load custom hyperopt class + """ + def __init__(self, config: dict = {}) -> None: + """ + Load the custom class from config parameter + :param config: + :return: + """ + self.logger = Logger(name=__name__).get_logger() + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + if 'hyperopt' in config: + hyperopt = config['hyperopt'] + else: + hyperopt = Constants.DEFAULT_HYPEROPT + + # Load the hyperopt + self._load_hyperopt(hyperopt) + + def _load_hyperopt(self, hyperopt_name: str) -> None: + """ + Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt + Set the object into self.custom_hyperopt + :param hyperopt_name: name of the module to import + :return: None + """ + + try: + # Start by sanitizing the file name (remove any extensions) + hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) + + # Search where can be the hyperopt file + path = self._search_hyperopt(filename=hyperopt_name) + + # Load the hyperopt + self.custom_hyperopt = self._load_class(path + hyperopt_name) + + # Fallback to the default hyperopt + except (ImportError, TypeError) as error: + self.logger.error( + "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" + " or contains Python code errors", + hyperopt_name + ) + self.logger.error( + "The error is:\n%s.", + error + ) + + def _load_class(self, filename: str) -> IHyperOpt: + """ + Import a hyperopt as a module + :param filename: path to the hyperopt (path from freqtrade/optimize/) + :return: return the hyperopt class + """ + module = importlib.import_module(filename, __package__) + custom_hyperopt = getattr(module, module.class_name) + + self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + return custom_hyperopt() + + @staticmethod + def _sanitize_module_name(filename: str) -> str: + """ + Remove any extension from filename + :param filename: filename to sanatize + :return: return the filename without extensions + """ + filename = os.path.basename(filename) + filename = os.path.splitext(filename)[0] + return filename + + @staticmethod + def _search_hyperopt(filename: str) -> str: + """ + Search for the hyperopt file in different folder + 1. search into the user_data/hyperopts folder + 2. search into the freqtrade/optimize folder + 3. if nothing found, return None + :param hyperopt_name: module name to search + :return: module path where is the hyperopt + """ + pwd = os.path.dirname(os.path.realpath(__file__)) + '/' + user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') + hyperopt_folder = os.path.join(pwd, filename + '.py') + + path = None + if os.path.isfile(user_data): + path = 'user_data.hyperopts.' + elif os.path.isfile(hyperopt_folder): + path = '.' + + return path + + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell hyperopt + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + return self.custom_hyperopt.populate_indicators(dataframe) + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + return self.custom_hyperopt.buy_strategy_generator(params) + + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + return self.custom_hyperopt.indicator_space() + + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + return self.custom_hyperopt.generate_roi_table(params) + + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + return self.custom_hyperopt.stoploss_space() + + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ + return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py new file mode 100644 index 000000000..6ef95bb49 --- /dev/null +++ b/freqtrade/optimize/default_hyperopt.py @@ -0,0 +1,314 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce + +import numpy +from hyperopt import hp + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'DefaultHyperOpts' + + +class DefaultHyperOpts(IHyperOpt): + """ + Default hyperopt provided by freqtrade bot. + You can override it with your own hyperopt + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + return dataframe + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index b2d05d603..5e23458d7 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -9,22 +9,20 @@ import multiprocessing import os import sys from argparse import Namespace -from functools import reduce from math import exp from operator import itemgetter -from typing import Any, Callable, Dict, List +from typing import Any, Dict, List -import talib.abstract as ta from pandas import DataFrame from sklearn.externals.joblib import Parallel, delayed, dump, load from skopt import Optimizer -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Dimension -import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting +from freqtrade.optimize.custom_hyperopt import CustomHyperOpt logger = logging.getLogger(__name__) @@ -42,6 +40,9 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + + self.custom_hyperopt = CustomHyperOpt(self.config) + # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days self.target_trades = 600 @@ -74,24 +75,6 @@ class Hyperopt(Backtesting): arg_dict = {dim.name: value for dim, value in zip(dimensions, params)} return arg_dict - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - def save_trials(self) -> None: """ Save hyperopt trials to file @@ -149,59 +132,6 @@ class Hyperopt(Backtesting): result = trade_loss + profit_loss + duration_loss return result - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss search space - """ - return [ - Real(-0.5, -0.02, name='stoploss'), - ] - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - def has_space(self, space: str) -> bool: """ Tell if a space value is contained in the configuration @@ -216,61 +146,19 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces += Hyperopt.indicator_space() + spaces = {**spaces, **self.custom_hyperopt.indicator_space()} if self.has_space('roi'): - spaces += Hyperopt.roi_space() + spaces = {**spaces, **self.custom_hyperopt.roi_space()} if self.has_space('stoploss'): - spaces += Hyperopt.stoploss_space() + spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} return spaces - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - def generate_optimizer(self, _params) -> Dict: - params = self.get_args(_params) - + def generate_optimizer(self, params: Dict) -> Dict: if self.has_space('roi'): - self.strategy.minimal_roi = self.generate_roi_table(params) + self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.advise_buy = self.buy_strategy_generator(params) + self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] @@ -351,7 +239,7 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = Hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/interface.py new file mode 100644 index 000000000..8bc3866b2 --- /dev/null +++ b/freqtrade/optimize/interface.py @@ -0,0 +1,59 @@ +""" +IHyperOpt interface +This module defines the interface to apply for hyperopts +""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Callable + +from pandas import DataFrame + + +class IHyperOpt(ABC): + """ + Interface for freqtrade hyperopts + Defines the mandatory structure must follow any custom strategies + + Attributes you can use: + minimal_roi -> Dict: Minimal ROI designed for the strategy + stoploss -> float: optimal stoploss designed for the strategy + ticker_interval -> int: value of the ticker interval to use for the strategy + """ + + @abstractmethod + def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + """ + Populate indicators that will be used in the Buy and Sell strategy + :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :return: a Dataframe with all mandatory indicators for the strategies + """ + + @abstractmethod + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Create a buy strategy generator + """ + + @abstractmethod + def indicator_space(self) -> Dict[str, Any]: + """ + Create an indicator space + """ + + @abstractmethod + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + """ + Create an roi table + """ + + @abstractmethod + def stoploss_space(self) -> Dict[str, Any]: + """ + Create a stoploss space + """ + + @abstractmethod + def roi_space(self) -> Dict[str, Any]: + """ + Create a roi space + """ diff --git a/freqtrade/tests/optimize/test_hyperopt.py b/freqtrade/tests/optimize/test_hyperopt.py index c93f2d316..85d140b6d 100644 --- a/freqtrade/tests/optimize/test_hyperopt.py +++ b/freqtrade/tests/optimize/test_hyperopt.py @@ -175,7 +175,7 @@ def test_roi_table_generation(hyperopt) -> None: 'roi_p3': 3, } - assert hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} + assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} def test_start_calls_optimizer(mocker, default_conf, caplog) -> None: @@ -243,7 +243,8 @@ def test_populate_indicators(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe @@ -255,9 +256,10 @@ def test_buy_strategy_generator(hyperopt) -> None: tick = load_tickerdata_file(None, 'UNITTEST/BTC', '1m') tickerlist = {'UNITTEST/BTC': tick} dataframes = hyperopt.strategy.tickerdata_to_dataframe(tickerlist) - dataframe = hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) - populate_buy_trend = hyperopt.buy_strategy_generator( + populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( { 'adx-value': 20, 'fastd-value': 20, diff --git a/user_data/hyperopts/__init__.py b/user_data/hyperopts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py new file mode 100644 index 000000000..0ee43c64f --- /dev/null +++ b/user_data/hyperopts/test_hyperopt.py @@ -0,0 +1,283 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable +from functools import reduce +from math import exp + +import numpy +import talib.abstract as ta +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.indicator_helpers import fishers_inverse +from freqtrade.optimize.interface import IHyperOpt + + +# Update this variable if you change the class name +class_name = 'TestHyperOpt' + + +# This class is a sample. Feel free to customize it. +class TestHyperOpt(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md + + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + + Performance Note: For the best performance be frugal on the number of indicators + you are using. Let uncomment only the indicator you are using in your strategies + or your hyperopt configuration, otherwise you will waste your memory and CPU usage. + """ + + # Momentum Indicator + # ------------------------------------ + + # ADX + dataframe['adx'] = ta.ADX(dataframe) + + """ + # Awesome oscillator + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + + # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + dataframe['cci'] = ta.CCI(dataframe) + + # MACD + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + # MFI + dataframe['mfi'] = ta.MFI(dataframe) + + # Minus Directional Indicator / Movement + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # Plus Directional Indicator / Movement + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + + # ROC + dataframe['roc'] = ta.ROC(dataframe) + + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + """ + + # Overlap Studies + # ------------------------------------ + + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + + """ + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + + # SAR Parabol + dataframe['sar'] = ta.SAR(dataframe) + + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + """ + + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + + # Cycle Indicator + # ------------------------------------ + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + """ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + """ + + return dataframe + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} + ]), + 'uptrend_tema': hp.choice('uptrend_tema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'middle_bb_tema'}, + ]), + } + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: + prevtema = dataframe['tema'].shift(1) + conditions.append(dataframe['tema'] > prevtema) + + # TRIGGERS + triggers = { + 'middle_bb_tema': ( + dataframe['tema'] > dataframe['bb_middleband'] + ), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def roi_space() -> Dict[str, Any]: + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } \ No newline at end of file From 5816d1c1bdf54031b407574a1ff1198d4dc1562e Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Thu, 22 Mar 2018 21:07:22 +1100 Subject: [PATCH 02/61] Updated documentation for new hyperopt --- docs/hyperopt.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index e2dcf3e95..36e5d2d74 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -21,16 +21,19 @@ and still take a long time. We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) -### Configure your Guards and Triggers +### 1. Configure your Guards and Triggers -There are two places you need to change to add a new buy strategy for testing: -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L231-L264). -- Inside [hyperopt_space()](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py#L213-L224) -and the associated methods `indicator_space`, `roi_space`, `stoploss_space`. +There are two places you need to change in your strategy file to add a +new buy strategy for testing: -There you have two different type of indicators: 1. `guards` and 2. `triggers`. -1. Guards are conditions like "never buy if ADX < 10", or "never buy if -current price is over EMA10". +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). + +There you have two different type of indicators: 1. `guards` and 2. +`triggers`. + +1. Guards are conditions like "never buy if ADX < 10", or never buy if +current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". @@ -124,9 +127,11 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -python3 ./freqtrade/main.py -c config.json hyperopt -e 5000 +python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` +Use `` and `` as the names of the custom strategy and custom hyperopt used. + The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. From 40368bd1b20655a7c77768e82fb194a9d0155958 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sat, 31 Mar 2018 16:49:06 +1100 Subject: [PATCH 03/61] Added more hyperopt documentation --- docs/bot-usage.md | 2 ++ docs/hyperopt.md | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 83a8ee833..f6c15f26d 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -204,6 +204,8 @@ optional arguments: number) --timerange TIMERANGE specify what timerange of data to use. + --hyperopt PATH specify hyperopt file (default: + freqtrade/optimize/default_hyperopt.py) -e INT, --epochs INT specify number of epochs (default: 100) -s {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...], --spaces {all,buy,roi,stoploss} [{all,buy,roi,stoploss} ...] Specify which parameters to hyperopt. Space separate diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 36e5d2d74..0484613c2 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,15 +19,23 @@ and still take a long time. ## Prepare Hyperopting -We recommend you start by taking a look at `hyperopt.py` file located in [freqtrade/optimize](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/optimize/hyperopt.py) +## Prepare Hyperopt +Before we start digging in Hyperopt, we recommend you to take a look at +an example hyperopt file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) + +### 1. Install a Custom Hyperopt File +This is very simple. Put your hyperopt file into the folder +`user_data/hyperopts`. + +Let assume you want a hyperopt file `awesome_hyperopt.py`: +1. Copy the file `user_data/hyperopts/test_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` + -### 1. Configure your Guards and Triggers - -There are two places you need to change in your strategy file to add a -new buy strategy for testing: - -- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). +### 2. Configure your Guards and Triggers +There are two places you need to change in your hyperopt file to add a +new buy hyperopt for testing: +- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). There you have two different type of indicators: 1. `guards` and 2. `triggers`. @@ -130,7 +138,8 @@ We strongly recommend to use `screen` or `tmux` to prevent any connection loss. python3 ./freqtrade/main.py -s --hyperopt -c config.json hyperopt -e 5000 ``` -Use `` and `` as the names of the custom strategy and custom hyperopt used. +Use `` and `` as the names of the custom strategy +(only required for generating sells) and the custom hyperopt used. The `-e` flag will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. From e0f420983e5b9b628a3af0437bea61efda4e27b0 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sat, 31 Mar 2018 17:21:15 +1100 Subject: [PATCH 04/61] Updated logger in custom_hyperopt --- freqtrade/optimize/custom_hyperopt.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py index 688d58df8..80168c46e 100644 --- a/freqtrade/optimize/custom_hyperopt.py +++ b/freqtrade/optimize/custom_hyperopt.py @@ -4,6 +4,7 @@ This module load custom hyperopts """ import importlib +import logging import os import sys from typing import Dict, Any, Callable @@ -11,11 +12,12 @@ from typing import Dict, Any, Callable from pandas import DataFrame from freqtrade.constants import Constants -from freqtrade.logger import Logger from freqtrade.optimize.interface import IHyperOpt sys.path.insert(0, r'../../user_data/hyperopts') +logger = logging.getLogger(__name__) + class CustomHyperOpt(object): """ @@ -27,7 +29,6 @@ class CustomHyperOpt(object): :param config: :return: """ - self.logger = Logger(name=__name__).get_logger() # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt if 'hyperopt' in config: @@ -58,12 +59,12 @@ class CustomHyperOpt(object): # Fallback to the default hyperopt except (ImportError, TypeError) as error: - self.logger.error( + logger.error( "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" " or contains Python code errors", hyperopt_name ) - self.logger.error( + logger.error( "The error is:\n%s.", error ) @@ -77,7 +78,7 @@ class CustomHyperOpt(object): module = importlib.import_module(filename, __package__) custom_hyperopt = getattr(module, module.class_name) - self.logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) + logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) return custom_hyperopt() @staticmethod From 477515c4b5904b81cd9d6a6c99aa93f59daf39a8 Mon Sep 17 00:00:00 2001 From: Stephen Dade Date: Sun, 1 Apr 2018 21:06:50 +1000 Subject: [PATCH 05/61] Now using resolver for custom hyperopts --- freqtrade/arguments.py | 6 +- freqtrade/constants.py | 2 +- freqtrade/optimize/custom_hyperopt.py | 153 -------------------------- freqtrade/optimize/hyperopt.py | 7 +- freqtrade/optimize/resolver.py | 104 +++++++++++++++++ user_data/hyperopts/test_hyperopt.py | 4 - 6 files changed, 112 insertions(+), 164 deletions(-) delete mode 100644 freqtrade/optimize/custom_hyperopt.py create mode 100644 freqtrade/optimize/resolver.py diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index a36b3e66a..429a7ca5a 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -105,12 +105,12 @@ class Arguments(object): metavar='PATH', ) self.parser.add_argument( - '--hyperopt', - help='specify hyperopt file (default: %(default)s)', + '--customhyperopt', + help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', default=Constants.DEFAULT_HYPEROPT, type=str, - metavar='PATH', + metavar='NAME', ) self.parser.add_argument( '--dynamic-whitelist', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index a03efdc4b..e76c54ba2 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -9,7 +9,7 @@ TICKER_INTERVAL = 5 # min HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_STRATEGY = 'DefaultStrategy' -DEFAULT_HYPEROPT = 'default_hyperopt' +DEFAULT_HYPEROPT = 'DefaultHyperOpts' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' diff --git a/freqtrade/optimize/custom_hyperopt.py b/freqtrade/optimize/custom_hyperopt.py deleted file mode 100644 index 80168c46e..000000000 --- a/freqtrade/optimize/custom_hyperopt.py +++ /dev/null @@ -1,153 +0,0 @@ -# pragma pylint: disable=attribute-defined-outside-init - -""" -This module load custom hyperopts -""" -import importlib -import logging -import os -import sys -from typing import Dict, Any, Callable - -from pandas import DataFrame - -from freqtrade.constants import Constants -from freqtrade.optimize.interface import IHyperOpt - -sys.path.insert(0, r'../../user_data/hyperopts') - -logger = logging.getLogger(__name__) - - -class CustomHyperOpt(object): - """ - This class contains all the logic to load custom hyperopt class - """ - def __init__(self, config: dict = {}) -> None: - """ - Load the custom class from config parameter - :param config: - :return: - """ - - # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - if 'hyperopt' in config: - hyperopt = config['hyperopt'] - else: - hyperopt = Constants.DEFAULT_HYPEROPT - - # Load the hyperopt - self._load_hyperopt(hyperopt) - - def _load_hyperopt(self, hyperopt_name: str) -> None: - """ - Search and load the custom hyperopt. If no hyperopt found, fallback on the default hyperopt - Set the object into self.custom_hyperopt - :param hyperopt_name: name of the module to import - :return: None - """ - - try: - # Start by sanitizing the file name (remove any extensions) - hyperopt_name = self._sanitize_module_name(filename=hyperopt_name) - - # Search where can be the hyperopt file - path = self._search_hyperopt(filename=hyperopt_name) - - # Load the hyperopt - self.custom_hyperopt = self._load_class(path + hyperopt_name) - - # Fallback to the default hyperopt - except (ImportError, TypeError) as error: - logger.error( - "Impossible to load Hyperopt 'user_data/hyperopts/%s.py'. This file does not exist" - " or contains Python code errors", - hyperopt_name - ) - logger.error( - "The error is:\n%s.", - error - ) - - def _load_class(self, filename: str) -> IHyperOpt: - """ - Import a hyperopt as a module - :param filename: path to the hyperopt (path from freqtrade/optimize/) - :return: return the hyperopt class - """ - module = importlib.import_module(filename, __package__) - custom_hyperopt = getattr(module, module.class_name) - - logger.info("Load hyperopt class: %s (%s.py)", module.class_name, filename) - return custom_hyperopt() - - @staticmethod - def _sanitize_module_name(filename: str) -> str: - """ - Remove any extension from filename - :param filename: filename to sanatize - :return: return the filename without extensions - """ - filename = os.path.basename(filename) - filename = os.path.splitext(filename)[0] - return filename - - @staticmethod - def _search_hyperopt(filename: str) -> str: - """ - Search for the hyperopt file in different folder - 1. search into the user_data/hyperopts folder - 2. search into the freqtrade/optimize folder - 3. if nothing found, return None - :param hyperopt_name: module name to search - :return: module path where is the hyperopt - """ - pwd = os.path.dirname(os.path.realpath(__file__)) + '/' - user_data = os.path.join(pwd, '..', '..', 'user_data', 'hyperopts', filename + '.py') - hyperopt_folder = os.path.join(pwd, filename + '.py') - - path = None - if os.path.isfile(user_data): - path = 'user_data.hyperopts.' - elif os.path.isfile(hyperopt_folder): - path = '.' - - return path - - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: - """ - Populate indicators that will be used in the Buy and Sell hyperopt - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() - :return: a Dataframe with all mandatory indicators for the strategies - """ - return self.custom_hyperopt.populate_indicators(dataframe) - - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a buy strategy generator - """ - return self.custom_hyperopt.buy_strategy_generator(params) - - def indicator_space(self) -> Dict[str, Any]: - """ - Create an indicator space - """ - return self.custom_hyperopt.indicator_space() - - def generate_roi_table(self, params: Dict) -> Dict[int, float]: - """ - Create an roi table - """ - return self.custom_hyperopt.generate_roi_table(params) - - def stoploss_space(self) -> Dict[str, Any]: - """ - Create a stoploss space - """ - return self.custom_hyperopt.stoploss_space() - - def roi_space(self) -> Dict[str, Any]: - """ - Create a roi space - """ - return self.custom_hyperopt.roi_space() diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 5e23458d7..07c24ff18 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,8 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.custom_hyperopt import CustomHyperOpt +from freqtrade.optimize.resolver import HyperOptResolver + logger = logging.getLogger(__name__) @@ -40,8 +41,8 @@ class Hyperopt(Backtesting): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - - self.custom_hyperopt = CustomHyperOpt(self.config) + self.config = config + self.custom_hyperopt = HyperOptResolver(self.config).hyperopt # set TARGET_TRADES to suit your number concurrent trades so its realistic # to the number of days diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py new file mode 100644 index 000000000..78637483b --- /dev/null +++ b/freqtrade/optimize/resolver.py @@ -0,0 +1,104 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom hyperopts +""" +import importlib.util +import inspect +import logging +import os +from typing import Optional, Dict, Type + +from freqtrade.constants import Constants +from freqtrade.optimize.interface import IHyperOpt + + +logger = logging.getLogger(__name__) + + +class HyperOptResolver(object): + """ + This class contains all the logic to load custom hyperopt class + """ + + __slots__ = ['hyperopt'] + + def __init__(self, config: Optional[Dict] = None) -> None: + """ + Load the custom class from config parameter + :param config: configuration dictionary or None + """ + config = config or {} + + # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt + hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) + + def _load_hyperopt( + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + """ + Search and loads the specified hyperopt. + :param hyperopt_name: name of the module to import + :param extra_dir: additional directory to search for the given hyperopt + :return: HyperOpt instance or None + """ + current_path = os.path.dirname(os.path.realpath(__file__)) + abs_paths = [ + os.path.join(current_path, '..', '..', 'user_data', 'hyperopts'), + current_path, + ] + + if extra_dir: + # Add extra hyperopt directory on top of search paths + abs_paths.insert(0, extra_dir) + + for path in abs_paths: + hyperopt = self._search_hyperopt(path, hyperopt_name) + if hyperopt: + logger.info('Using resolved hyperopt %s from \'%s\'', hyperopt_name, path) + return hyperopt + + raise ImportError( + "Impossible to load Hyperopt '{}'. This class does not exist" + " or contains Python code errors".format(hyperopt_name) + ) + + @staticmethod + def _get_valid_hyperopts(module_path: str, hyperopt_name: str) -> Optional[Type[IHyperOpt]]: + """ + Returns a list of all possible hyperopts for the given module_path + :param module_path: absolute path to the module + :param hyperopt_name: Class name of the hyperopt + :return: Tuple with (name, class) or None + """ + + # Generate spec based on absolute path + spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + valid_hyperopts_gen = ( + obj for name, obj in inspect.getmembers(module, inspect.isclass) + if hyperopt_name == name and IHyperOpt in obj.__bases__ + ) + return next(valid_hyperopts_gen, None) + + @staticmethod + def _search_hyperopt(directory: str, hyperopt_name: str) -> Optional[IHyperOpt]: + """ + Search for the hyperopt_name in the given directory + :param directory: relative or absolute directory path + :return: name of the hyperopt class + """ + logger.debug('Searching for hyperopt %s in \'%s\'', hyperopt_name, directory) + for entry in os.listdir(directory): + # Only consider python files + if not entry.endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + hyperopt = HyperOptResolver._get_valid_hyperopts( + os.path.abspath(os.path.join(directory, entry)), hyperopt_name + ) + if hyperopt: + return hyperopt() + return None diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py index 0ee43c64f..b742af6c9 100644 --- a/user_data/hyperopts/test_hyperopt.py +++ b/user_data/hyperopts/test_hyperopt.py @@ -15,10 +15,6 @@ from freqtrade.indicator_helpers import fishers_inverse from freqtrade.optimize.interface import IHyperOpt -# Update this variable if you change the class name -class_name = 'TestHyperOpt' - - # This class is a sample. Feel free to customize it. class TestHyperOpt(IHyperOpt): """ From 8044846d3741408731967410692f7cf51d916671 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Nov 2018 07:05:40 +0100 Subject: [PATCH 06/61] Fix some refactoring problems --- freqtrade/arguments.py | 2 +- freqtrade/optimize/resolver.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 429a7ca5a..ceaf3e7e5 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -108,7 +108,7 @@ class Arguments(object): '--customhyperopt', help='specify hyperopt class name (default: %(default)s)', dest='hyperopt', - default=Constants.DEFAULT_HYPEROPT, + default=constants.DEFAULT_HYPEROPT, type=str, metavar='NAME', ) diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/resolver.py index 78637483b..1e0316c12 100644 --- a/freqtrade/optimize/resolver.py +++ b/freqtrade/optimize/resolver.py @@ -9,7 +9,7 @@ import logging import os from typing import Optional, Dict, Type -from freqtrade.constants import Constants +from freqtrade.constants import DEFAULT_HYPEROPT from freqtrade.optimize.interface import IHyperOpt @@ -31,7 +31,7 @@ class HyperOptResolver(object): config = config or {} # Verify the hyperopt is in the configuration, otherwise fallback to the default hyperopt - hyperopt_name = config.get('hyperopt') or Constants.DEFAULT_HYPEROPT + hyperopt_name = config.get('hyperopt') or DEFAULT_HYPEROPT self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) def _load_hyperopt( From 7b62e71f235e2e4a5240fc1bf0b348db37b95040 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 7 Nov 2018 19:46:04 +0100 Subject: [PATCH 07/61] Fix some tests and rebase issues --- freqtrade/optimize/__init__.py | 1 + freqtrade/optimize/default_hyperopt.py | 278 +++++------------------- freqtrade/optimize/hyperopt.py | 15 +- user_data/hyperopts/sample_hyperopt.py | 138 ++++++++++++ user_data/hyperopts/test_hyperopt.py | 279 ------------------------- 5 files changed, 194 insertions(+), 517 deletions(-) create mode 100644 user_data/hyperopts/sample_hyperopt.py delete mode 100644 user_data/hyperopts/test_hyperopt.py diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 52766f78e..b1407de18 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -20,6 +20,7 @@ from pandas import DataFrame from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +from freqtrade.optimize.default_hyperopt import DefaultHyperOpts # noqa: F401 logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 6ef95bb49..e127fd6d8 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -2,11 +2,11 @@ import talib.abstract as ta from pandas import DataFrame -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List from functools import reduce import numpy -from hyperopt import hp +from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib from freqtrade.optimize.interface import IHyperOpt @@ -21,193 +21,52 @@ class DefaultHyperOpts(IHyperOpt): """ @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - """ + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['adx'] = ta.ADX(dataframe) - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - dataframe['cci'] = ta.CCI(dataframe) macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] dataframe['mfi'] = ta.MFI(dataframe) - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['roc'] = ta.ROC(dataframe) dataframe['rsi'] = ta.RSI(dataframe) - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - # Stoch fast stoch_fast = ta.STOCHF(dataframe) dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - # SAR Parabolic dataframe['sar'] = ta.SAR(dataframe) - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - return dataframe - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Buy strategy Hyperopt will build and use """ conditions = [] # GUARDS AND TRENDS - if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: - conditions.append(dataframe['ema50'] > dataframe['ema100']) - if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: - conditions.append(dataframe['macd'] < 0) - if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: - conditions.append(dataframe['ema5'] > dataframe['ema10']) - if 'mfi' in params and params['mfi']['enabled']: - conditions.append(dataframe['mfi'] < params['mfi']['value']) - if 'fastd' in params and params['fastd']['enabled']: - conditions.append(dataframe['fastd'] < params['fastd']['value']) - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'rsi' in params and params['rsi']['enabled']: - conditions.append(dataframe['rsi'] < params['rsi']['value']) - if 'over_sar' in params and params['over_sar']['enabled']: - conditions.append(dataframe['close'] > dataframe['sar']) - if 'green_candle' in params and params['green_candle']['enabled']: - conditions.append(dataframe['close'] > dataframe['open']) - if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: - prevsma = dataframe['sma'].shift(1) - conditions.append(dataframe['sma'] > prevsma) + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) # TRIGGERS - triggers = { - 'lower_bb': ( - dataframe['close'] < dataframe['bb_lowerband'] - ), - 'lower_bb_tema': ( - dataframe['tema'] < dataframe['bb_lowerband'] - ), - 'faststoch10': (qtpylib.crossed_above( - dataframe['fastd'], 10.0 - )), - 'ao_cross_zero': (qtpylib.crossed_above( - dataframe['ao'], 0.0 - )), - 'ema3_cross_ema10': (qtpylib.crossed_above( - dataframe['ema3'], dataframe['ema10'] - )), - 'macd_cross_signal': (qtpylib.crossed_above( + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( dataframe['macd'], dataframe['macdsignal'] - )), - 'sar_reversal': (qtpylib.crossed_above( + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( dataframe['close'], dataframe['sar'] - )), - 'ht_sine': (qtpylib.crossed_above( - dataframe['htleadsine'], dataframe['htsine'] - )), - 'heiken_reversal_bull': ( - (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & - (dataframe['ha_low'] == dataframe['ha_open']) - ), - 'di_cross': (qtpylib.crossed_above( - dataframe['plus_di'], dataframe['minus_di'] - )), - } - conditions.append(triggers.get(params['trigger']['type'])) + )) dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -218,64 +77,21 @@ class DefaultHyperOpts(IHyperOpt): return populate_buy_trend @staticmethod - def indicator_space() -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Define your Hyperopt space for searching strategy parameters """ - return { - 'macd_below_zero': hp.choice('macd_below_zero', [ - {'enabled': False}, - {'enabled': True} - ]), - 'mfi': hp.choice('mfi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} - ]), - 'fastd': hp.choice('fastd', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} - ]), - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} - ]), - 'rsi': hp.choice('rsi', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} - ]), - 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'over_sar': hp.choice('over_sar', [ - {'enabled': False}, - {'enabled': True} - ]), - 'green_candle': hp.choice('green_candle', [ - {'enabled': False}, - {'enabled': True} - ]), - 'uptrend_sma': hp.choice('uptrend_sma', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'lower_bb'}, - {'type': 'lower_bb_tema'}, - {'type': 'faststoch10'}, - {'type': 'ao_cross_zero'}, - {'type': 'ema3_cross_ema10'}, - {'type': 'macd_cross_signal'}, - {'type': 'sar_reversal'}, - {'type': 'ht_sine'}, - {'type': 'heiken_reversal_bull'}, - {'type': 'di_cross'}, - ]), - } + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: @@ -291,24 +107,24 @@ class DefaultHyperOpts(IHyperOpt): return roi_table @staticmethod - def stoploss_space() -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ Stoploss Value to search """ - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } + return [ + Real(-0.5, -0.02, name='stoploss'), + ] @staticmethod - def roi_space() -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Values to search for each ROI steps """ - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 07c24ff18..f5f4222d6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -105,7 +105,7 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -147,19 +147,20 @@ class Hyperopt(Backtesting): """ spaces: List[Dimension] = [] if self.has_space('buy'): - spaces = {**spaces, **self.custom_hyperopt.indicator_space()} + spaces += self.custom_hyperopt.indicator_space() if self.has_space('roi'): - spaces = {**spaces, **self.custom_hyperopt.roi_space()} + spaces += self.custom_hyperopt.roi_space() if self.has_space('stoploss'): - spaces = {**spaces, **self.custom_hyperopt.stoploss_space()} + spaces += self.custom_hyperopt.stoploss_space() return spaces - def generate_optimizer(self, params: Dict) -> Dict: + def generate_optimizer(self, _params: Dict) -> Dict: + params = self.get_args(_params) if self.has_space('roi'): - self.analyze.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) + self.strategy.minimal_roi = self.custom_hyperopt.generate_roi_table(params) if self.has_space('buy'): - self.populate_buy_trend = self.custom_hyperopt.buy_strategy_generator(params) + self.advise_buy = self.custom_hyperopt.buy_strategy_generator(params) if self.has_space('stoploss'): self.strategy.stoploss = params['stoploss'] diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py new file mode 100644 index 000000000..77e3b1696 --- /dev/null +++ b/user_data/hyperopts/sample_hyperopt.py @@ -0,0 +1,138 @@ +# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement + +import talib.abstract as ta +from pandas import DataFrame +from typing import Dict, Any, Callable, List +from functools import reduce + +import numpy +from skopt.space import Categorical, Dimension, Integer, Real + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.optimize.interface import IHyperOpt + +class_name = 'SampleHyperOpts' + + +# This class is a sample. Feel free to customize it. +class SampleHyperOpts(IHyperOpt): + """ + This is a test hyperopt to inspire you. + More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + You can: + - Rename the class name (Do not forget to update class_name) + - Add any methods you want to build your hyperopt + - Add any lib you need to build your hyperopt + You must keep: + - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, + roi_space, generate_roi_table, stoploss_space + """ + + @staticmethod + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe['adx'] = ta.ADX(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['sar'] = ta.SAR(dataframe) + return dataframe + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'mfi-enabled' in params and params['mfi-enabled']: + conditions.append(dataframe['mfi'] < params['mfi-value']) + if 'fastd-enabled' in params and params['fastd-enabled']: + conditions.append(dataframe['fastd'] < params['fastd-value']) + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) + + # TRIGGERS + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + if params['trigger'] == 'sar_reversal': + conditions.append(qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table that will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def stoploss_space() -> List[Dimension]: + """ + Stoploss Value to search + """ + return [ + Real(-0.5, -0.02, name='stoploss'), + ] + + @staticmethod + def roi_space() -> List[Dimension]: + """ + Values to search for each ROI steps + """ + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + Real(0.01, 0.04, name='roi_p1'), + Real(0.01, 0.07, name='roi_p2'), + Real(0.01, 0.20, name='roi_p3'), + ] diff --git a/user_data/hyperopts/test_hyperopt.py b/user_data/hyperopts/test_hyperopt.py deleted file mode 100644 index b742af6c9..000000000 --- a/user_data/hyperopts/test_hyperopt.py +++ /dev/null @@ -1,279 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -import talib.abstract as ta -from pandas import DataFrame -from typing import Dict, Any, Callable -from functools import reduce -from math import exp - -import numpy -import talib.abstract as ta -from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.indicator_helpers import fishers_inverse -from freqtrade.optimize.interface import IHyperOpt - - -# This class is a sample. Feel free to customize it. -class TestHyperOpt(IHyperOpt): - """ - This is a test hyperopt to inspire you. - More information in https://github.com/gcarq/freqtrade/blob/develop/docs/hyperopt.md - - You can: - - Rename the class name (Do not forget to update class_name) - - Add any methods you want to build your hyperopt - - Add any lib you need to build your hyperopt - - You must keep: - - the prototype for the methods: populate_indicators, indicator_space, buy_strategy_generator, - roi_space, generate_roi_table, stoploss_space - """ - - @staticmethod - def populate_indicators(dataframe: DataFrame) -> DataFrame: - """ - Adds several different TA indicators to the given DataFrame - - Performance Note: For the best performance be frugal on the number of indicators - you are using. Let uncomment only the indicator you are using in your strategies - or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - """ - - # Momentum Indicator - # ------------------------------------ - - # ADX - dataframe['adx'] = ta.ADX(dataframe) - - """ - # Awesome oscillator - dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - - # Commodity Channel Index: values Oversold:<-100, Overbought:>100 - dataframe['cci'] = ta.CCI(dataframe) - - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['macdhist'] = macd['macdhist'] - - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - - # Minus Directional Indicator / Movement - dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # Plus Directional Indicator / Movement - dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - dataframe['plus_di'] = ta.PLUS_DI(dataframe) - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # ROC - dataframe['roc'] = ta.ROC(dataframe) - - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - - # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - rsi = 0.1 * (dataframe['rsi'] - 50) - dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) - - # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # Stoch - stoch = ta.STOCH(dataframe) - dataframe['slowd'] = stoch['slowd'] - dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # Stoch RSI - stoch_rsi = ta.STOCHRSI(dataframe) - dataframe['fastd_rsi'] = stoch_rsi['fastd'] - dataframe['fastk_rsi'] = stoch_rsi['fastk'] - """ - - # Overlap Studies - # ------------------------------------ - - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_middleband'] = bollinger['mid'] - dataframe['bb_upperband'] = bollinger['upper'] - - """ - # EMA - Exponential Moving Average - dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) - dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) - dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) - dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) - dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) - - # SAR Parabol - dataframe['sar'] = ta.SAR(dataframe) - - # SMA - Simple Moving Average - dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - """ - - # TEMA - Triple Exponential Moving Average - dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) - - # Cycle Indicator - # ------------------------------------ - # Hilbert Transform Indicator - SineWave - hilbert = ta.HT_SINE(dataframe) - dataframe['htsine'] = hilbert['sine'] - dataframe['htleadsine'] = hilbert['leadsine'] - - # Pattern Recognition - Bullish candlestick patterns - # ------------------------------------ - """ - # Hammer: values [0, 100] - dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) - # Inverted Hammer: values [0, 100] - dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) - # Dragonfly Doji: values [0, 100] - dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) - # Piercing Line: values [0, 100] - dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] - # Morningstar: values [0, 100] - dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] - # Three White Soldiers: values [0, 100] - dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ - - # Pattern Recognition - Bearish candlestick patterns - # ------------------------------------ - """ - # Hanging Man: values [0, 100] - dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) - # Shooting Star: values [0, 100] - dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) - # Gravestone Doji: values [0, 100] - dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) - # Dark Cloud Cover: values [0, 100] - dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) - # Evening Doji Star: values [0, 100] - dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) - # Evening Star: values [0, 100] - dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - - # Pattern Recognition - Bullish/Bearish candlestick patterns - # ------------------------------------ - """ - # Three Line Strike: values [0, -100, 100] - dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) - # Spinning Top: values [0, -100, 100] - dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] - # Engulfing: values [0, -100, 100] - dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] - # Harami: values [0, -100, 100] - dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] - # Three Outside Up/Down: values [0, -100, 100] - dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] - # Three Inside Up/Down: values [0, -100, 100] - dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ - - # Chart type - # ------------------------------------ - """ - # Heikinashi stategy - heikinashi = qtpylib.heikinashi(dataframe) - dataframe['ha_open'] = heikinashi['open'] - dataframe['ha_close'] = heikinashi['close'] - dataframe['ha_high'] = heikinashi['high'] - dataframe['ha_low'] = heikinashi['low'] - """ - - return dataframe - - @staticmethod - def indicator_space() -> Dict[str, Any]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return { - 'adx': hp.choice('adx', [ - {'enabled': False}, - {'enabled': True, 'value': hp.quniform('adx-value', 50, 80, 5)} - ]), - 'uptrend_tema': hp.choice('uptrend_tema', [ - {'enabled': False}, - {'enabled': True} - ]), - 'trigger': hp.choice('trigger', [ - {'type': 'middle_bb_tema'}, - ]), - } - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx' in params and params['adx']['enabled']: - conditions.append(dataframe['adx'] > params['adx']['value']) - if 'uptrend_tema' in params and params['uptrend_tema']['enabled']: - prevtema = dataframe['tema'].shift(1) - conditions.append(dataframe['tema'] > prevtema) - - # TRIGGERS - triggers = { - 'middle_bb_tema': ( - dataframe['tema'] > dataframe['bb_middleband'] - ), - } - conditions.append(triggers.get(params['trigger']['type'])) - - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def roi_space() -> Dict[str, Any]: - return { - 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), - 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), - 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), - 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), - 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), - 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), - } - - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def stoploss_space() -> Dict[str, Any]: - return { - 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), - } \ No newline at end of file From 4f800bfbc88335df5047292c00cc5b19c7d2b08c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 14 Nov 2018 20:25:43 +0100 Subject: [PATCH 08/61] Fix pickling-error --- freqtrade/optimize/hyperopt.py | 4 ++-- requirements.txt | 1 + setup.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f5f4222d6..f9a2924a4 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -14,7 +14,7 @@ from operator import itemgetter from typing import Any, Dict, List from pandas import DataFrame -from sklearn.externals.joblib import Parallel, delayed, dump, load +from joblib import Parallel, delayed, dump, load, wrap_non_picklable_objects from skopt import Optimizer from skopt.space import Dimension @@ -219,7 +219,7 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(self.generate_optimizer)(v) for v in asked) + return parallel(delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ diff --git a/requirements.txt b/requirements.txt index 2671660ea..11d1bea6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ urllib3==1.24.1 wrapt==1.10.11 pandas==0.23.4 scikit-learn==0.20.0 +joblib==0.13.0 scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 diff --git a/setup.py b/setup.py index 8853ef7f8..119ad03db 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', + 'joblib' 'scipy', 'jsonschema', 'TA-Lib', From 6a71f80a9e9aebf770227048c5efa9abef8714cf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 06:58:24 +0100 Subject: [PATCH 09/61] Add support for different order types --- freqtrade/exchange/__init__.py | 8 ++++---- freqtrade/freqtradebot.py | 9 +++++++-- freqtrade/strategy/default_strategy.py | 7 +++++++ freqtrade/strategy/interface.py | 7 +++++++ freqtrade/tests/exchange/test_exchange.py | 20 ++++++++++---------- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 4af9db6db..8e0116368 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -249,7 +249,7 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, rate: float, amount: float) -> Dict: + def buy(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { @@ -270,7 +270,7 @@ class Exchange(object): amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) - return self._api.create_limit_buy_order(pair, amount, rate) + return self._api.create_order(pair, ordertype, 'buy', amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit buy order on market {pair}.' @@ -287,7 +287,7 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) - def sell(self, pair: str, rate: float, amount: float) -> Dict: + def sell(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { @@ -307,7 +307,7 @@ class Exchange(object): amount = self.symbol_amount_prec(pair, amount) rate = self.symbol_price_prec(pair, rate) - return self._api.create_limit_sell_order(pair, amount, rate) + return self._api.create_order(pair, ordertype, 'sell', amount, rate) except ccxt.InsufficientFunds as e: raise DependencyException( f'Insufficient funds to create limit sell order on market {pair}.' diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..75f935c0c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -475,7 +475,8 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit - order_id = self.exchange.buy(pair, buy_limit, amount)['id'] + order_id = self.exchange.buy(pair=pair, rate=buy_limit, amount=amount, + ordertype=self.strategy.order_types['buy'])['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, @@ -762,8 +763,12 @@ class FreqtradeBot(object): :param sellreason: Reason the sell was triggered :return: None """ + sell_type = 'sell' + if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + sell_type = 'stoploss' # Execute sell and update trade record - order_id = self.exchange.sell(str(trade.pair), limit, trade.amount)['id'] + order_id = self.exchange.sell(pair=str(trade.pair), rate=limit, amount=trade.amount, + ordertype=self.strategy.order_types[sell_type])['id'] trade.open_order_id = order_id trade.close_rate_requested = limit trade.sell_reason = sell_reason.value diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index f1646779b..458847636 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,6 +28,13 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' + # Optional order types + order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 212559c8c..9d6c5f098 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -70,6 +70,13 @@ class IStrategy(ABC): # associated ticker interval ticker_interval: str + # Optional order types + order_types: Dict = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + # run "populate_indicators" only for new candle process_only_new_candles: bool = False diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 788ef4518..fe37c6ac1 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -381,7 +381,7 @@ def test_buy_dry_run(default_conf, mocker): def test_buy_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - api_mock.create_limit_buy_order = MagicMock(return_value={ + api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { 'foo': 'bar' @@ -397,22 +397,22 @@ def test_buy_prod(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InsufficientFunds) + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(DependencyException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.InvalidOrder) + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(TemporaryError): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.NetworkError) + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(OperationalException): - api_mock.create_limit_buy_order = MagicMock(side_effect=ccxt.BaseError) + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.buy(pair='ETH/BTC', rate=200, amount=1) @@ -429,7 +429,7 @@ def test_sell_dry_run(default_conf, mocker): def test_sell_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) - api_mock.create_limit_sell_order = MagicMock(return_value={ + api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { 'foo': 'bar' @@ -446,22 +446,22 @@ def test_sell_prod(default_conf, mocker): # test exception handling with pytest.raises(DependencyException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InsufficientFunds) + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(DependencyException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.InvalidOrder) + api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(TemporaryError): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.NetworkError) + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) with pytest.raises(OperationalException): - api_mock.create_limit_sell_order = MagicMock(side_effect=ccxt.BaseError) + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) exchange.sell(pair='ETH/BTC', rate=200, amount=1) From e6baa9ccf2edad645f45bc10bc4b5d4673918a8e Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:31:24 +0100 Subject: [PATCH 10/61] Switch tests to kwarguments --- freqtrade/tests/test_freqtradebot.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4cba9e308..cef89c250 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -553,7 +553,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) freqtrade.create_trade() - rate, amount = buy_mock.call_args[0][1], buy_mock.call_args[0][2] + rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] assert rate * amount >= default_conf['stake_amount'] @@ -863,10 +863,10 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert freqtrade.execute_buy(pair, stake_amount) assert get_bid.call_count == 1 assert buy_mm.call_count == 1 - call_args = buy_mm.call_args_list[0][0] - assert call_args[0] == pair - assert call_args[1] == bid - assert call_args[2] == stake_amount / bid + call_args = buy_mm.call_args_list[0][1] + assert call_args['pair'] == pair + assert call_args['rate'] == bid + assert call_args['amount'] == stake_amount / bid # Test calling with price fix_price = 0.06 @@ -875,10 +875,10 @@ def test_execute_buy(mocker, default_conf, fee, markets, limit_buy_order) -> Non assert get_bid.call_count == 1 assert buy_mm.call_count == 2 - call_args = buy_mm.call_args_list[1][0] - assert call_args[0] == pair - assert call_args[1] == fix_price - assert call_args[2] == stake_amount / fix_price + call_args = buy_mm.call_args_list[1][1] + assert call_args['pair'] == pair + assert call_args['rate'] == fix_price + assert call_args['amount'] == stake_amount / fix_price def test_process_maybe_execute_buy(mocker, default_conf) -> None: From dcf9930858d92ffed54671b1e864a3bd6fd6f27f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:36:04 +0100 Subject: [PATCH 11/61] improve hyperopt documentation (links) --- docs/hyperopt.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0484613c2..8df864c29 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -21,21 +21,21 @@ and still take a long time. ## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at -an example hyperopt file located into [user_data/strategies/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) +an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) ### 1. Install a Custom Hyperopt File This is very simple. Put your hyperopt file into the folder `user_data/hyperopts`. Let assume you want a hyperopt file `awesome_hyperopt.py`: -1. Copy the file `user_data/hyperopts/test_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` +1. Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` ### 2. Configure your Guards and Triggers There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: -- Inside [populate_buy_trend()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). -- Inside [indicator_space()](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). +- Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). +- Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). There you have two different type of indicators: 1. `guards` and 2. `triggers`. From b7abf7dda92bf78da1e075fc31a3b7e42327d5a0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Fri, 16 Nov 2018 13:34:08 +0100 Subject: [PATCH 12/61] Update ccxt from 1.17.498 to 1.17.500 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66593c264..6b356cd9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.498 +ccxt==1.17.500 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 24ed9a8b7d93f502ca2a12a3ef8db62628691a3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 10:14:18 +0100 Subject: [PATCH 13/61] Add loading order_types from config file --- config_full.json.example | 5 +++++ freqtrade/strategy/resolver.py | 9 ++++++++ freqtrade/tests/strategy/test_strategy.py | 26 +++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/config_full.json.example b/config_full.json.example index 9dba8f539..b0719bcc6 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -33,6 +33,11 @@ "order_book_min": 1, "order_book_max": 9 }, + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market" + }, "exchange": { "name": "bittrex", "key": "your_exchange_key", diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index aee47580c..31bd21ec8 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -75,6 +75,15 @@ class StrategyResolver(object): else: config['process_only_new_candles'] = self.strategy.process_only_new_candles + if 'order_types' in config: + self.strategy.order_types = config['order_types'] + logger.info( + "Override strategy 'order_types' with value in config file: %s.", + config['order_types'] + ) + else: + config['order_types'] = self.strategy.order_types + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index abc531689..e6204b5f5 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -182,6 +182,32 @@ def test_strategy_override_process_only_new_candles(caplog): ) in caplog.record_tuples +def test_strategy_override_order_types(caplog): + caplog.set_level(logging.INFO) + + order_types = { + 'buy': 'market', + 'sell': 'limit', + 'stoploss': 'limit' + } + + config = { + 'strategy': 'DefaultStrategy', + 'order_types': order_types + } + resolver = StrategyResolver(config) + + assert resolver.strategy.order_types + for method in ['buy', 'sell', 'stoploss']: + assert resolver.strategy.order_types[method] == order_types[method] + + assert ('freqtrade.strategy.resolver', + logging.INFO, + "Override strategy 'order_types' with value in config file:" + " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." + ) in caplog.record_tuples + + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) resolver = StrategyResolver({'strategy': 'TestStrategyLegacy', From 6e78efd9719f188f5d47d14db82309b9dddf02fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 10:24:42 +0100 Subject: [PATCH 14/61] Document "order_types" setting --- docs/configuration.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index d70a47b38..5a3f2b371 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,7 @@ The table below will list all configuration parameters. | `ask_strategy.use_order_book` | false | No | Allows selling of open traded pair using the rates in Order Book Asks. | `ask_strategy.order_book_min` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. | `ask_strategy.order_book_max` | 0 | No | Bot will scan from the top min to max Order Book Asks searching for a profitable rate. +| `order_types` | None | No | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`). | `exchange.name` | bittrex | Yes | Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename). | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. @@ -138,6 +139,22 @@ use the `last` price and values between those interpolate between ask and last price. Using `ask` price will guarantee quick success in bid, but bot will also end up paying more then would probably have been necessary. +### Understand order_types + +`order_types` contains a dict mapping order-types to market-types. This allows to buy using limit orders, sell using limit-orders, and create stoploss orders using market. +This can be set in the configuration or in the strategy. Configuration overwrites strategy configurations. + +If this is configured, all 3 values (`"buy"`, `"sell"` and `"stoploss"`) need to be present, otherwise the bot warn about it and will fail to start. +The below is the default which is used if this is not configured in either Strategy or configuration. + +``` json + "order_types": { + "buy": "limit", + "sell": "limit", + "stoploss": "market" + }, +``` + ### What values for exchange.name? Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports 115 cryptocurrency From 3ab0cf49af865f31d74db03e883b8c78dbb5532b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 10:26:15 +0100 Subject: [PATCH 15/61] Add order_types to sample strategy --- freqtrade/strategy/default_strategy.py | 2 +- user_data/strategies/test_strategy.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 458847636..59e280b6e 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - # Optional order types + # Optional order type mapping order_types = { 'buy': 'limit', 'sell': 'limit', diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index 7c3892b77..fd2e9ab75 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -48,6 +48,13 @@ class TestStrategy(IStrategy): # run "populate_indicators" only for new candle ta_on_candle = False + # Optional order type mapping + order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'stoploss': 'market' + } + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame From 54a86d72f28550e5904981c7b1fe909383dc91dc Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 12:59:16 +0100 Subject: [PATCH 16/61] Raise error if one of the required ordertypes is not present --- freqtrade/constants.py | 1 + freqtrade/strategy/resolver.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index b7c069c45..d5c23fe1d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ DEFAULT_STRATEGY = 'DefaultStrategy' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' +REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] TICKER_INTERVAL_MINUTES = { diff --git a/freqtrade/strategy/resolver.py b/freqtrade/strategy/resolver.py index 31bd21ec8..3f25e4838 100644 --- a/freqtrade/strategy/resolver.py +++ b/freqtrade/strategy/resolver.py @@ -84,6 +84,10 @@ class StrategyResolver(object): else: config['order_types'] = self.strategy.order_types + if not all(k in self.strategy.order_types for k in constants.REQUIRED_ORDERTYPES): + raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + f"Order-types mapping is incomplete.") + # Sort and apply type conversions self.strategy.minimal_roi = OrderedDict(sorted( {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), From 9ba281c141eb2735798fd5298b654369f0b7876f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:05:35 +0100 Subject: [PATCH 17/61] add supported limit values --- freqtrade/constants.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d5c23fe1d..fdbff886e 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -13,6 +13,7 @@ DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite://' UNLIMITED_STAKE_AMOUNT = 'unlimited' REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss'] +ORDERTYPE_POSSIBILITIES = ['limit', 'market'] TICKER_INTERVAL_MINUTES = { @@ -102,6 +103,15 @@ CONF_SCHEMA = { 'order_book_max': {'type': 'number', 'minimum': 1, 'maximum': 50} } }, + 'order_types': { + 'type': 'object', + 'properties': { + 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES} + }, + 'required': ['buy', 'sell', 'stoploss'] + }, 'exchange': {'$ref': '#/definitions/exchange'}, 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { From e485aff597d68d9f649109976a8dd85517bf8235 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:12:11 +0100 Subject: [PATCH 18/61] Test failed load on invalid ordertypes --- freqtrade/tests/strategy/test_strategy.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index e6204b5f5..d1a87ecfa 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -207,6 +207,16 @@ def test_strategy_override_order_types(caplog): " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'}." ) in caplog.record_tuples + config = { + 'strategy': 'DefaultStrategy', + 'order_types': {'buy': 'market'} + } + # Raise error for invalid configuration + with pytest.raises(ImportError, + match=r"Impossible to load Strategy 'DefaultStrategy'. " + r"Order-types mapping is incomplete."): + StrategyResolver(config) + def test_deprecate_populate_indicators(result): default_location = path.join(path.dirname(path.realpath(__file__))) From 543873263a711ccf54f90c3d0f8ed7f827b2adf1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:13:16 +0100 Subject: [PATCH 19/61] remove need for escaping quote --- freqtrade/tests/strategy/test_strategy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/strategy/test_strategy.py b/freqtrade/tests/strategy/test_strategy.py index d1a87ecfa..a38050f24 100644 --- a/freqtrade/tests/strategy/test_strategy.py +++ b/freqtrade/tests/strategy/test_strategy.py @@ -88,8 +88,8 @@ def test_load_strategy_invalid_directory(result, caplog): def test_load_not_found_strategy(): strategy = StrategyResolver() with pytest.raises(ImportError, - match=r'Impossible to load Strategy \'NotFoundStrategy\'.' - r' This class does not exist or contains Python code errors'): + match=r"Impossible to load Strategy 'NotFoundStrategy'." + r" This class does not exist or contains Python code errors"): strategy._load_strategy(strategy_name='NotFoundStrategy', config={}) From ef1e20bfe8d2e3a009bad9bf46c12f41afe9af80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:23:13 +0100 Subject: [PATCH 20/61] Don't add default value for ordertype sort parameters to align with ccxt --- freqtrade/exchange/__init__.py | 8 ++++---- freqtrade/freqtradebot.py | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8e0116368..57db4a125 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -249,14 +249,14 @@ class Exchange(object): price = ceil(big_price) / pow(10, symbol_prec) return price - def buy(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: + def buy(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_buy_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, - 'type': 'limit', + 'type': ordertype, 'side': 'buy', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), @@ -287,14 +287,14 @@ class Exchange(object): except ccxt.BaseError as e: raise OperationalException(e) - def sell(self, pair: str, rate: float, amount: float, ordertype: str = 'limt') -> Dict: + def sell(self, pair: str, ordertype: str, amount: float, rate: float) -> Dict: if self._conf['dry_run']: order_id = f'dry_run_sell_{randint(0, 10**6)}' self._dry_run_open_orders[order_id] = { 'pair': pair, 'price': rate, 'amount': amount, - 'type': 'limit', + 'type': ordertype, 'side': 'sell', 'remaining': 0.0, 'datetime': arrow.utcnow().isoformat(), diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 75f935c0c..c7532ead7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -475,8 +475,8 @@ class FreqtradeBot(object): amount = stake_amount / buy_limit - order_id = self.exchange.buy(pair=pair, rate=buy_limit, amount=amount, - ordertype=self.strategy.order_types['buy'])['id'] + order_id = self.exchange.buy(pair=pair, ordertype=self.strategy.order_types['buy'], + amount=amount, rate=buy_limit)['id'] self.rpc.send_msg({ 'type': RPCMessageType.BUY_NOTIFICATION, @@ -767,8 +767,9 @@ class FreqtradeBot(object): if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' # Execute sell and update trade record - order_id = self.exchange.sell(pair=str(trade.pair), rate=limit, amount=trade.amount, - ordertype=self.strategy.order_types[sell_type])['id'] + order_id = self.exchange.sell(pair=str(trade.pair), + ordertype=self.strategy.order_types[sell_type], + amount=trade.amount, rate=limit)['id'] trade.open_order_id = order_id trade.close_rate_requested = limit trade.sell_reason = sell_reason.value From a9a157af0fa58e1a0994747aeefd2985df9ac317 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:29:42 +0100 Subject: [PATCH 21/61] Align tests and test if ordertype is passed to ccxt correctly --- freqtrade/tests/exchange/test_exchange.py | 37 +++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fe37c6ac1..12a8fbcf4 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -373,7 +373,7 @@ def test_buy_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) assert 'id' in order assert 'dry_run_buy_' in order['id'] @@ -381,6 +381,7 @@ def test_buy_dry_run(default_conf, mocker): def test_buy_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { @@ -390,38 +391,43 @@ def test_buy_prod(default_conf, mocker): default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = exchange.buy(pair='ETH/BTC', rate=200, amount=1) + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][1] == order_type + order_type = 'limit' + api_mock.create_order.reset_mock() + order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][1] == order_type # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(OperationalException): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.buy(pair='ETH/BTC', rate=200, amount=1) + exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) def test_sell_dry_run(default_conf, mocker): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf) - order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', ordertype='limit', amount=1, rate=200) assert 'id' in order assert 'dry_run_sell_' in order['id'] @@ -429,6 +435,7 @@ def test_sell_dry_run(default_conf, mocker): def test_sell_prod(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_sell_{}'.format(randint(0, 10 ** 6)) + order_type = 'market' api_mock.create_order = MagicMock(return_value={ 'id': order_id, 'info': { @@ -439,31 +446,37 @@ def test_sell_prod(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock) - order = exchange.sell(pair='ETH/BTC', rate=200, amount=1) + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][1] == order_type + + order_type = 'limit' + api_mock.create_order.reset_mock() + order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][1] == order_type # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InvalidOrder) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) with pytest.raises(OperationalException): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError) exchange = get_patched_exchange(mocker, default_conf, api_mock) - exchange.sell(pair='ETH/BTC', rate=200, amount=1) + exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) def test_get_balance_dry_run(default_conf, mocker): From 681659f2d2f0f5fd7310f749e3fb28d762577987 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 17 Nov 2018 13:34:06 +0100 Subject: [PATCH 22/61] Update ccxt from 1.17.500 to 1.17.502 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6b356cd9e..56e0d080a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.500 +ccxt==1.17.502 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 492868a966e352251dc15d74eaed4b133c8da475 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 13:34:23 +0100 Subject: [PATCH 23/61] Seperate different tests within one test clearer --- freqtrade/tests/exchange/test_exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 12a8fbcf4..fd4512bd2 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -396,8 +396,9 @@ def test_buy_prod(default_conf, mocker): assert 'info' in order assert order['id'] == order_id assert api_mock.create_order.call_args[0][1] == order_type - order_type = 'limit' + api_mock.create_order.reset_mock() + order_type = 'limit' order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert api_mock.create_order.call_args[0][1] == order_type @@ -452,8 +453,8 @@ def test_sell_prod(default_conf, mocker): assert order['id'] == order_id assert api_mock.create_order.call_args[0][1] == order_type - order_type = 'limit' api_mock.create_order.reset_mock() + order_type = 'limit' order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert api_mock.create_order.call_args[0][1] == order_type From 69dd56b237a8c964dddd42ec3eed70220830097d Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 18:47:13 +0100 Subject: [PATCH 24/61] wallet sync drafted --- freqtrade/freqtradebot.py | 16 +++++++++++++++- freqtrade/persistence.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..428e5a726 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange from freqtrade.edge import Edge -from freqtrade.persistence import Trade +from freqtrade.persistence import Trade, Wallet from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType @@ -800,3 +800,17 @@ class FreqtradeBot(object): # Send the message self.rpc.send_msg(msg) Trade.session.flush() + + def update_wallets(self) -> bool: + wallets = self.exchange.get_balances() + + for currency in wallets: + wallet = Wallet( + exchange=self.exchange._api.id, + currency=currency, + free=wallets[currency]['free'], + used=wallets[currency]['used'], + total=wallets[currency]['total'] + ) + Wallet.session.add(wallet) + Wallet.session.flush() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 51a8129fb..ac7833d79 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -9,7 +9,7 @@ from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect) + create_engine, inspect, PrimaryKeyConstraint) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session @@ -50,8 +50,13 @@ def init(config: Dict) -> None: f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade.session = session() Trade.query = session.query_property() + + Wallet.session = session() + Wallet.query = session.query_property() + _DECL_BASE.metadata.create_all(engine) check_migrate(engine) @@ -341,3 +346,26 @@ class Trade(_DECL_BASE): ) profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") + + +class Wallet(_DECL_BASE): + """ + Class for wallet structure + It is a mirror of wallets on an exchange + """ + __tablename__ = 'wallets' + + exchange = Column(String, nullable=False, primary_key=True, index=True) + currency = Column(String, nullable=False, primary_key=True, index=True) + + free = Column(Float, index=True) + used = Column(Float) + total = Column(Float) + base = Column(Boolean, index=True, default=False) + quote = Column(Boolean, index=True, default=False) + + __table_args__ = ( + PrimaryKeyConstraint( + exchange, + currency), + {}) From 968184ef0db70040eedb7be2b1734baa11e06e3c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 19:40:22 +0100 Subject: [PATCH 25/61] Swap default mode to all limit (defaults to how it was before) --- freqtrade/strategy/default_strategy.py | 2 +- freqtrade/strategy/interface.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 59e280b6e..b282a5938 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -32,7 +32,7 @@ class DefaultStrategy(IStrategy): order_types = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'market' + 'stoploss': 'limit' } def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 9d6c5f098..139bcd8be 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -74,7 +74,7 @@ class IStrategy(ABC): order_types: Dict = { 'buy': 'limit', 'sell': 'limit', - 'stoploss': 'market' + 'stoploss': 'limit' } # run "populate_indicators" only for new candle From c11984d943af8a59a548ddbee7a78d49fcb8bdde Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 19:54:55 +0100 Subject: [PATCH 26/61] Check if exchange supports all configured market types --- freqtrade/exchange/__init__.py | 11 ++++++++- freqtrade/tests/conftest.py | 1 + freqtrade/tests/exchange/test_exchange.py | 30 +++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 57db4a125..11836ed4e 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -102,7 +102,7 @@ class Exchange(object): self.markets = self._load_markets() # Check if all pairs are available self.validate_pairs(config['exchange']['pair_whitelist']) - + self.validate_ordertypes(config.get('order_types', {})) if config.get('ticker_interval'): # Check if timeframe is available self.validate_timeframes(config['ticker_interval']) @@ -218,6 +218,15 @@ class Exchange(object): raise OperationalException( f'Invalid ticker {timeframe}, this Exchange supports {timeframes}') + def validate_ordertypes(self, order_types: Dict) -> None: + """ + Checks if order-types configured in strategy/config are supported + """ + if any(v == 'market' for k, v in order_types.items()): + if not self.exchange_has('createMarketOrder'): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') + def exchange_has(self, endpoint: str) -> bool: """ Checks if exchange implements a specific API endpoint. diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8a497725f..b6c022b45 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -30,6 +30,7 @@ def log_has(line, logs): def patch_exchange(mocker, api_mock=None) -> None: mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value="Bittrex")) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value="bittrex")) if api_mock: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index fd4512bd2..10644a9be 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -355,6 +355,36 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): Exchange(default_conf) +def test_validate_order_types(default_conf, mocker): + api_mock = MagicMock() + + type(api_mock).has = PropertyMock(return_value={'createMarketOrder': True}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + Exchange(default_conf) + + type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + + default_conf['order_types'] = {'buy': 'limit', 'sell': 'limit', 'stoploss': 'market'} + + with pytest.raises(OperationalException, + match=r'Exchange .* does not support market orders.'): + Exchange(default_conf) + + +def test_validate_order_types_not_in_config(default_conf, mocker): + api_mock = MagicMock() + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + + conf = copy.deepcopy(default_conf) + Exchange(conf) + + def test_exchange_has(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf) assert not exchange.exchange_has('ASDFASDF') From b3e08831f7a233e9fccd67d6d7749d5577896e54 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 17 Nov 2018 20:09:05 +0100 Subject: [PATCH 27/61] Remove rate for market orders --- freqtrade/exchange/__init__.py | 4 ++-- freqtrade/tests/exchange/test_exchange.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 11836ed4e..ae07e36e9 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -277,7 +277,7 @@ class Exchange(object): try: # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None return self._api.create_order(pair, ordertype, 'buy', amount, rate) except ccxt.InsufficientFunds as e: @@ -314,7 +314,7 @@ class Exchange(object): try: # Set the precision for amount and price(rate) as accepted by the exchange amount = self.symbol_amount_prec(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.symbol_price_prec(pair, rate) if ordertype != 'market' else None return self._api.create_order(pair, ordertype, 'sell', amount, rate) except ccxt.InsufficientFunds as e: diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 10644a9be..207f14efe 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -419,18 +419,28 @@ def test_buy_prod(default_conf, mocker): } }) default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock) order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None api_mock.create_order.reset_mock() order_type = 'limit' order = exchange.buy(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'buy' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 # test exception handling with pytest.raises(DependencyException): @@ -476,17 +486,27 @@ def test_sell_prod(default_conf, mocker): default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock) + mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) assert 'id' in order assert 'info' in order assert order['id'] == order_id + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] is None api_mock.create_order.reset_mock() order_type = 'limit' order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) + assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' assert api_mock.create_order.call_args[0][1] == order_type + assert api_mock.create_order.call_args[0][2] == 'sell' + assert api_mock.create_order.call_args[0][3] == 1 + assert api_mock.create_order.call_args[0][4] == 200 # test exception handling with pytest.raises(DependencyException): From 82cb0e4d95fdd663381977e9720797f96cb6bd31 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:16:32 +0100 Subject: [PATCH 28/61] =?UTF-8?q?putting=20wallets=20into=20a=20class=20(d?= =?UTF-8?q?oesn=E2=80=99t=20need=20to=20be=20in=20persistence)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/finance/wallets.py | 39 ++++++++++++++++++++++++++++++++++++ freqtrade/persistence.py | 23 --------------------- 2 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 freqtrade/finance/wallets.py diff --git a/freqtrade/finance/wallets.py b/freqtrade/finance/wallets.py new file mode 100644 index 000000000..c8d8bab2f --- /dev/null +++ b/freqtrade/finance/wallets.py @@ -0,0 +1,39 @@ +# pragma pylint: disable=W0603 +""" Wallet """ +import logging +from typing import Dict +from collections import namedtuple +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class Wallets(object): + + # wallet data structure + wallet = namedtuple( + 'wallet', + ['exchange', 'currency', 'free', 'used', 'total'] + ) + + def __init__(self, exchange: Exchange) -> None: + self.exchange = exchange + self.wallets: Dict[str, self.wallet] = {} + + def _update_wallets(self) -> None: + balances = self.exchange.get_balances() + + for currency in balances: + info = { + 'exchange': self.exchange.id, + 'currency': currency, + 'free': balances[currency['free']], + 'used': balances[currency['used']], + 'total': balances[currency['total']] + } + self.wallets[currency] = self.wallet(**info) + + logger.info('Wallets synced ...') + + def update(self) -> None: + self._update_wallets() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index ac7833d79..82daa0b74 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -346,26 +346,3 @@ class Trade(_DECL_BASE): ) profit_percent = (close_trade_price / open_trade_price) - 1 return float(f"{profit_percent:.8f}") - - -class Wallet(_DECL_BASE): - """ - Class for wallet structure - It is a mirror of wallets on an exchange - """ - __tablename__ = 'wallets' - - exchange = Column(String, nullable=False, primary_key=True, index=True) - currency = Column(String, nullable=False, primary_key=True, index=True) - - free = Column(Float, index=True) - used = Column(Float) - total = Column(Float) - base = Column(Boolean, index=True, default=False) - quote = Column(Boolean, index=True, default=False) - - __table_args__ = ( - PrimaryKeyConstraint( - exchange, - currency), - {}) From afe52efc8ac1f05a0f9210252fa4937bb9b1f5eb Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:17:39 +0100 Subject: [PATCH 29/61] removing wallet from freq --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 428e5a726..570b925cc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange from freqtrade.edge import Edge -from freqtrade.persistence import Trade, Wallet +from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType From b815c8fe2dd37385708b9313cf689b9179449117 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:22:54 +0100 Subject: [PATCH 30/61] updating wallets whenever a trade happens --- freqtrade/freqtradebot.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 570b925cc..d9f17b999 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,6 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange +from freqtrade.finance.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType @@ -56,6 +57,7 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + self.wallets = Wallets(self.exchange) # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ @@ -505,6 +507,10 @@ class FreqtradeBot(object): ) Trade.session.add(trade) Trade.session.flush() + + # Updating wallets + self.wallets.update() + return True def process_maybe_execute_buy(self) -> bool: @@ -549,7 +555,11 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair - return self.handle_trade(trade) + result = self.handle_trade(trade) + if result: + self.wallets.update() + return result + except DependencyException as exception: logger.warning('Unable to sell trade: %s', exception) return False From 12f07ee126145b105623b0e7d28a711845bf5265 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:26:41 +0100 Subject: [PATCH 31/61] space removed --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d9f17b999..631f4d3b1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -555,7 +555,7 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair - result = self.handle_trade(trade) + result = self.handle_trade(trade) if result: self.wallets.update() return result From a0658bb50448b65c5997ea19c96c10a0d7aefe38 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:27:42 +0100 Subject: [PATCH 32/61] comments added --- freqtrade/freqtradebot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 631f4d3b1..568769fff 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -556,8 +556,11 @@ class FreqtradeBot(object): if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair result = self.handle_trade(trade) + + # Updating wallets if any trade occured if result: self.wallets.update() + return result except DependencyException as exception: From d5b47abe98af09f150176c904b87b51c2e7d1f25 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:31:06 +0100 Subject: [PATCH 33/61] Wallet table removed --- freqtrade/persistence.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 82daa0b74..2936819a6 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -54,9 +54,6 @@ def init(config: Dict) -> None: Trade.session = session() Trade.query = session.query_property() - Wallet.session = session() - Wallet.query = session.query_property() - _DECL_BASE.metadata.create_all(engine) check_migrate(engine) From f4bb203782eb54f8d8557edd2b3d27a020ddc9db Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 21:59:21 +0100 Subject: [PATCH 34/61] removing persistence update --- freqtrade/freqtradebot.py | 14 -------------- freqtrade/persistence.py | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 568769fff..0a75ce3d7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -813,17 +813,3 @@ class FreqtradeBot(object): # Send the message self.rpc.send_msg(msg) Trade.session.flush() - - def update_wallets(self) -> bool: - wallets = self.exchange.get_balances() - - for currency in wallets: - wallet = Wallet( - exchange=self.exchange._api.id, - currency=currency, - free=wallets[currency]['free'], - used=wallets[currency]['used'], - total=wallets[currency]['total'] - ) - Wallet.session.add(wallet) - Wallet.session.flush() diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 2936819a6..aa380d20b 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -9,7 +9,7 @@ from typing import Any, Dict, Optional import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, inspect, PrimaryKeyConstraint) + create_engine, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm.scoping import scoped_session From 606e41d5743594bc69a30e548de1046e3480a1ea Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 22:58:27 +0100 Subject: [PATCH 35/61] wallet tests added --- freqtrade/tests/finance/test_wallets.py | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 freqtrade/tests/finance/test_wallets.py diff --git a/freqtrade/tests/finance/test_wallets.py b/freqtrade/tests/finance/test_wallets.py new file mode 100644 index 000000000..e226d117f --- /dev/null +++ b/freqtrade/tests/finance/test_wallets.py @@ -0,0 +1,32 @@ +# pragma pylint: disable=missing-docstring +from freqtrade.tests.conftest import get_patched_freqtradebot +from unittest.mock import MagicMock + + +def test_sync_wallet_at_boot(mocker, default_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.0, + "used": 2.0, + "total": 3.0 + }, + "GAS": { + "free": 0.260739, + "used": 0.0, + "total": 0.260739 + }, + }) + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.0 + assert freqtrade.wallets.wallets['BNT'].used == 2.0 + assert freqtrade.wallets.wallets['BNT'].total == 3.0 + + assert freqtrade.wallets.wallets['GAS'].free == 0.260739 + assert freqtrade.wallets.wallets['GAS'].used == 0.0 + assert freqtrade.wallets.wallets['GAS'].total == 0.260739 From 7cb8b28f5885dfeca3ba99723e377668f385bddf Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 17 Nov 2018 23:03:07 +0100 Subject: [PATCH 36/61] wallet sync added --- freqtrade/finance/wallets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/finance/wallets.py b/freqtrade/finance/wallets.py index c8d8bab2f..7155f90ac 100644 --- a/freqtrade/finance/wallets.py +++ b/freqtrade/finance/wallets.py @@ -19,6 +19,7 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange self.wallets: Dict[str, self.wallet] = {} + self._update_wallets() def _update_wallets(self) -> None: balances = self.exchange.get_balances() @@ -27,10 +28,11 @@ class Wallets(object): info = { 'exchange': self.exchange.id, 'currency': currency, - 'free': balances[currency['free']], - 'used': balances[currency['used']], - 'total': balances[currency['total']] + 'free': balances[currency]['free'], + 'used': balances[currency]['used'], + 'total': balances[currency]['total'] } + self.wallets[currency] = self.wallet(**info) logger.info('Wallets synced ...') From a92619f18ca9d56361960d68c213098ada3e23a9 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:34:31 +0100 Subject: [PATCH 37/61] Added empty lines related to last commit removed --- freqtrade/persistence.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index aa380d20b..51a8129fb 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -50,10 +50,8 @@ def init(config: Dict) -> None: f'is no valid database URL! (See {_SQL_DOCS_URL})') session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.session = session() Trade.query = session.query_property() - _DECL_BASE.metadata.create_all(engine) check_migrate(engine) From 608ce98e1a5c67ce4bd0556cfdccf4d0375c6096 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:38:31 +0100 Subject: [PATCH 38/61] moving wallets to root --- freqtrade/freqtradebot.py | 2 +- freqtrade/{finance => }/wallets.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename freqtrade/{finance => }/wallets.py (100%) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a75ce3d7..83b5b85e5 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,7 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange -from freqtrade.finance.wallets import Wallets +from freqtrade.wallets import Wallets from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType diff --git a/freqtrade/finance/wallets.py b/freqtrade/wallets.py similarity index 100% rename from freqtrade/finance/wallets.py rename to freqtrade/wallets.py From 9c549f451390822e22c6b4e141aa239234c9b85d Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:39:31 +0100 Subject: [PATCH 39/61] removing unnecessary private function --- freqtrade/wallets.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 7155f90ac..fc2769831 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -19,9 +19,9 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange self.wallets: Dict[str, self.wallet] = {} - self._update_wallets() + self.update() - def _update_wallets(self) -> None: + def update(self) -> None: balances = self.exchange.get_balances() for currency in balances: @@ -36,6 +36,3 @@ class Wallets(object): self.wallets[currency] = self.wallet(**info) logger.info('Wallets synced ...') - - def update(self) -> None: - self._update_wallets() From c03337804808914afed09bcaafdd0e50d0cba135 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 18 Nov 2018 14:57:03 +0100 Subject: [PATCH 40/61] change dict type to Any --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index fc2769831..b4b89bc59 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict +from typing import Dict, Any from collections import namedtuple from freqtrade.exchange import Exchange @@ -18,7 +18,7 @@ class Wallets(object): def __init__(self, exchange: Exchange) -> None: self.exchange = exchange - self.wallets: Dict[str, self.wallet] = {} + self.wallets: Dict[str, Any] = {} self.update() def update(self) -> None: From b680681b34e25d0641d2c65ca959c395403b9d2f Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 19 Nov 2018 11:16:07 +0100 Subject: [PATCH 41/61] updating wallet at handle timeout functions too --- freqtrade/freqtradebot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 83b5b85e5..570a806be 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -340,7 +340,9 @@ class FreqtradeBot(object): else: stake_amount = self.config['stake_amount'] + # TODO: should come from the wallet avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) + #avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) @@ -707,8 +709,10 @@ class FreqtradeBot(object): if order['status'] == 'open': if order['side'] == 'buy' and ordertime < buy_timeoutthreashold: self.handle_timedout_limit_buy(trade, order) + self.wallets.update() elif order['side'] == 'sell' and ordertime < sell_timeoutthreashold: self.handle_timedout_limit_sell(trade, order) + self.wallets.update() # FIX: 20180110, why is cancel.order unconditionally here, whereas # it is conditionally called in the From 003480ad9031012f99a56c0d03af6b4a5415549a Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 19 Nov 2018 13:01:17 +0100 Subject: [PATCH 42/61] flake indentation --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 570a806be..2a2a05c84 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -342,7 +342,7 @@ class FreqtradeBot(object): # TODO: should come from the wallet avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) - #avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free + # avaliable_amount = self.wallets.wallets[self.config['stake_currency']].free if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: open_trades = len(Trade.query.filter(Trade.is_open.is_(True)).all()) From cf2d68501c02c0cc96e95102f0ce5a78798fba79 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 19 Nov 2018 13:34:07 +0100 Subject: [PATCH 43/61] Update ccxt from 1.17.502 to 1.17.513 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 56e0d080a..9cb83d517 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.502 +ccxt==1.17.513 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From e69f9439118c07a9abb2f5b1467ec36181cc1bf2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Nov 2018 19:53:16 +0100 Subject: [PATCH 44/61] Add missing semicolon --- docs/hyperopt.md | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8df864c29..9fd11e5de 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -19,7 +19,6 @@ and still take a long time. ## Prepare Hyperopting -## Prepare Hyperopt Before we start digging in Hyperopt, we recommend you to take a look at an example hyperopt file located into [user_data/hyperopts/](https://github.com/gcarq/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py) diff --git a/setup.py b/setup.py index 119ad03db..d2a9b97f9 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', - 'joblib' + 'joblib', 'scipy', 'jsonschema', 'TA-Lib', From ce092742dab519130c69e73ac85980a0868fec1c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 20 Nov 2018 13:34:07 +0100 Subject: [PATCH 45/61] Update ccxt from 1.17.513 to 1.17.518 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9cb83d517..a845ddeb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.513 +ccxt==1.17.518 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 5a550ef2af5227ae217a6fa5f57ff05a4697b2ec Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:36:17 +0100 Subject: [PATCH 46/61] Fix docs typo in hyperopt --- docs/hyperopt.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 9fd11e5de..dffe84d1d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -36,8 +36,7 @@ new buy hyperopt for testing: - Inside [populate_buy_trend()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L230-L251). - Inside [indicator_space()](https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/test_hyperopt.py#L207-L223). -There you have two different type of indicators: 1. `guards` and 2. -`triggers`. +There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. From 5dd013c3b19dbb1fa1680d391aa9c068526418c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:40:45 +0100 Subject: [PATCH 47/61] Rename hyperopt interface and resolver --- freqtrade/optimize/default_hyperopt.py | 3 +-- freqtrade/optimize/hyperopt.py | 2 +- freqtrade/optimize/{interface.py => hyperopt_interface.py} | 0 freqtrade/optimize/{resolver.py => hyperopt_resolver.py} | 2 +- user_data/hyperopts/sample_hyperopt.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) rename freqtrade/optimize/{interface.py => hyperopt_interface.py} (100%) rename freqtrade/optimize/{resolver.py => hyperopt_resolver.py} (98%) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index e127fd6d8..723dfde17 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -5,11 +5,10 @@ from pandas import DataFrame from typing import Dict, Any, Callable, List from functools import reduce -import numpy from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt class_name = 'DefaultHyperOpts' diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index f9a2924a4..bd6e43daa 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,7 +22,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.optimize import load_data from freqtrade.optimize.backtesting import Backtesting -from freqtrade.optimize.resolver import HyperOptResolver +from freqtrade.optimize.hyperopt_resolver import HyperOptResolver logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/interface.py b/freqtrade/optimize/hyperopt_interface.py similarity index 100% rename from freqtrade/optimize/interface.py rename to freqtrade/optimize/hyperopt_interface.py diff --git a/freqtrade/optimize/resolver.py b/freqtrade/optimize/hyperopt_resolver.py similarity index 98% rename from freqtrade/optimize/resolver.py rename to freqtrade/optimize/hyperopt_resolver.py index 1e0316c12..1c3a9d577 100644 --- a/freqtrade/optimize/resolver.py +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -10,7 +10,7 @@ import os from typing import Optional, Dict, Type from freqtrade.constants import DEFAULT_HYPEROPT -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt logger = logging.getLogger(__name__) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 77e3b1696..83dc5f71a 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -9,7 +9,7 @@ import numpy from skopt.space import Categorical, Dimension, Integer, Real import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.interface import IHyperOpt +from freqtrade.optimize.hyperopt_interface import IHyperOpt class_name = 'SampleHyperOpts' From 7757c53b0668d53fac0c9b178e2def511c4101da Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 17:43:49 +0100 Subject: [PATCH 48/61] Small fixes --- freqtrade/optimize/hyperopt.py | 9 ++++++--- setup.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index bd6e43daa..70d20673c 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -105,7 +105,8 @@ class Hyperopt(Backtesting): best_result['params'] ) if 'roi_t1' in best_result['params']: - logger.info('ROI table:\n%s', self.custom_hyperopt.generate_roi_table(best_result['params'])) + logger.info('ROI table:\n%s', + self.custom_hyperopt.generate_roi_table(best_result['params'])) def log_results(self, results) -> None: """ @@ -219,7 +220,8 @@ class Hyperopt(Backtesting): ) def run_optimizer_parallel(self, parallel, asked) -> List: - return parallel(delayed(wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) + return parallel(delayed( + wrap_non_picklable_objects(self.generate_optimizer))(v) for v in asked) def load_previous_results(self): """ read trials file if we have one """ @@ -241,7 +243,8 @@ class Hyperopt(Backtesting): ) if self.has_space('buy'): - self.strategy.advise_indicators = self.custom_hyperopt.populate_indicators # type: ignore + self.strategy.advise_indicators = \ + self.custom_hyperopt.populate_indicators # type: ignore dump(self.strategy.tickerdata_to_dataframe(data), TICKERDATA_PICKLE) self.exchange = None # type: ignore self.load_previous_results() diff --git a/setup.py b/setup.py index d2a9b97f9..34d4c8755 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ setup(name='freqtrade', 'wrapt', 'pandas', 'scikit-learn', - 'joblib', 'scipy', + 'joblib', 'jsonschema', 'TA-Lib', 'tabulate', From a3b600411570eaf23443c19bb3322b2c36d14cc6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 20 Nov 2018 19:41:07 +0100 Subject: [PATCH 49/61] IHyperopt: all methods static, somef ixes for mypy --- freqtrade/optimize/default_hyperopt.py | 3 ++- freqtrade/optimize/hyperopt_interface.py | 21 ++++++++++++++------- freqtrade/optimize/hyperopt_resolver.py | 4 ++-- user_data/hyperopts/sample_hyperopt.py | 3 ++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/default_hyperopt.py b/freqtrade/optimize/default_hyperopt.py index 723dfde17..6139f8140 100644 --- a/freqtrade/optimize/default_hyperopt.py +++ b/freqtrade/optimize/default_hyperopt.py @@ -36,7 +36,8 @@ class DefaultHyperOpts(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 8bc3866b2..d42206658 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -4,9 +4,10 @@ This module defines the interface to apply for hyperopts """ from abc import ABC, abstractmethod -from typing import Dict, Any, Callable +from typing import Dict, Any, Callable, List from pandas import DataFrame +from skopt.space import Dimension class IHyperOpt(ABC): @@ -20,40 +21,46 @@ class IHyperOpt(ABC): ticker_interval -> int: value of the ticker interval to use for the strategy """ + @staticmethod @abstractmethod - def populate_indicators(self, dataframe: DataFrame) -> DataFrame: + def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() :return: a Dataframe with all mandatory indicators for the strategies """ + @staticmethod @abstractmethod - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Create a buy strategy generator """ + @staticmethod @abstractmethod - def indicator_space(self) -> Dict[str, Any]: + def indicator_space() -> List[Dimension]: """ Create an indicator space """ + @staticmethod @abstractmethod - def generate_roi_table(self, params: Dict) -> Dict[int, float]: + def generate_roi_table(params: Dict) -> Dict[int, float]: """ Create an roi table """ + @staticmethod @abstractmethod - def stoploss_space(self) -> Dict[str, Any]: + def stoploss_space() -> List[Dimension]: """ Create a stoploss space """ + @staticmethod @abstractmethod - def roi_space(self) -> Dict[str, Any]: + def roi_space() -> List[Dimension]: """ Create a roi space """ diff --git a/freqtrade/optimize/hyperopt_resolver.py b/freqtrade/optimize/hyperopt_resolver.py index 1c3a9d577..3d019e8df 100644 --- a/freqtrade/optimize/hyperopt_resolver.py +++ b/freqtrade/optimize/hyperopt_resolver.py @@ -35,7 +35,7 @@ class HyperOptResolver(object): self.hyperopt = self._load_hyperopt(hyperopt_name, extra_dir=config.get('hyperopt_path')) def _load_hyperopt( - self, hyperopt_name: str, extra_dir: Optional[str] = None) -> Optional[IHyperOpt]: + self, hyperopt_name: str, extra_dir: Optional[str] = None) -> IHyperOpt: """ Search and loads the specified hyperopt. :param hyperopt_name: name of the module to import @@ -75,7 +75,7 @@ class HyperOptResolver(object): # Generate spec based on absolute path spec = importlib.util.spec_from_file_location('user_data.hyperopts', module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints valid_hyperopts_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) diff --git a/user_data/hyperopts/sample_hyperopt.py b/user_data/hyperopts/sample_hyperopt.py index 83dc5f71a..f11236a82 100644 --- a/user_data/hyperopts/sample_hyperopt.py +++ b/user_data/hyperopts/sample_hyperopt.py @@ -45,7 +45,8 @@ class SampleHyperOpts(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ Define the buy strategy parameters to be used by hyperopt """ From d745e577b4229803dad1ccf1cd5072620197f9ce Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 21 Nov 2018 13:34:06 +0100 Subject: [PATCH 50/61] Update ccxt from 1.17.518 to 1.17.522 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a845ddeb9..821ff8b4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.518 +ccxt==1.17.522 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 64129897f9fe696e991af1456f75ddf035f3b799 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 14:00:15 +0100 Subject: [PATCH 51/61] refresh_ticker should be called just once per iteration. --- freqtrade/freqtradebot.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index eb92375ec..5ab78ab35 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -194,9 +194,6 @@ class FreqtradeBot(object): self.edge.calculate() self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) - # Refreshing candles - self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) - # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From 68f81aa2afb61509bca05a80da16d2d9ee1d5387 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 17:27:45 +0100 Subject: [PATCH 52/61] test wallets moved to tests folder --- freqtrade/tests/{finance => }/test_wallets.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename freqtrade/tests/{finance => }/test_wallets.py (100%) diff --git a/freqtrade/tests/finance/test_wallets.py b/freqtrade/tests/test_wallets.py similarity index 100% rename from freqtrade/tests/finance/test_wallets.py rename to freqtrade/tests/test_wallets.py From 5b689402130d2a45ccdc8a65bbac669b5e247bdb Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 17:48:53 +0100 Subject: [PATCH 53/61] update wallet in casse order remaining is zero --- freqtrade/freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2a2a05c84..7d6fdd261 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -703,6 +703,7 @@ class FreqtradeBot(object): # Check if trade is still actually open if int(order['remaining']) == 0: + self.wallets.update() continue # Check if trade is still actually open From aeb372c2f04ac992e5d6c82dca34ddb770100424 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 17:54:14 +0100 Subject: [PATCH 54/61] test wallet when api return changes --- freqtrade/tests/test_wallets.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index e226d117f..a5295579b 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -26,7 +26,32 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['BNT'].free == 1.0 assert freqtrade.wallets.wallets['BNT'].used == 2.0 assert freqtrade.wallets.wallets['BNT'].total == 3.0 - assert freqtrade.wallets.wallets['GAS'].free == 0.260739 assert freqtrade.wallets.wallets['GAS'].used == 0.0 assert freqtrade.wallets.wallets['GAS'].total == 0.260739 + + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.2, + "used": 1.9, + "total": 3.5 + }, + "GAS": { + "free": 0.270739, + "used": 0.1, + "total": 0.260439 + }, + }) + ) + + freqtrade.wallets.update() + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.2 + assert freqtrade.wallets.wallets['BNT'].used == 1.9 + assert freqtrade.wallets.wallets['BNT'].total == 3.5 + assert freqtrade.wallets.wallets['GAS'].free == 0.270739 + assert freqtrade.wallets.wallets['GAS'].used == 0.1 + assert freqtrade.wallets.wallets['GAS'].total == 0.260439 From cb3cf960d734373618f878bde2632317a366dd25 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 19:47:28 +0100 Subject: [PATCH 55/61] tests added in case of missing data --- freqtrade/tests/test_wallets.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/freqtrade/tests/test_wallets.py b/freqtrade/tests/test_wallets.py index a5295579b..cc10d665c 100644 --- a/freqtrade/tests/test_wallets.py +++ b/freqtrade/tests/test_wallets.py @@ -55,3 +55,30 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.wallets['GAS'].free == 0.270739 assert freqtrade.wallets.wallets['GAS'].used == 0.1 assert freqtrade.wallets.wallets['GAS'].total == 0.260439 + + +def test_sync_wallet_missing_data(mocker, default_conf): + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value={ + "BNT": { + "free": 1.0, + "used": 2.0, + "total": 3.0 + }, + "GAS": { + "free": 0.260739, + "total": 0.260739 + }, + }) + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert len(freqtrade.wallets.wallets) == 2 + assert freqtrade.wallets.wallets['BNT'].free == 1.0 + assert freqtrade.wallets.wallets['BNT'].used == 2.0 + assert freqtrade.wallets.wallets['BNT'].total == 3.0 + assert freqtrade.wallets.wallets['GAS'].free == 0.260739 + assert freqtrade.wallets.wallets['GAS'].used is None + assert freqtrade.wallets.wallets['GAS'].total == 0.260739 From 88f61581d924138b0926a5459a634459c517cf68 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 19:47:51 +0100 Subject: [PATCH 56/61] 1) NamedTuple refactored 2) Missing data handled --- freqtrade/wallets.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b4b89bc59..5ee33f41e 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,13 +1,21 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict, Any +from typing import Dict, Any, NamedTuple from collections import namedtuple from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) +class Wallet(NamedTuple): + exchange: str = None + currency: str = None + free: float = 0 + used: float = 0 + total: float = 0 + + class Wallets(object): # wallet data structure @@ -25,14 +33,12 @@ class Wallets(object): balances = self.exchange.get_balances() for currency in balances: - info = { - 'exchange': self.exchange.id, - 'currency': currency, - 'free': balances[currency]['free'], - 'used': balances[currency]['used'], - 'total': balances[currency]['total'] - } - - self.wallets[currency] = self.wallet(**info) + self.wallets[currency] = Wallet( + self.exchange.id, + currency, + balances[currency].get('free', None), + balances[currency].get('used', None), + balances[currency].get('total', None) + ) logger.info('Wallets synced ...') From b129750f4dbff4daca892b6ebefdf1648637caff Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 19:58:28 +0100 Subject: [PATCH 57/61] =?UTF-8?q?adding=20=E2=80=9Coptional=E2=80=9D=20in?= =?UTF-8?q?=20str?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/wallets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 5ee33f41e..531e83a28 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict, Any, NamedTuple +from typing import Dict, Any, NamedTuple, Optional from collections import namedtuple from freqtrade.exchange import Exchange @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class Wallet(NamedTuple): - exchange: str = None - currency: str = None + exchange: Optional[str] = None + currency: Optional[str] = None free: float = 0 used: float = 0 total: float = 0 From 4d75e9059c0f3c10a8c910bea83ff74292ea7367 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 21:05:20 +0100 Subject: [PATCH 58/61] None ripped off for optional as wallet must have exchange and currency --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 531e83a28..3b4152b8d 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class Wallet(NamedTuple): - exchange: Optional[str] = None - currency: Optional[str] = None + exchange: Optional[str] + currency: Optional[str] free: float = 0 used: float = 0 total: float = 0 From 3a2134db24d21285cafa8b65c6257cc0e380b928 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 23:35:44 +0100 Subject: [PATCH 59/61] removed Optional --- freqtrade/wallets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 3b4152b8d..65a5e880f 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) class Wallet(NamedTuple): - exchange: Optional[str] - currency: Optional[str] + exchange: str + currency: str free: float = 0 used: float = 0 total: float = 0 From 4b86b2b7e3fa89e16d5a40625f9af12cf488151c Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 21 Nov 2018 23:36:48 +0100 Subject: [PATCH 60/61] Happy flake8 ! --- freqtrade/wallets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 65a5e880f..82f527d2c 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -1,7 +1,7 @@ # pragma pylint: disable=W0603 """ Wallet """ import logging -from typing import Dict, Any, NamedTuple, Optional +from typing import Dict, Any, NamedTuple from collections import namedtuple from freqtrade.exchange import Exchange From eb53281434df691ef954dcb8b964ad7c4eeba74e Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 22 Nov 2018 00:04:20 +0100 Subject: [PATCH 61/61] python beginner problem resolved --- freqtrade/edge/__init__.py | 39 ++++++++++++----------- freqtrade/tests/edge/test_edge.py | 14 ++++---- freqtrade/tests/optimize/test_edge_cli.py | 13 ++------ 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index dedaa19a3..009b80664 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,8 +1,7 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging -from typing import Any, Dict -from collections import namedtuple +from typing import Any, Dict, NamedTuple import arrow import numpy as np @@ -18,6 +17,16 @@ from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) +class PairInfo(NamedTuple): + stoploss: float + winrate: float + risk_reward_ratio: float + required_risk_reward: float + expectancy: float + nb_trades: int + avg_trade_duration: float + + class Edge(): """ Calculates Win Rate, Risk Reward Ratio, Expectancy @@ -30,13 +39,6 @@ class Edge(): config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - # pair info data type - _pair_info = namedtuple( - 'pair_info', - ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy', - 'nb_trades', 'avg_trade_duration'] - ) - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config @@ -294,16 +296,15 @@ class Edge(): final = {} for x in df.itertuples(): - info = { - 'stoploss': x.stoploss, - 'winrate': x.winrate, - 'risk_reward_ratio': x.risk_reward_ratio, - 'required_risk_reward': x.required_risk_reward, - 'expectancy': x.expectancy, - 'nb_trades': x.nb_trades, - 'avg_trade_duration': x.avg_trade_duration - } - final[x.pair] = self._pair_info(**info) + final[x.pair] = PairInfo( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy, + x.nb_trades, + x.avg_trade_duration + ) # Returning a list of pairs in order of "expectancy" return final diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 14c9114c3..fac055c17 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -4,7 +4,7 @@ import pytest import logging from freqtrade.tests.conftest import get_patched_freqtradebot -from freqtrade.edge import Edge +from freqtrade.edge import Edge, PairInfo from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, @@ -128,9 +128,9 @@ def test_adjust(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) @@ -143,9 +143,9 @@ def test_stoploss(mocker, default_conf): edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), - 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) + 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'C/D': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), + 'N/O': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60) } )) diff --git a/freqtrade/tests/optimize/test_edge_cli.py b/freqtrade/tests/optimize/test_edge_cli.py index f8db3dec4..0d0f64e0c 100644 --- a/freqtrade/tests/optimize/test_edge_cli.py +++ b/freqtrade/tests/optimize/test_edge_cli.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import json from typing import List -from freqtrade.edge import Edge +from freqtrade.edge import PairInfo from freqtrade.arguments import Arguments from freqtrade.optimize.edge_cli import (EdgeCli, setup_configuration, start) from freqtrade.tests.conftest import log_has, patch_exchange @@ -123,17 +123,8 @@ def test_generate_edge_table(edge_conf, mocker): edge_cli = EdgeCli(edge_conf) results = {} - info = { - 'stoploss': -0.01, - 'winrate': 0.60, - 'risk_reward_ratio': 2, - 'required_risk_reward': 1, - 'expectancy': 3, - 'nb_trades': 10, - 'avg_trade_duration': 60 - } + results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - results['ETH/BTC'] = Edge._pair_info(**info) assert edge_cli._generate_edge_table(results).count(':|') == 7 assert edge_cli._generate_edge_table(results).count('| ETH/BTC |') == 1 assert edge_cli._generate_edge_table(results).count(