diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 0cba621af..002aa2ddf 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -62,49 +62,64 @@ class Bittrex(Exchange): def buy(self, pair: str, rate: float, amount: float) -> str: data = _API.buy_limit(pair.replace('_', '-'), amount, rate) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( - message=data['message'], - pair=pair, - rate=rate, - amount=amount)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=data['message'], + pair=pair, + rate=rate, + amount=amount)) return data['result']['uuid'] def sell(self, pair: str, rate: float, amount: float) -> str: data = _API.sell_limit(pair.replace('_', '-'), amount, rate) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( - message=data['message'], - pair=pair, - rate=rate, - amount=amount)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=data['message'], + pair=pair, + rate=rate, + amount=amount)) return data['result']['uuid'] def get_balance(self, currency: str) -> float: data = _API.get_balance(currency) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({currency})'.format( - message=data['message'], - currency=currency)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({currency})'.format( + message=data['message'], + currency=currency)) return float(data['result']['Balance'] or 0.0) def get_balances(self): data = _API.get_balances() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message}'.format(message=data['message'])) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message}'.format(message=data['message'])) return data['result'] def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: if refresh or pair not in self.cached_ticker.keys(): data = _API.get_ticker(pair.replace('_', '-')) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair})'.format( - message=data['message'], - pair=pair)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) keys = ['Bid', 'Ask', 'Last'] if not data.get('result') or\ not all(key in data.get('result', {}) for key in keys) or\ @@ -147,10 +162,13 @@ class Bittrex(Exchange): 'in response params=({})'.format(prop, pair)) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({pair})'.format( - message=data['message'], - pair=pair)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({pair})'.format( + message=data['message'], + pair=pair)) return data['result'] @@ -176,10 +194,13 @@ class Bittrex(Exchange): def cancel_order(self, order_id: str) -> None: data = _API.cancel(order_id) if not data['success']: - Bittrex._validate_response(data) - raise OperationalException('{message} params=({order_id})'.format( - message=data['message'], - order_id=order_id)) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException('{message} params=({order_id})'.format( + message=data['message'], + order_id=order_id)) def get_pair_detail_url(self, pair: str) -> str: return self.PAIR_DETAIL_METHOD + '?MarketName={}'.format(pair.replace('_', '-')) @@ -187,22 +208,31 @@ class Bittrex(Exchange): def get_markets(self) -> List[str]: data = _API.get_markets() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException(data['message']) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) return [m['MarketName'].replace('-', '_') for m in data['result']] def get_market_summaries(self) -> List[Dict]: data = _API.get_market_summaries() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException(data['message']) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) return data['result'] def get_wallet_health(self) -> List[Dict]: data = _API_V2.get_wallet_health() if not data['success']: - Bittrex._validate_response(data) - raise OperationalException(data['message']) + if 'APIKEY_INVALID' in str(data['message']): + print('Api Key...') + else: + Bittrex._validate_response(data) + raise OperationalException(data['message']) return [{ 'Currency': entry['Health']['Currency'], 'IsActive': entry['Health']['IsActive'], diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d8af47326..17ddcec09 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -113,7 +113,7 @@ class Backtesting(object): open_date=buy_row.date, stake_amount=stake_amount, amount=stake_amount / buy_row.open, - fee=exchange.get_fee() + fee=0.0025 ) # calculate win/lose forwards from buy point diff --git a/hyperopt-config.json b/hyperopt-config.json new file mode 100644 index 000000000..960e9e7e0 --- /dev/null +++ b/hyperopt-config.json @@ -0,0 +1,73 @@ +{ + "max_open_trades": 15, + "stake_currency": "BTC", + "stake_amount": 0.00075, + "fiat_display_currency": "USD", + "dry_run": false, + "unfilledtimeout": 600, + "ticker_interval": 5, + "bid_strategy": { + "ask_last_balance": 0.0 + }, + "exchange": { + "name": "bittrex", + "key": "XXXXXXXXXXXX", + "secret": "XXXXXXXX", + "pair_whitelist": [ + "BTC_ETH", + "BTC_XRP", + "BTC_BCC", + "BTC_LTC", + "BTC_ADA", + "BTC_XMR", + "BTC_DASH", + "BTC_TRX", + "BTC_ETC", + "BTC_ZEC", + "BTC_WAVES", + "BTC_STEEM", + "BTC_STRAT", + "BTC_DCR", + "BTC_REP", + "BTC_SNT", + "BTC_KMD", + "BTC_ARK", + "BTC_ARDR", + "BTC_MONA", + "BTC_DGB", + "BTC_PIVX", + "BTC_SYS", + "BTC_FCT", + "BTC_BAT", + "BTC_GNT", + "BTC_XZC", + "BTC_EMC", + "BTC_NXT", + "BTC_SALT", + "BTC_PAY", + "BTC_PART", + "BTC_GBYTE", + "BTC_BNT", + "BTC_POWR", + "BTC_NXS", + "BTC_SRN" + ], + "pair_blacklist": [ + "BTC_DOGE" + ] + }, + "experimental": { + "use_sell_signal": false, + "sell_profit_only": false + }, + "telegram": { + "enabled": true, + "token": "XXXXXXXXXXXXXX", + "chat_id": "XXXXXXXXXXX" + }, + "initial_state": "running", + "internals": { + "process_throttle_secs": 5 + } +} + diff --git a/hyperopt-mongodb-note.md b/hyperopt-mongodb-note.md new file mode 100644 index 000000000..1e88bd0e0 --- /dev/null +++ b/hyperopt-mongodb-note.md @@ -0,0 +1,49 @@ +Basically, to sum things up: + +The locations of your hyperopt directories may vary. Some reside in .local. + +Make these changes per this PR in these files: + +https://github.com/hyperopt/hyperopt/pull/287/files + +/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py + +removetrial['state'] == base.JOB_STATE_RUNNING + +addtrial['state'] = base.JOB_STATE_RUNNING + +Make these changes per this PR into the hyperopt files: + +https://github.com/hyperopt/hyperopt/pull/288/files + +remove:import six.moves.cPickle as pickle + +add: import dill as pickle in the following files: + +/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py + +/usr/local/lib/python3.6/dist-packages/hyperopt/main.py + +/usr/local/lib/python3.6/dist-packages/hyperopt/mongoexp.py + +/usr/local/lib/python3.6/dist-packages/hyperopt/utils.py + +Install dill: + +pip3.6 install dill + +replace freqtrade/optimize/hyperopt.py with the hyperopt-mongodb.py in the freqtrade root. + +Management tool for mongodb: + +https://robomongo.org/download + + +Then, run the scipts. 5000 epochs can take a few hours even on a high-end machine. You can use the mongodb took above to check the progress on the local machine. + + +Lastly, read the test_strategy.py file located in user_data/strategy/test_strategy.py + +Once you've done that, run scripts. You can restart the worker and the freqtrader hyperopt process but I do not advise it. Monitor the output with mongodb. + + diff --git a/hyperopt-mongodb.py b/hyperopt-mongodb.py new file mode 100644 index 000000000..2a4131efe --- /dev/null +++ b/hyperopt-mongodb.py @@ -0,0 +1,615 @@ +# pragma pylint: disable=too-many-instance-attributes, pointless-string-statement + +""" +This module contains the hyperopt logic +""" + +import json +import logging +import os +import pickle +import signal +import sys +from argparse import Namespace +from functools import reduce +from math import exp +from operator import itemgetter +from typing import Dict, Any, Callable + +import numpy +import talib.abstract as ta +from hyperopt import STATUS_FAIL, STATUS_OK, Trials, fmin, hp, space_eval, tpe +from hyperopt.mongoexp import MongoTrials +from pandas import DataFrame + +import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.logger import Logger +from freqtrade.optimize import load_data +from freqtrade.optimize.backtesting import Backtesting +from user_data.hyperopt_conf import hyperopt_optimize_conf + + + +class Testz(): + db_name = 'freqtrade_hyperopt' + MongoTrials( + arg='mongo://127.0.0.1:1234/{}/jobs'.format(db_name), + exp_key='exp1' + ) + + +class Hyperopt(Backtesting): + """ + Hyperopt class, this class contains all the logic to run a hyperopt simulation + + To run a backtest: + hyperopt = Hyperopt(config) + hyperopt.start() + """ + def __init__(self, config: Dict[str, Any]) -> None: + + super().__init__(config) + + # Rename the logging to display Hyperopt file instead of Backtesting + self.logging = Logger(name=__name__, level=config['loglevel']) + self.logger = self.logging.get_logger() + + # set TARGET_TRADES to suit your number concurrent trades so its realistic + # to the number of days + self.target_trades = 600 + self.total_tries = config.get('epochs', 0) + self.current_tries = 0 + self.current_best_loss = 100 + + # max average trade duration in minutes + # if eval ends with higher value, we consider it a failed eval + self.max_accepted_trade_duration = 300 + + # this is expexted avg profit * expected trade count + # for example 3.5%, 1100 trades, self.expected_max_profit = 3.85 + # check that the reported Σ% values do not exceed this! + self.expected_max_profit = 3.0 + + # Configuration and data used by hyperopt + self.processed = None + + # Hyperopt Trials + self.trials_file = os.path.join('user_data', 'hyperopt_trials.pickle') + self.trials = Testz() + + @staticmethod + def populate_indicators(dataframe: DataFrame) -> DataFrame: + """ + Adds several different TA indicators to the given DataFrame + """ + dataframe['adx'] = ta.ADX(dataframe) + dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) + dataframe['cci'] = ta.CCI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + dataframe['mfi'] = ta.MFI(dataframe) + dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + dataframe['minus_di'] = ta.MINUS_DI(dataframe) + dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + dataframe['plus_di'] = ta.PLUS_DI(dataframe) + dataframe['roc'] = ta.ROC(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) + rsi = 0.1 * (dataframe['rsi'] - 50) + dataframe['fisher_rsi'] = (numpy.exp(2 * rsi) - 1) / (numpy.exp(2 * rsi) + 1) + # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) + dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + # Stoch + stoch = ta.STOCH(dataframe) + dataframe['slowd'] = stoch['slowd'] + dataframe['slowk'] = stoch['slowk'] + # Stoch fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + # Stoch RSI + stoch_rsi = ta.STOCHRSI(dataframe) + dataframe['fastd_rsi'] = stoch_rsi['fastd'] + dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # Bollinger bands + bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) + dataframe['bb_lowerband'] = bollinger['lower'] + dataframe['bb_middleband'] = bollinger['mid'] + dataframe['bb_upperband'] = bollinger['upper'] + # EMA - Exponential Moving Average + dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) + dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) + dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) + dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) + # SAR Parabolic + dataframe['sar'] = ta.SAR(dataframe) + # SMA - Simple Moving Average + dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # TEMA - Triple Exponential Moving Average + dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) + # Hilbert Transform Indicator - SineWave + hilbert = ta.HT_SINE(dataframe) + dataframe['htsine'] = hilbert['sine'] + dataframe['htleadsine'] = hilbert['leadsine'] + + # Pattern Recognition - Bullish candlestick patterns + # ------------------------------------ + """ + # Hammer: values [0, 100] + dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) + # Inverted Hammer: values [0, 100] + dataframe['CDLINVERTEDHAMMER'] = ta.CDLINVERTEDHAMMER(dataframe) + # Dragonfly Doji: values [0, 100] + dataframe['CDLDRAGONFLYDOJI'] = ta.CDLDRAGONFLYDOJI(dataframe) + # Piercing Line: values [0, 100] + dataframe['CDLPIERCING'] = ta.CDLPIERCING(dataframe) # values [0, 100] + # Morningstar: values [0, 100] + dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] + # Three White Soldiers: values [0, 100] + dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] + """ + + # Pattern Recognition - Bearish candlestick patterns + # ------------------------------------ + """ + # Hanging Man: values [0, 100] + dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) + # Shooting Star: values [0, 100] + dataframe['CDLSHOOTINGSTAR'] = ta.CDLSHOOTINGSTAR(dataframe) + # Gravestone Doji: values [0, 100] + dataframe['CDLGRAVESTONEDOJI'] = ta.CDLGRAVESTONEDOJI(dataframe) + # Dark Cloud Cover: values [0, 100] + dataframe['CDLDARKCLOUDCOVER'] = ta.CDLDARKCLOUDCOVER(dataframe) + # Evening Doji Star: values [0, 100] + dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) + # Evening Star: values [0, 100] + dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) + """ + + # Pattern Recognition - Bullish/Bearish candlestick patterns + # ------------------------------------ + """ + # Three Line Strike: values [0, -100, 100] + dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) + # Spinning Top: values [0, -100, 100] + dataframe['CDLSPINNINGTOP'] = ta.CDLSPINNINGTOP(dataframe) # values [0, -100, 100] + # Engulfing: values [0, -100, 100] + dataframe['CDLENGULFING'] = ta.CDLENGULFING(dataframe) # values [0, -100, 100] + # Harami: values [0, -100, 100] + dataframe['CDLHARAMI'] = ta.CDLHARAMI(dataframe) # values [0, -100, 100] + # Three Outside Up/Down: values [0, -100, 100] + dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] + # Three Inside Up/Down: values [0, -100, 100] + dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] + """ + + # Chart type + # ------------------------------------ + # Heikinashi stategy + heikinashi = qtpylib.heikinashi(dataframe) + dataframe['ha_open'] = heikinashi['open'] + dataframe['ha_close'] = heikinashi['close'] + dataframe['ha_high'] = heikinashi['high'] + dataframe['ha_low'] = heikinashi['low'] + + return dataframe + + def save_trials(self) -> None: + """ + Save hyperopt trials to file + """ + self.logger.info('Saving Trials to \'%s\'', self.trials_file) + pickle.dump(self.trials, open(self.trials_file, 'wb')) + + def read_trials(self) -> Trials: + """ + Read hyperopt trials file + """ + self.logger.info('Reading Trials from \'%s\'', self.trials_file) + trials = pickle.load(open(self.trials_file, 'rb')) + os.remove(self.trials_file) + return trials + + def log_trials_result(self) -> None: + """ + Display Best hyperopt result + """ + vals = json.dumps(self.trials.best_trial['misc']['vals'], indent=4) + results = self.trials.best_trial['result']['result'] + self.logger.info('Best result:\n%s\nwith values:\n%s', results, vals) + + def log_results(self, results) -> None: + """ + Log results if it is better than any previous evaluation + """ + if results['loss'] < self.current_best_loss: + self.current_best_loss = results['loss'] + log_msg = '{:5d}/{}: {}. Loss {:.5f}'.format( + results['current_tries'], + results['total_tries'], + results['result'], + results['loss'] + ) + self.logger.info(log_msg) + else: + print('.', end='') + sys.stdout.flush() + + def calculate_loss(self, total_profit: float, trade_count: int, trade_duration: float) -> float: + """ + Objective function, returns smaller number for more optimal results + """ + trade_loss = 1 - 0.25 * exp(-(trade_count - self.target_trades) ** 2 / 10 ** 5.8) + profit_loss = max(0, 1 - total_profit / self.expected_max_profit) + duration_loss = 0.4 * min(trade_duration / self.max_accepted_trade_duration, 1) + return trade_loss + profit_loss + duration_loss + + @staticmethod + def generate_roi_table(params: Dict) -> Dict[int, float]: + """ + Generate the ROI table thqt will be used by Hyperopt + """ + roi_table = {} + roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] + roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] + roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] + roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 + + return roi_table + + @staticmethod + def roi_space() -> Dict[str, Any]: + """ + Values to search for each ROI steps + """ + return { + 'roi_t1': hp.quniform('roi_t1', 10, 120, 20), + 'roi_t2': hp.quniform('roi_t2', 10, 60, 15), + 'roi_t3': hp.quniform('roi_t3', 10, 40, 10), + 'roi_p1': hp.quniform('roi_p1', 0.01, 0.04, 0.01), + 'roi_p2': hp.quniform('roi_p2', 0.01, 0.07, 0.01), + 'roi_p3': hp.quniform('roi_p3', 0.01, 0.20, 0.01), + } + + @staticmethod + def stoploss_space() -> Dict[str, Any]: + """ + Stoploss Value to search + """ + return { + 'stoploss': hp.quniform('stoploss', -0.5, -0.02, 0.02), + } + + @staticmethod + def indicator_space() -> Dict[str, Any]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 10, 25, 5)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 15, 45, 5)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 20, 50, 5)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 5)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + } + + def has_space(self, space: str) -> bool: + """ + Tell if a space value is contained in the configuration + """ + if space in self.config['spaces'] or 'all' in self.config['spaces']: + return True + return False + + def hyperopt_space(self) -> Dict[str, Any]: + """ + Return the space to use during Hyperopt + """ + spaces = {} + if self.has_space('buy'): + spaces = {**spaces, **Hyperopt.indicator_space()} + if self.has_space('roi'): + spaces = {**spaces, **Hyperopt.roi_space()} + if self.has_space('stoploss'): + spaces = {**spaces, **Hyperopt.stoploss_space()} + return spaces + + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + """ + Buy strategy Hyperopt will build and use + """ + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend + + def generate_optimizer(self, params: Dict) -> Dict: + if self.has_space('roi'): + self.analyze.strategy.minimal_roi = self.generate_roi_table(params) + + if self.has_space('buy'): + self.populate_buy_trend = self.buy_strategy_generator(params) + + if self.has_space('stoploss'): + self.analyze.strategy.stoploss = params['stoploss'] + + results = self.backtest( + { + 'stake_amount': self.config['stake_amount'], + 'processed': self.processed, + 'realistic': self.config.get('realistic_simulation', False), + } + ) + result_explanation = self.format_results(results) + + total_profit = results.profit_percent.sum() + trade_count = len(results.index) + trade_duration = results.duration.mean() + + if trade_count == 0 or trade_duration > self.max_accepted_trade_duration: + print('.', end='') + return { + 'status': STATUS_FAIL, + 'loss': float('inf') + } + + loss = self.calculate_loss(total_profit, trade_count, trade_duration) + + self.current_tries += 1 + + self.log_results( + { + 'loss': loss, + 'current_tries': self.current_tries, + 'total_tries': self.total_tries, + 'result': result_explanation, + } + ) + + return { + 'loss': loss, + 'status': STATUS_OK, + 'result': result_explanation, + } + + @staticmethod + def format_results(results: DataFrame) -> str: + """ + Return the format result in a string + """ + return ('{:6d} trades. Avg profit {: 5.2f}%. ' + 'Total profit {: 11.8f} BTC ({:.4f}Σ%). Avg duration {:5.1f} mins.').format( + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_BTC.sum(), + results.profit_percent.sum(), + results.duration.mean(), + ) + + def start(self) -> None: + timerange = Arguments.parse_timerange(self.config.get('timerange')) + data = load_data( + datadir=self.config.get('datadir'), + pairs=self.config['exchange']['pair_whitelist'], + ticker_interval=self.ticker_interval, + timerange=timerange + ) + + if self.has_space('buy'): + self.analyze.populate_indicators = Hyperopt.populate_indicators + self.processed = self.tickerdata_to_dataframe(data) + + if self.config.get('mongodb'): + self.logger.info('Using mongodb ...') + self.logger.info( + 'Start scripts/start-mongodb.sh and start-hyperopt-worker.sh manually!' + ) + else: + self.logger.info('Preparing Trials..') + signal.signal(signal.SIGINT, self.signal_handler) + # read trials file if we have one + if os.path.exists(self.trials_file) and os.path.getsize(self.trials_file) > 0: + self.trials = self.read_trials() + + self.current_tries = len(self.trials.results) + self.total_tries += self.current_tries + self.logger.info( + 'Continuing with trials. Current: %d, Total: %d', + self.current_tries, + self.total_tries + ) + try: + best_parameters = fmin( + fn=self.generate_optimizer, + space=self.hyperopt_space(), + algo=tpe.suggest, + max_evals=self.total_tries, + trials=MongoTrials(arg='mongo://127.0.0.1:1234/freqtrade_hyperopt/jobs', exp_key='exp1') + ) + # change the Logging format + self.logging.set_format('\n%(message)s') + results = sorted(self.trials.results, key=itemgetter('loss')) + best_result = results[0]['result'] + # Improve best parameter logging display + except ValueError: + best_parameters = {} + best_result = 'Sorry, Hyperopt was not able to find good parameters. Please ' \ + 'try with more epochs (param: -e).' + if best_parameters: + best_parameters = space_eval( + self.hyperopt_space(), + best_parameters + ) + + self.logger.info('Best parameters:\n%s', json.dumps(best_parameters, indent=4)) + if 'roi_t1' in best_parameters: + self.logger.info('ROI table:\n%s', self.generate_roi_table(best_parameters)) + + self.logger.info('Best Result:\n%s', best_result) + + # Store trials result to file to resume next time + self.save_trials() + + def signal_handler(self, sig, frame) -> None: + """ + Hyperopt SIGINT handler + """ + self.logger.info( + 'Hyperopt received %s', + signal.Signals(sig).name + ) + + self.save_trials() + self.log_trials_result() + sys.exit(0) + + + +def start(args: Namespace) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + + # Remove noisy log messages + logging.getLogger('hyperopt.mongoexp').setLevel(logging.WARNING) + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) + + # Initialize logger + logger = Logger(name=__name__).get_logger() + logger.info('Starting freqtrade in Hyperopt mode') + + # Initialize configuration + # Monkey patch the configuration with hyperopt_conf.py + configuration = Configuration(args) + optimize_config = hyperopt_optimize_conf() + config = configuration._load_common_config(optimize_config) + config = configuration._load_backtesting_config(config) + config = configuration._load_hyperopt_config(config) + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() diff --git a/scripts/start-hyperopt-worker.py b/scripts/start-hyperopt-worker.py index 8b0ae6326..b04ad1029 100755 --- a/scripts/start-hyperopt-worker.py +++ b/scripts/start-hyperopt-worker.py @@ -19,6 +19,7 @@ command = [ '--mongo=127.0.0.1:1234/{}'.format(DB_NAME), '--poll-interval=0.1', '--workdir={}'.format(WORK_DIR), + '--reserve-timeout=900', ] processes = [subprocess.Popen(command) for i in range(PROC_COUNT)] diff --git a/user_data/hyperopt_conf.py b/user_data/hyperopt_conf.py index 8e044e549..9c31b3b84 100644 --- a/user_data/hyperopt_conf.py +++ b/user_data/hyperopt_conf.py @@ -11,31 +11,52 @@ def hyperopt_optimize_conf() -> dict: :return: """ return { - 'max_open_trades': 3, + 'max_open_trades': 15, 'stake_currency': 'BTC', - 'stake_amount': 0.01, - "minimal_roi": { - '40': 0.0, - '30': 0.01, - '20': 0.02, - '0': 0.04, - }, - 'stoploss': -0.10, + 'stake_amount': 0.00075, + 'ticker_interval': 5, "bid_strategy": { "ask_last_balance": 0.0 }, "exchange": { "pair_whitelist": [ - "BTC_ETH", - "BTC_LTC", - "BTC_ETC", - "BTC_DASH", - "BTC_ZEC", - "BTC_XLM", - "BTC_NXT", - "BTC_POWR", - "BTC_ADA", - "BTC_XMR" + 'BTC_ETH', + 'BTC_XRP', + 'BTC_BCC', + 'BTC_LTC', + 'BTC_ADA', + 'BTC_XMR', + 'BTC_DASH', + 'BTC_TRX', + 'BTC_ETC', + 'BTC_ZEC', + 'BTC_WAVES', + 'BTC_STEEM', + 'BTC_STRAT', + 'BTC_DCR', + 'BTC_REP', + 'BTC_SNT', + 'BTC_KMD', + 'BTC_ARK', + 'BTC_ARDR', + 'BTC_MONA', + 'BTC_DGB', + 'BTC_PIVX', + 'BTC_SYS', + 'BTC_FCT', + 'BTC_BAT', + 'BTC_GNT', + 'BTC_XZC', + 'BTC_EMC', + 'BTC_NXT', + 'BTC_SALT', + 'BTC_PAY', + 'BTC_PART', + 'BTC_GBYTE', + 'BTC_BNT', + 'BTC_POWR', + 'BTC_NXS', + 'BTC_SRN' ] } } diff --git a/user_data/strategies/test_strategy.py b/user_data/strategies/test_strategy.py index a164812c4..6f26c7b48 100644 --- a/user_data/strategies/test_strategy.py +++ b/user_data/strategies/test_strategy.py @@ -1,6 +1,9 @@ # --- Do not remove these libs --- from freqtrade.strategy.interface import IStrategy +from typing import Dict, List +from hyperopt import hp +from functools import reduce from pandas import DataFrame # -------------------------------- @@ -10,12 +13,40 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib import numpy # noqa -# Update this variable if you change the class name -class_name = 'TestStrategy' +# Make a backup of default_strategy then move this file over your default strategy in freqtrade/strategy and to run: +# 'python3.6 freqtrade/main.py -c config.json hyperopt -s all --realistic-simulation -i 5' +# Make sure to take note of the if statements at the bottom of the file, and the pattern of > < and >= <= and the triggers configuration. +# Then, take note of the results: There is one trigger in the results, map the trigger as an example: +# 'Trigger: 7' +# The 7th trigger in this file's if statement such as: +# 'sar_reversal': (qtpylib.crossed_above( +# dataframe['close'], dataframe['sar'] +# +# Drop the trigger into 'def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:' into your default_strategy backup as: +# dataframe['close'], dataframe['sar'] & +# Then for your other values such as: 'Mfi-Value: 15.00' +# The if statement looks like: conditions.append(dataframe['mfi'] < params['mfi']['value'] +# Drop this in as: +# (dataframe['mfi'] < 15.00) & +# The last condition in 'def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:' will have no & -# This class is a sample. Feel free to customize it. -class TestStrategy(IStrategy): +### As an example the sell/buy trend looks like this: +# dataframe.loc[ +# ( +# (dataframe['adx'] > 70) & +# (dataframe['tema'] < dataframe['tema'].shift(1)) +# ), +# 'sell'] = 1 +# + +# Once done, move your backup strategy back over this one and backtest. + + +class_name = 'DefaultStrategy' + + +class DefaultStrategy(IStrategy): """ This is a test strategy to inspire you. More information in https://github.com/gcarq/freqtrade/blob/develop/docs/bot-optimization.md @@ -62,7 +93,7 @@ class TestStrategy(IStrategy): # ADX dataframe['adx'] = ta.ADX(dataframe) - """ + # Awesome oscillator dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) @@ -114,18 +145,28 @@ class TestStrategy(IStrategy): stoch_rsi = ta.STOCHRSI(dataframe) dataframe['fastd_rsi'] = stoch_rsi['fastd'] dataframe['fastk_rsi'] = stoch_rsi['fastk'] - """ + # Overlap Studies # ------------------------------------ + + # Previous Bollinger bands + # Because ta.BBANDS implementation is broken with small numbers, it actually + # returns middle band for all the three bands. Switch to qtpylib.bollinger_bands + # and use middle band instead. + # Is broken + """ + dataframe['blower'] = ta.BBANDS(dataframe, nbdevup=2, nbdevdn=2)['lowerband'] + """ + # Bollinger bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] - """ + # EMA - Exponential Moving Average dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) @@ -138,7 +179,6 @@ class TestStrategy(IStrategy): # SMA - Simple Moving Average dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) - """ # TEMA - Triple Exponential Moving Average dataframe['tema'] = ta.TEMA(dataframe, timeperiod=9) @@ -152,7 +192,7 @@ class TestStrategy(IStrategy): # Pattern Recognition - Bullish candlestick patterns # ------------------------------------ - """ + # Hammer: values [0, 100] dataframe['CDLHAMMER'] = ta.CDLHAMMER(dataframe) # Inverted Hammer: values [0, 100] @@ -165,11 +205,11 @@ class TestStrategy(IStrategy): dataframe['CDLMORNINGSTAR'] = ta.CDLMORNINGSTAR(dataframe) # values [0, 100] # Three White Soldiers: values [0, 100] dataframe['CDL3WHITESOLDIERS'] = ta.CDL3WHITESOLDIERS(dataframe) # values [0, 100] - """ + # Pattern Recognition - Bearish candlestick patterns # ------------------------------------ - """ + # Hanging Man: values [0, 100] dataframe['CDLHANGINGMAN'] = ta.CDLHANGINGMAN(dataframe) # Shooting Star: values [0, 100] @@ -182,11 +222,10 @@ class TestStrategy(IStrategy): dataframe['CDLEVENINGDOJISTAR'] = ta.CDLEVENINGDOJISTAR(dataframe) # Evening Star: values [0, 100] dataframe['CDLEVENINGSTAR'] = ta.CDLEVENINGSTAR(dataframe) - """ - + # Pattern Recognition - Bullish/Bearish candlestick patterns # ------------------------------------ - """ + # Three Line Strike: values [0, -100, 100] dataframe['CDL3LINESTRIKE'] = ta.CDL3LINESTRIKE(dataframe) # Spinning Top: values [0, -100, 100] @@ -199,18 +238,18 @@ class TestStrategy(IStrategy): dataframe['CDL3OUTSIDE'] = ta.CDL3OUTSIDE(dataframe) # values [0, -100, 100] # Three Inside Up/Down: values [0, -100, 100] dataframe['CDL3INSIDE'] = ta.CDL3INSIDE(dataframe) # values [0, -100, 100] - """ + # Chart type # ------------------------------------ - """ + # Heikinashi stategy heikinashi = qtpylib.heikinashi(dataframe) dataframe['ha_open'] = heikinashi['open'] dataframe['ha_close'] = heikinashi['close'] dataframe['ha_high'] = heikinashi['high'] dataframe['ha_low'] = heikinashi['low'] - """ + return dataframe @@ -220,12 +259,65 @@ class TestStrategy(IStrategy): :param dataframe: DataFrame :return: DataFrame with buy column """ - dataframe.loc[ - ( - (dataframe['adx'] > 30) & - (dataframe['tema'] <= dataframe['bb_middleband']) & - (dataframe['tema'] > dataframe['tema'].shift(1)) + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), 'buy'] = 1 return dataframe @@ -239,8 +331,142 @@ class TestStrategy(IStrategy): dataframe.loc[ ( (dataframe['adx'] > 70) & - (dataframe['tema'] > dataframe['bb_middleband']) & (dataframe['tema'] < dataframe['tema'].shift(1)) ), 'sell'] = 1 return dataframe + + def hyperopt_space(self) -> List[Dict]: + """ + Define your Hyperopt space for the strategy + :return: Dict + """ + space = { + 'macd_below_zero': hp.choice('macd_below_zero', [ + {'enabled': False}, + {'enabled': True} + ]), + 'mfi': hp.choice('mfi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('mfi-value', 5, 25, 1)} + ]), + 'fastd': hp.choice('fastd', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('fastd-value', 10, 50, 1)} + ]), + 'adx': hp.choice('adx', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('adx-value', 15, 50, 1)} + ]), + 'rsi': hp.choice('rsi', [ + {'enabled': False}, + {'enabled': True, 'value': hp.quniform('rsi-value', 20, 40, 1)} + ]), + 'uptrend_long_ema': hp.choice('uptrend_long_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_short_ema': hp.choice('uptrend_short_ema', [ + {'enabled': False}, + {'enabled': True} + ]), + 'over_sar': hp.choice('over_sar', [ + {'enabled': False}, + {'enabled': True} + ]), + 'green_candle': hp.choice('green_candle', [ + {'enabled': False}, + {'enabled': True} + ]), + 'uptrend_sma': hp.choice('uptrend_sma', [ + {'enabled': False}, + {'enabled': True} + ]), + 'trigger': hp.choice('trigger', [ + {'type': 'lower_bb'}, + {'type': 'lower_bb_tema'}, + {'type': 'faststoch10'}, + {'type': 'ao_cross_zero'}, + {'type': 'ema3_cross_ema10'}, + {'type': 'macd_cross_signal'}, + {'type': 'sar_reversal'}, + {'type': 'ht_sine'}, + {'type': 'heiken_reversal_bull'}, + {'type': 'di_cross'}, + ]), + 'stoploss': hp.uniform('stoploss', -0.5, -0.02), + } + return space + + def buy_strategy_generator(self, params) -> None: + """ + Define the buy strategy parameters to be used by hyperopt + """ + def populate_buy_trend(dataframe: DataFrame) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'uptrend_long_ema' in params and params['uptrend_long_ema']['enabled']: + conditions.append(dataframe['ema50'] > dataframe['ema100']) + if 'macd_below_zero' in params and params['macd_below_zero']['enabled']: + conditions.append(dataframe['macd'] < 0) + if 'uptrend_short_ema' in params and params['uptrend_short_ema']['enabled']: + conditions.append(dataframe['ema5'] > dataframe['ema10']) + if 'mfi' in params and params['mfi']['enabled']: + conditions.append(dataframe['mfi'] < params['mfi']['value']) + if 'fastd' in params and params['fastd']['enabled']: + conditions.append(dataframe['fastd'] < params['fastd']['value']) + if 'adx' in params and params['adx']['enabled']: + conditions.append(dataframe['adx'] > params['adx']['value']) + if 'rsi' in params and params['rsi']['enabled']: + conditions.append(dataframe['rsi'] < params['rsi']['value']) + if 'over_sar' in params and params['over_sar']['enabled']: + conditions.append(dataframe['close'] > dataframe['sar']) + if 'green_candle' in params and params['green_candle']['enabled']: + conditions.append(dataframe['close'] > dataframe['open']) + if 'uptrend_sma' in params and params['uptrend_sma']['enabled']: + prevsma = dataframe['sma'].shift(1) + conditions.append(dataframe['sma'] > prevsma) + + # TRIGGERS + triggers = { + 'lower_bb': ( + dataframe['close'] < dataframe['bb_lowerband'] + ), + 'lower_bb_tema': ( + dataframe['tema'] < dataframe['bb_lowerband'] + ), + 'faststoch10': (qtpylib.crossed_above( + dataframe['fastd'], 10.0 + )), + 'ao_cross_zero': (qtpylib.crossed_above( + dataframe['ao'], 0.0 + )), + 'ema3_cross_ema10': (qtpylib.crossed_above( + dataframe['ema3'], dataframe['ema10'] + )), + 'macd_cross_signal': (qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )), + 'sar_reversal': (qtpylib.crossed_above( + dataframe['close'], dataframe['sar'] + )), + 'ht_sine': (qtpylib.crossed_above( + dataframe['htleadsine'], dataframe['htsine'] + )), + 'heiken_reversal_bull': ( + (qtpylib.crossed_above(dataframe['ha_close'], dataframe['ha_open'])) & + (dataframe['ha_low'] == dataframe['ha_open']) + ), + 'di_cross': (qtpylib.crossed_above( + dataframe['plus_di'], dataframe['minus_di'] + )), + } + conditions.append(triggers.get(params['trigger']['type'])) + + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend