From 90e3c387577cbc6992c9e8825dd86b0619a8dbac Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 14 Jul 2018 22:54:23 +0000 Subject: [PATCH 001/174] First cut, Bslap science project replacement for freqtrade backtest analysis - appprox 300-500x quicker to execute - fixes stop on close take close price bug in FT Bslap is configurable but by default stops are triggerd on low and pay stop price Not implimented dynamic stops or roi --- freqtrade/optimize/backtesting.py | 495 ++++++++++++++++++++++++++++-- 1 file changed, 463 insertions(+), 32 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 05bcdf4b7..14e4fe1f5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -21,6 +21,8 @@ from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade +from profilehooks import profile +from freqtrade.strategy.resolver import IStrategy, StrategyResolver logger = logging.getLogger(__name__) @@ -67,6 +69,8 @@ class Backtesting(object): self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() + self.stop_loss_value = self.analyze.strategy.stoploss + @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ @@ -186,6 +190,7 @@ class Backtesting(object): return btr return None + @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -201,55 +206,481 @@ class Backtesting(object): realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ - headers = ['date', 'buy', 'open', 'close', 'sell'] + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) trades = [] trade_count_lock: Dict = {} + ########################### Call out BSlap instead of using FT + bslap_results: list = [] + last_bslap_resultslist = [] for pair, pair_data in processed.items(): - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() + self.populate_buy_trend(pair_data))[headers].copy() - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + bslap_pair_results = self.backslap_pair(ticker_data, pair) - ticker_data.drop(ticker_data.head(1).index, inplace=True) + last_bslap_results = bslap_results + bslap_results = last_bslap_results + bslap_pair_results + bslap_results_df = DataFrame(bslap_results) + print(bslap_results_df.dtypes()) - # Convert from Pandas to list for performance reasons - # (Looping Pandas is slow.) - ticker = [x for x in ticker_data.itertuples()] + return bslap_results_df + ########################### Original BT loop + # for pair, pair_data in processed.items(): + # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + # + # ticker_data = self.populate_sell_trend( + # self.populate_buy_trend(pair_data))[headers].copy() + # + # # to avoid using data from future, we buy/sell with signal from previous candle + # ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + # ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + # + # ticker_data.drop(ticker_data.head(1).index, inplace=True) + # + # # Convert from Pandas to list for performance reasons + # # (Looping Pandas is slow.) + # ticker = [x for x in ticker_data.itertuples()] + # + # lock_pair_until = None + # for index, row in enumerate(ticker): + # if row.buy == 0 or row.sell == 1: + # continue # skip rows where no buy signal or that would immediately sell off + # + # if realistic: + # if lock_pair_until is not None and row.date <= lock_pair_until: + # continue + # if max_open_trades > 0: + # # Check if max_open_trades has already been reached for the given date + # if not trade_count_lock.get(row.date, 0) < max_open_trades: + # continue + # + # trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + # + # trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + # trade_count_lock, args) + # + # + # if trade_entry: + # lock_pair_until = trade_entry.close_time + # trades.append(trade_entry) + # else: + # # Set lock_pair_until to end of testing period if trade could not be closed + # # This happens only if the buy-signal was with the last candle + # lock_pair_until = ticker_data.iloc[-1].date + # + # return DataFrame.from_records(trades, columns=BacktestResult._fields) + ######################## Original BT loop end - lock_pair_until = None - for index, row in enumerate(ticker): - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. - if realistic: - if lock_pair_until is not None and row.date <= lock_pair_until: - continue - if max_open_trades > 0: - # Check if max_open_trades has already been reached for the given date - if not trade_count_lock.get(row.date, 0) < max_open_trades: - continue + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + """ + t_open_ind: int - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + # Create a view on our buy index starting after last trade exit + # Search for next buy + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + return t_open_ind - trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + def backslap_pair(self, ticker_data, pair): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st - if trade_entry: - lock_pair_until = trade_entry.close_time - trades.append(trade_entry) + stop = self.stop_loss_value + p_stop = (-stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + ### backslap debug wrap + debug_2loops = False # only loop twice, for faster debug + debug_timing = False # print timing for each step + debug = False # print values, to check accuracy + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + #### backslap config + """ + A couple legacy Pandas vars still used for pretty debug output. + If have debug enabled the code uses these fields for dataframe output + + Ensure bto, sto, sco are aligned with Numpy values next + to align debug and actual. Options are: + buy - open - close - sell - high - low - np_stop_pri + """ + bto = buys_triggered_on = "close" + sto = stops_triggered_on = "low" ## Should be low, FT uses close + sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + np_buy: int = 0 + np_open: int = 1 + np_close: int = 2 + np_sell: int = 3 + np_high: int = 4 + np_low: int = 5 + np_stop: int = 6 + np_bto: int = np_close # buys_triggered_on - should be close + np_bco: int = np_open # buys calculated on - open of the next candle. + np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close + np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + # + ### End Config + + pair: str = pair + loop: int = 1 + + #ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("----------------------------------------------------------- Loop", loop, pair) + if debug_2loops: + if loop == 4: + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first intance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + ''' + Finds index for first buy = 1 flag, use .values numpy array for speed + + Create a slice, from first buy index onwards. + Slice will be used to find exit conditions after trade open + ''' + if debug_timing: + st = s() + + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind) + + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Calculate np_t_stop_pri (Trade Stop Price) based on the buy price + + As stop in based on buy price we are interested in buy + - Buys are Triggered On np_bto, typically the CLOSE of candle + - Buys are Calculated On np_bco, default is OPEN of the next candle. + as we only see the CLOSE after it has happened. + The assumption is we have bought at first available price, the OPEN + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug_timing: + t_t = f(st) + print("1-numpy", str.format('{0:.17f}', t_t)) + st = s() + + """ + 1)Create a View from our open trade forward + + The view is our search space for the next Stop or Sell + We use a numpy view: + Using a numpy for speed on views, 1,000 faster than pandas + Pandas cannot assure it will always return a view, copies are + 3 orders of magnitude slower + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + """ + np_t_open_v = np_bslap[t_open_ind:] + + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Find first stop index after Trade Open: + + First index in np_t_open_v (numpy view of bslap dataframe) + Using a numpy view a orders of magnitude faster + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) \ + + t_open_ind + + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Find first sell index after trade open + + First index in t_open_slice where ['sell'] = 1 + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) \ + + t_open_ind + + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + Determine which was hit first stop or sell, use as exit + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + ''' + if np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = np_t_stop_ind # Set Exit row index + t_exit_type = 'stop' # Set Exit type (sell|stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + else: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = np_t_sell_ind # Set Exit row index + t_exit_type = 'sell' # Set Exit type (sell|stop) + np_t_exit_pri = np_open # The price field our SELL exit will use + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the currenct stop price based on buy price_ + # Don't care about performance in debug + # (add an extra column if printing as df has date in col1 not in npy) + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("=================== BUY ", pair) + print("Numpy Array BUY Index is:", t_open_ind) + print("DataFrame BUY Index is:", t_open_ind + 1, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 2, "\n") + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + print(bslap.iloc[t_open_ind + 1]['date']) + + # Stop - Stops trigger price sto, and price received sco. (Stop Trigger|Calculated On) + print("=================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind) + print("DataFrame STOP Index is:", np_t_stop_ind + 1, "displaying DF \n") + print("First Stop Index after Trade open in candle", np_t_stop_ind + 1, ": \n", + str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sto]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + print("Tokens will be sold at rate:", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) + print("HINT, STOPs should close in-candle, i.e", np_t_stop_ind + 1, + ": As live STOPs are not linked to O-C times") + + st_is = np_t_stop_ind - 1 # Print stop index start, line before + st_if = np_t_stop_ind + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + print("=================== SELL ", pair) + print("Numpy Array SELL Index is:", np_t_sell_ind) + print("DataFrame SELL Index is:", np_t_sell_ind + 1, "displaying DF \n") + print("First Sell Index after Trade open in in candle", np_t_sell_ind + 1) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + np_t_sell_ind + 2, "\n") + sl_is = np_t_sell_ind - 1 # Print sell index start, line before + sl_if = np_t_sell_ind + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + print("=================== EXIT ", pair) + print("Exit type is :", t_exit_type) + # print((bslap.iloc[t_exit_ind], "\n")) + print("trade exit price field is", np_t_exit_pri, "\n") + + ''' + Trade entry is always the next candles "open" price + We trigger on close, so cannot see that till after + its closed. + + The exception to this is a STOP which is calculated in candle + ''' + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == 'stop': + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri else: - # Set lock_pair_until to end of testing period if trade could not be closed - # This happens only if the buy-signal was with the last candle - lock_pair_until = ticker_data.iloc[-1].date + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == 'sell': + np_trade_exit_price = np_bslap[t_exit_ind + 1, np_t_exit_pri] + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enterprice is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop - 1, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind: + """ + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + + Break loop and go on to next pair. + + TODO + add handing here to record none closed open trades + """ + + if debug: + print(bslap_pair_results) + + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + # Index will change if incandle stop or look back as close Open and Sell + if t_exit_type == 'stop': + close_index: int = t_exit_ind + 1 + elif t_exit_type == 'sell': + close_index: int = t_exit_ind + 2 + else: + close_index: int = t_exit_ind + 1 + # Munge the date / delta + start_date: str = bslap.iloc[t_open_ind + 1]['date'] + end_date: str = bslap.iloc[close_index]['date'] + + def __datetime(date_str): + return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') + + # trade_start = __datetime(start_date) + # trade_end = __datetime(end_date) + # trade_mins = (trade_end - trade_start).total_seconds() / 60 + # build trade dictionary + bslap_result["pair"] = pair + bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price) // np_trade_enter_price * 100 + bslap_result["profit_abs"] = "" + bslap_result["open_time"] = start_date + bslap_result["close_time"] = end_date + bslap_result["open_index"] = t_open_ind + 1 + bslap_result["close_index"] = close_index + # bslap_result["trade_duration"] = trade_mins + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = str.format('{0:.10f}', np_trade_enter_price) + bslap_result["close_rate"] = str.format('{0:.10f}', np_trade_exit_price) + bslap_result["exit_type"] = t_exit_type + # Add trade dictionary to list + bslap_pair_results.append(bslap_result) + if debug: + print(bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results + + - return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ From 07175ebc5a9d3fc2a068bbd21727f356e00c1930 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sat, 14 Jul 2018 23:45:06 +0000 Subject: [PATCH 002/174] up --- freqtrade/optimize/backtesting.py | 37 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 14e4fe1f5..9110694ce 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -22,7 +22,7 @@ from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from profilehooks import profile -from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from collections import OrderedDict logger = logging.getLogger(__name__) @@ -190,7 +190,7 @@ class Backtesting(object): return btr return None - @profile + def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -218,15 +218,18 @@ class Backtesting(object): for pair, pair_data in processed.items(): ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) bslap_pair_results = self.backslap_pair(ticker_data, pair) last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results - bslap_results_df = DataFrame(bslap_results) - print(bslap_results_df.dtypes()) + #bslap_results_df = DataFrame(bslap_results) - return bslap_results_df + res = DataFrame.from_records(bslap_results, columns=BacktestResult._fields) + print(res) + return res ########################### Original BT loop # for pair, pair_data in processed.items(): # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -297,14 +300,15 @@ class Backtesting(object): import numpy as np import timeit import utils_find_1st as utf1st + from datetime import datetime stop = self.stop_loss_value p_stop = (-stop + 1) # What stop really means, e.g 0.01 is 0.99 of price ### backslap debug wrap - debug_2loops = False # only loop twice, for faster debug + debug_2loops = True # only loop twice, for faster debug debug_timing = False # print timing for each step - debug = False # print values, to check accuracy + debug = True # print values, to check accuracy if debug: from pandas import set_option set_option('display.max_rows', 5000) @@ -638,24 +642,25 @@ class Backtesting(object): else: close_index: int = t_exit_ind + 1 # Munge the date / delta - start_date: str = bslap.iloc[t_open_ind + 1]['date'] - end_date: str = bslap.iloc[close_index]['date'] + start_date = bslap.iloc[t_open_ind + 1]['date'] + end_date = bslap.iloc[close_index]['date'] - def __datetime(date_str): - return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') + # def __datetime(date_str): + # return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') + + trade_start = start_date + trade_end = end_date + trade_mins = (trade_end - trade_start).total_seconds() / 60 - # trade_start = __datetime(start_date) - # trade_end = __datetime(end_date) - # trade_mins = (trade_end - trade_start).total_seconds() / 60 # build trade dictionary bslap_result["pair"] = pair - bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price) // np_trade_enter_price * 100 + bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price)/np_trade_enter_price bslap_result["profit_abs"] = "" bslap_result["open_time"] = start_date bslap_result["close_time"] = end_date bslap_result["open_index"] = t_open_ind + 1 bslap_result["close_index"] = close_index - # bslap_result["trade_duration"] = trade_mins + bslap_result["trade_duration"] = trade_mins bslap_result["open_at_end"] = False bslap_result["open_rate"] = str.format('{0:.10f}', np_trade_enter_price) bslap_result["close_rate"] = str.format('{0:.10f}', np_trade_exit_price) From 71c3106f8f94cd480c404b20d46c8b8abf555142 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 15 Jul 2018 09:30:01 +0000 Subject: [PATCH 003/174] Added ABS and Fees Fixed Index Alignment that was off moving from scratch to FT Fixed Stoploss, its a negative in FT, had been using positve stop -1 in scratch --- freqtrade/optimize/backtesting.py | 77 ++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9110694ce..3cfb588b9 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -214,22 +214,28 @@ class Backtesting(object): trade_count_lock: Dict = {} ########################### Call out BSlap instead of using FT bslap_results: list = [] - last_bslap_resultslist = [] + last_bslap_results: list = [] + for pair, pair_data in processed.items(): ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + ticker_data.drop(ticker_data.head(1).index, inplace=True) + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + #call bslap - results are a list of dicts bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results - #bslap_results_df = DataFrame(bslap_results) - res = DataFrame.from_records(bslap_results, columns=BacktestResult._fields) - print(res) - return res + bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + return bslap_results_df + ########################### Original BT loop # for pair, pair_data in processed.items(): # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -302,13 +308,28 @@ class Backtesting(object): import utils_find_1st as utf1st from datetime import datetime - stop = self.stop_loss_value - p_stop = (-stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - ### backslap debug wrap debug_2loops = True # only loop twice, for faster debug debug_timing = False # print timing for each step debug = True # print values, to check accuracy + + # Read Stop Loss Values and Stake + stop = self.stop_loss_value + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + stake = self.config.get('stake_amount') + + # Set fees + # TODO grab these from the environment, do not hard set + # Fees + open_fee = 0.05 + close_fee = 0.05 + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: from pandas import set_option set_option('display.max_rows', 5000) @@ -385,9 +406,10 @@ class Backtesting(object): while t_exit_ind < np_buy_arr_len: loop = loop + 1 if debug or debug_timing: - print("----------------------------------------------------------- Loop", loop, pair) + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) if debug_2loops: - if loop == 4: + if loop == 2: + print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") break ''' Dev phases @@ -541,10 +563,10 @@ class Backtesting(object): print("=================== STOP ", pair) print("Numpy Array STOP Index is:", np_t_stop_ind) print("DataFrame STOP Index is:", np_t_stop_ind + 1, "displaying DF \n") - print("First Stop Index after Trade open in candle", np_t_stop_ind + 1, ": \n", + print("First Stop after Trade open in candle", t_open_ind + 1, "is ", np_t_stop_ind + 1,": \n", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sto]), "is less than", str.format('{0:.17f}', np_t_stop_pri)) - print("Tokens will be sold at rate:", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) + print("If stop is first exit match sell rate is :", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) print("HINT, STOPs should close in-candle, i.e", np_t_stop_ind + 1, ": As live STOPs are not linked to O-C times") @@ -610,7 +632,7 @@ class Backtesting(object): # Loop control - catch no closed trades. if debug: - print("---------------------------------------- end of loop", loop - 1, + print("---------------------------------------- end of loop", loop, " Dataframe Exit Index is: ", t_exit_ind) print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) @@ -641,23 +663,26 @@ class Backtesting(object): close_index: int = t_exit_ind + 2 else: close_index: int = t_exit_ind + 1 - # Munge the date / delta - start_date = bslap.iloc[t_open_ind + 1]['date'] - end_date = bslap.iloc[close_index]['date'] + # Munge the date / delta (bt already date formats...just subract) + trade_start = bslap.iloc[t_open_ind + 1]['date'] + trade_end = bslap.iloc[close_index]['date'] # def __datetime(date_str): # return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') - - trade_start = start_date - trade_end = end_date trade_mins = (trade_end - trade_start).total_seconds() / 60 + # Profit ABS. + # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) + sumpaid: float = (np_trade_enter_price * stake) * open_fee + sumrecieved: float = (np_trade_exit_price * stake) * close_fee + profit_abs: float = sumrecieved - sumpaid + # build trade dictionary bslap_result["pair"] = pair - bslap_result["profit_percent"] = ( np_trade_exit_price - np_trade_enter_price)/np_trade_enter_price - bslap_result["profit_abs"] = "" - bslap_result["open_time"] = start_date - bslap_result["close_time"] = end_date + bslap_result["profit_percent"] = (np_trade_exit_price - np_trade_enter_price) / np_trade_enter_price + bslap_result["profit_abs"] = str.format('{0:.10f}', profit_abs) + bslap_result["open_time"] = trade_start + bslap_result["close_time"] = trade_end bslap_result["open_index"] = t_open_ind + 1 bslap_result["close_index"] = close_index bslap_result["trade_duration"] = trade_mins From 4e68362d46d6ca7c3aa96ac8a5fa090d18ca3c4f Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 15 Jul 2018 10:33:00 +0000 Subject: [PATCH 004/174] Works with reporting output Bugs Calculating % prof ok, but abs wrong BAT/BTC DF is very broken all OHLC are the same - but exposes a buy after stop on last row "oddness" to be investigated / handled --- freqtrade/optimize/backtesting.py | 61 +++++++++++++++++-------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3cfb588b9..9698ef471 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -190,7 +190,7 @@ class Backtesting(object): return btr return None - + @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -309,9 +309,9 @@ class Backtesting(object): from datetime import datetime ### backslap debug wrap - debug_2loops = True # only loop twice, for faster debug + debug_2loops = False # only loop twice, for faster debug debug_timing = False # print timing for each step - debug = True # print values, to check accuracy + debug = False # print values, to check accuracy # Read Stop Loss Values and Stake stop = self.stop_loss_value @@ -354,8 +354,10 @@ class Backtesting(object): buy - open - close - sell - high - low - np_stop_pri """ bto = buys_triggered_on = "close" - sto = stops_triggered_on = "low" ## Should be low, FT uses close - sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close + # sto = stops_triggered_on = "low" ## Should be low, FT uses close + # sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close + sto = stops_triggered_on = "close" ## Should be low, FT uses close + sco = stops_calculated_on = "close" ## should use np_stop_pri, FT uses close ''' Numpy arrays are used for 100x speed up We requires setting Int values for @@ -371,8 +373,10 @@ class Backtesting(object): np_stop: int = 6 np_bto: int = np_close # buys_triggered_on - should be close np_bco: int = np_open # buys calculated on - open of the next candle. - np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close - np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close + #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close + np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close # ### End Config @@ -408,7 +412,7 @@ class Backtesting(object): if debug or debug_timing: print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) if debug_2loops: - if loop == 2: + if loop == 3: print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") break ''' @@ -673,23 +677,25 @@ class Backtesting(object): # Profit ABS. # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) - sumpaid: float = (np_trade_enter_price * stake) * open_fee - sumrecieved: float = (np_trade_exit_price * stake) * close_fee - profit_abs: float = sumrecieved - sumpaid + sumpaid: float = (np_trade_enter_price * stake) + sumpaid_fee: float = sumpaid * open_fee + sumrecieved: float = (np_trade_exit_price * stake) + sumrecieved_fee: float = sumrecieved * close_fee + profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee # build trade dictionary bslap_result["pair"] = pair bslap_result["profit_percent"] = (np_trade_exit_price - np_trade_enter_price) / np_trade_enter_price - bslap_result["profit_abs"] = str.format('{0:.10f}', profit_abs) + bslap_result["profit_abs"] = round(profit_abs, 15) bslap_result["open_time"] = trade_start bslap_result["close_time"] = trade_end - bslap_result["open_index"] = t_open_ind + 1 + bslap_result["open_index"] = t_open_ind + 2 # +1 between np and df, +1 as we buy on next. bslap_result["close_index"] = close_index bslap_result["trade_duration"] = trade_mins bslap_result["open_at_end"] = False - bslap_result["open_rate"] = str.format('{0:.10f}', np_trade_enter_price) - bslap_result["close_rate"] = str.format('{0:.10f}', np_trade_exit_price) - bslap_result["exit_type"] = t_exit_type + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + #bslap_result["exit_type"] = t_exit_type # Add trade dictionary to list bslap_pair_results.append(bslap_result) if debug: @@ -704,7 +710,7 @@ class Backtesting(object): if debug_timing: t_t = f(st) - print("8", str.format('{0:.17f}', t_t)) + print("8+trade", str.format('{0:.17f}', t_t)) # Send back List of trade dicts return bslap_pair_results @@ -785,16 +791,17 @@ class Backtesting(object): ) ) - logger.info( - '\n=============================================== ' - 'LEFT OPEN TRADES REPORT' - ' ===============================================\n' - '%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) + ## TODO. Catch open trades for this report. + # logger.info( + # '\n=============================================== ' + # 'LEFT OPEN TRADES REPORT' + # ' ===============================================\n' + # '%s', + # self._generate_text_table( + # data, + # results.loc[results.open_at_end] + # ) + # ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From a8b62a21ccce9bc7e79687d86167e9559b9b8962 Mon Sep 17 00:00:00 2001 From: creslinux Date: Sun, 15 Jul 2018 17:03:47 +0000 Subject: [PATCH 005/174] hmm --- freqtrade/optimize/backtesting.py | 153 +++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 33 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 9698ef471..045b61a39 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -190,7 +190,6 @@ class Backtesting(object): return btr return None - @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -233,7 +232,13 @@ class Backtesting(object): last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + + print(bslap_results_df.dtypes) + return bslap_results_df ########################### Original BT loop @@ -283,6 +288,69 @@ class Backtesting(object): # return DataFrame.from_records(trades, columns=BacktestResult._fields) ######################## Original BT loop end + def vector_fill_results_table(self, bslap_results_df: DataFrame): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + debug = True + + # stake and fees + stake = self.config.get('stake_amount') + # TODO grab these from the environment, do not hard set + open_fee = 0.05 + close_fee = 0.05 + + if debug: + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + # Align with BT + bslap_results_df['open_time'] = pd.to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = pd.to_datetime(bslap_results_df['close_time']) + + # Populate duration + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + if debug: + print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) + + ## Spends, Takes, Profit, Absolute Profit + # Buy Price + bslap_results_df['buy_sum'] = stake * bslap_results_df['open_rate'] + bslap_results_df['buy_fee'] = bslap_results_df['buy_sum'] * open_fee + bslap_results_df['buy_spend'] = bslap_results_df['buy_sum'] + bslap_results_df['buy_fee'] + # Sell price + bslap_results_df['sell_sum'] = stake * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = bslap_results_df['sell_take'] / bslap_results_df['buy_spend'] - 1 + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + if debug: + print("\n") + print(bslap_results_df[['buy_spend', 'sell_take', 'profit_percent', 'profit_abs']]) + + return bslap_results_df + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): import utils_find_1st as utf1st """ @@ -301,6 +369,7 @@ class Backtesting(object): t_open_ind = t_open_ind + t_exit_ind # Align numpy index return t_open_ind + @profile def backslap_pair(self, ticker_data, pair): import pandas as pd import numpy as np @@ -316,19 +385,10 @@ class Backtesting(object): # Read Stop Loss Values and Stake stop = self.stop_loss_value p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - stake = self.config.get('stake_amount') - - # Set fees - # TODO grab these from the environment, do not hard set - # Fees - open_fee = 0.05 - close_fee = 0.05 if debug: print("Stop is ", stop, "value from stragey file") print("p_stop is", p_stop, "value used to multiply to entry price") - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) if debug: from pandas import set_option @@ -395,6 +455,12 @@ class Backtesting(object): # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + loop: int = 0 # how many time around the loop t_exit_ind = 0 # Start loop from first index t_exit_last = 0 # To test for exit @@ -657,9 +723,13 @@ class Backtesting(object): break else: """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + if debug_timing: + t_t = f(st) + print("8a-IfEls", str.format('{0:.17f}', t_t)) + st = s() # Index will change if incandle stop or look back as close Open and Sell if t_exit_type == 'stop': close_index: int = t_exit_ind + 1 @@ -668,39 +738,56 @@ class Backtesting(object): else: close_index: int = t_exit_ind + 1 - # Munge the date / delta (bt already date formats...just subract) - trade_start = bslap.iloc[t_open_ind + 1]['date'] - trade_end = bslap.iloc[close_index]['date'] - # def __datetime(date_str): - # return datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S+00:00') - trade_mins = (trade_end - trade_start).total_seconds() / 60 + if debug_timing: + t_t = f(st) + print("8b-Index", str.format('{0:.17f}', t_t)) + st = s() - # Profit ABS. - # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) - sumpaid: float = (np_trade_enter_price * stake) - sumpaid_fee: float = sumpaid * open_fee - sumrecieved: float = (np_trade_exit_price * stake) - sumrecieved_fee: float = sumrecieved * close_fee - profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee + # # Profit ABS. + # # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) + # sumpaid: float = (np_trade_enter_price * stake) + # sumpaid_fee: float = sumpaid * open_fee + # sumrecieved: float = (np_trade_exit_price * stake) + # sumrecieved_fee: float = sumrecieved * close_fee + # profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee + + if debug_timing: + t_t = f(st) + print("8d---ABS", str.format('{0:.17f}', t_t)) + st = s() + + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(numer of trades faster) to calc all in a single vector than 1 trade at a time - # build trade dictionary bslap_result["pair"] = pair - bslap_result["profit_percent"] = (np_trade_exit_price - np_trade_enter_price) / np_trade_enter_price - bslap_result["profit_abs"] = round(profit_abs, 15) - bslap_result["open_time"] = trade_start - bslap_result["close_time"] = trade_end + bslap_result["profit_percent"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower bslap_result["open_index"] = t_open_ind + 2 # +1 between np and df, +1 as we buy on next. bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = trade_mins + bslap_result["trade_duration"] = "1" # To be 1 vector calc across trades when loop complete bslap_result["open_at_end"] = False bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) - #bslap_result["exit_type"] = t_exit_type + bslap_result["exit_type"] = t_exit_type + + if debug_timing: + t_t = f(st) + print("8e-trade", str.format('{0:.17f}', t_t)) + st = s() # Add trade dictionary to list bslap_pair_results.append(bslap_result) + if debug: print(bslap_pair_results) + if debug_timing: + t_t = f(st) + print("8f--list", str.format('{0:.17f}', t_t)) + st = s() + """ Loop back to start. t_exit_last becomes where loop will seek to open new trades from. From 7174f27eb85296ef1886e3ebc0f71c5c3e7ec0d0 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 12:01:02 +0000 Subject: [PATCH 006/174] Rewrite to used algned numpy/dataframes updated logic added vector fill for abs/profit/duration in single hit on results. --- freqtrade/optimize/backtesting.py | 663 +++++++++++++++++------------- 1 file changed, 370 insertions(+), 293 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 045b61a39..2629ed5fa 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -10,7 +10,7 @@ from datetime import datetime from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow -from pandas import DataFrame +from pandas import DataFrame, to_datetime from tabulate import tabulate import freqtrade.optimize as optimize @@ -23,6 +23,7 @@ from freqtrade.misc import file_dump_json from freqtrade.persistence import Trade from profilehooks import profile from collections import OrderedDict +import timeit logger = logging.getLogger(__name__) @@ -190,6 +191,7 @@ class Backtesting(object): return btr return None + @profile def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -234,10 +236,13 @@ class Backtesting(object): # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) - bslap_results_df = self.vector_fill_results_table(bslap_results_df) + bslap_results_df = DataFrame(bslap_results) + bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + ### don't use this, itll drop exit type field + # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) - print(bslap_results_df.dtypes) + bslap_results_df = self.vector_fill_results_table(bslap_results_df) return bslap_results_df @@ -303,14 +308,13 @@ class Backtesting(object): :return: bslap_results Dataframe """ import pandas as pd - debug = True + debug = False # stake and fees - stake = self.config.get('stake_amount') - # TODO grab these from the environment, do not hard set - open_fee = 0.05 - close_fee = 0.05 - + stake = 0.015 + # 0.05% is 0.00,05 + open_fee = 0.0000 + close_fee = 0.0000 if debug: print("Stake is,", stake, "the sum of currency to spend per trade") print("The open fee is", open_fee, "The close fee is", close_fee) @@ -322,10 +326,6 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) - # Align with BT - bslap_results_df['open_time'] = pd.to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = pd.to_datetime(bslap_results_df['close_time']) - # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] if debug: @@ -347,29 +347,46 @@ class Backtesting(object): if debug: print("\n") - print(bslap_results_df[['buy_spend', 'sell_take', 'profit_percent', 'profit_abs']]) + print(bslap_results_df[ + ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_take', 'profit_percent', 'profit_abs', + 'exit_type']]) return bslap_results_df def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): import utils_find_1st as utf1st """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - """ + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + """ + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + def f(st): + return (timeit.default_timer() - st) + + st = s() t_open_ind: int - # Create a view on our buy index starting after last trade exit - # Search for next buy + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ np_buy_arr_v = np_buy_arr[t_exit_ind:] t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + return t_open_ind - @profile def backslap_pair(self, ticker_data, pair): import pandas as pd import numpy as np @@ -397,27 +414,12 @@ class Backtesting(object): pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) - def s(): st = timeit.default_timer() return st - def f(st): return (timeit.default_timer() - st) #### backslap config - """ - A couple legacy Pandas vars still used for pretty debug output. - If have debug enabled the code uses these fields for dataframe output - - Ensure bto, sto, sco are aligned with Numpy values next - to align debug and actual. Options are: - buy - open - close - sell - high - low - np_stop_pri - """ - bto = buys_triggered_on = "close" - # sto = stops_triggered_on = "low" ## Should be low, FT uses close - # sco = stops_calculated_on = "np_stop_pri" ## should use np_stop_pri, FT uses close - sto = stops_triggered_on = "close" ## Should be low, FT uses close - sco = stops_calculated_on = "close" ## should use np_stop_pri, FT uses close ''' Numpy arrays are used for 100x speed up We requires setting Int values for @@ -441,7 +443,6 @@ class Backtesting(object): ### End Config pair: str = pair - loop: int = 1 #ticker_data: DataFrame = ticker_dfs[t_file] bslap: DataFrame = ticker_data @@ -482,223 +483,342 @@ class Backtesting(object): print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") break ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first intance as trade exit + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - ''' - Finds index for first buy = 1 flag, use .values numpy array for speed - - Create a slice, from first buy index onwards. - Slice will be used to find exit conditions after trade open - ''' if debug_timing: st = s() - + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind) + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") if debug_timing: t_t = f(st) print("0-numpy", str.format('{0:.17f}', t_t)) st = s() - ''' - Calculate np_t_stop_pri (Trade Stop Price) based on the buy price + if t_open_ind != -1: - As stop in based on buy price we are interested in buy - - Buys are Triggered On np_bto, typically the CLOSE of candle - - Buys are Calculated On np_bco, default is OPEN of the next candle. - as we only see the CLOSE after it has happened. - The assumption is we have bought at first available price, the OPEN - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + """ + 1 - Create view to search within for our open trade + + The view is our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provided: np_t_open_v View of array after trade. + """ + np_t_open_v = np_bslap[t_open_ind:] - if debug_timing: - t_t = f(st) - print("1-numpy", str.format('{0:.17f}', t_t)) - st = s() + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() - """ - 1)Create a View from our open trade forward - - The view is our search space for the next Stop or Sell - We use a numpy view: - Using a numpy for speed on views, 1,000 faster than pandas - Pandas cannot assure it will always return a view, copies are - 3 orders of magnitude slower - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - """ - np_t_open_v = np_bslap[t_open_ind:] - - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - Find first stop index after Trade Open: - - First index in np_t_open_v (numpy view of bslap dataframe) - Using a numpy view a orders of magnitude faster - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) \ - + t_open_ind - - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - Find first sell index after trade open - - First index in t_open_slice where ['sell'] = 1 - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) \ - + t_open_ind - - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - Determine which was hit first stop or sell, use as exit - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - ''' - if np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = np_t_stop_ind # Set Exit row index - t_exit_type = 'stop' # Set Exit type (sell|stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - else: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = np_t_sell_ind # Set Exit row index - t_exit_type = 'sell' # Set Exit type (sell|stop) - np_t_exit_pri = np_open # The price field our SELL exit will use - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the currenct stop price based on buy price_ - # Don't care about performance in debug - # (add an extra column if printing as df has date in col1 not in npy) - bslap['np_stop_pri'] = np_t_stop_pri + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - # Buy - print("=================== BUY ", pair) - print("Numpy Array BUY Index is:", t_open_ind) - print("DataFrame BUY Index is:", t_open_ind + 1, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 2, "\n") - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - print(bslap.iloc[t_open_ind + 1]['date']) + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() - # Stop - Stops trigger price sto, and price received sco. (Stop Trigger|Calculated On) - print("=================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind) - print("DataFrame STOP Index is:", np_t_stop_ind + 1, "displaying DF \n") - print("First Stop after Trade open in candle", t_open_ind + 1, "is ", np_t_stop_ind + 1,": \n", - str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sto]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - print("If stop is first exit match sell rate is :", str.format('{0:.17f}', bslap.iloc[np_t_stop_ind][sco])) - print("HINT, STOPs should close in-candle, i.e", np_t_stop_ind + 1, - ": As live STOPs are not linked to O-C times") + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v Numpy view of ticker_data after trade open + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) - st_is = np_t_stop_ind - 1 # Print stop index start, line before - st_if = np_t_stop_ind + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - # Sell - print("=================== SELL ", pair) - print("Numpy Array SELL Index is:", np_t_sell_ind) - print("DataFrame SELL Index is:", np_t_sell_ind + 1, "displaying DF \n") - print("First Sell Index after Trade open in in candle", np_t_sell_ind + 1) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - np_t_sell_ind + 2, "\n") - sl_is = np_t_sell_ind - 1 # Print sell index start, line before - sl_if = np_t_sell_ind + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") + print("If -1 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() - # Chosen Exit (stop or sell) - print("=================== EXIT ", pair) - print("Exit type is :", t_exit_type) - # print((bslap.iloc[t_exit_ind], "\n")) - print("trade exit price field is", np_t_exit_pri, "\n") + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data - ''' - Trade entry is always the next candles "open" price - We trigger on close, so cannot see that till after - its closed. + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() - The exception to this is a STOP which is calculated in candle - ''' - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering in 1, candle we bought at OPEN is valid. + + Buys and sells are triggered at candle close + Both with action their postions at the open of the next candle Index + 1 + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logig test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Im setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Im setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind == -1 else np_t_stop_ind - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == 'stop': - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = 'stop' # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_type = 'sell' # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = "No Exit" + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == 'stop': + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == 'sell': np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == 'sell': - np_trade_exit_price = np_bslap[t_exit_ind + 1, np_t_exit_pri] - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() + # Catch no exit found + if t_exit_type == "No Exit": + np_trade_exit_price = 0 - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enterprice is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) # Loop control - catch no closed trades. if debug: @@ -706,87 +826,47 @@ class Backtesting(object): " Dataframe Exit Index is: ", t_exit_ind) print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - if t_exit_last >= t_exit_ind: + if t_exit_last >= t_exit_ind or t_exit_last == -1: """ + Break loop and go on to next pair. + When last trade exit equals index of last exit, there is no opportunity to close any more trades. - - Break loop and go on to next pair. - - TODO - add handing here to record none closed open trades """ - + # TODO :add handing here to record none closed open trades if debug: print(bslap_pair_results) - break else: """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - if debug_timing: - t_t = f(st) - print("8a-IfEls", str.format('{0:.17f}', t_t)) - st = s() - # Index will change if incandle stop or look back as close Open and Sell - if t_exit_type == 'stop': - close_index: int = t_exit_ind + 1 - elif t_exit_type == 'sell': - close_index: int = t_exit_ind + 2 - else: - close_index: int = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8b-Index", str.format('{0:.17f}', t_t)) - st = s() - - # # Profit ABS. - # # sumrecieved((rate * numTokens) * fee) - sumpaid ((rate * numTokens) * fee) - # sumpaid: float = (np_trade_enter_price * stake) - # sumpaid_fee: float = sumpaid * open_fee - # sumrecieved: float = (np_trade_exit_price * stake) - # sumrecieved_fee: float = sumrecieved * close_fee - # profit_abs: float = sumrecieved - sumpaid - sumpaid_fee - sumrecieved_fee - - if debug_timing: - t_t = f(st) - print("8d---ABS", str.format('{0:.17f}', t_t)) - st = s() - + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ # Build trade dictionary ## In general if a field can be calculated later from other fields leave blank here - ## Its X(numer of trades faster) to calc all in a single vector than 1 trade at a time + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result bslap_result["pair"] = pair - bslap_result["profit_percent"] = "1" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 2 # +1 between np and df, +1 as we buy on next. + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "1" # To be 1 vector calc across trades when loop complete + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete bslap_result["open_at_end"] = False bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) bslap_result["exit_type"] = t_exit_type - - if debug_timing: - t_t = f(st) - print("8e-trade", str.format('{0:.17f}', t_t)) - st = s() - # Add trade dictionary to list + # append the dict to the list and print list bslap_pair_results.append(bslap_result) if debug: - print(bslap_pair_results) - - if debug_timing: - t_t = f(st) - print("8f--list", str.format('{0:.17f}', t_t)) - st = s() + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) """ Loop back to start. t_exit_last becomes where loop @@ -802,9 +882,6 @@ class Backtesting(object): # Send back List of trade dicts return bslap_pair_results - - - def start(self) -> None: """ Run a backtesting end-to-end @@ -868,9 +945,9 @@ class Backtesting(object): self._store_backtest_result(self.config.get('exportfilename'), results) logger.info( - '\n================================================= ' - 'BACKTESTING REPORT' - ' ==================================================\n' + '\n====================================================== ' + 'BackSLAP REPORT' + ' =======================================================\n' '%s', self._generate_text_table( data, From eed29a6b8af3a1746c16cb142f6e418072ce6042 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 13:16:18 +0000 Subject: [PATCH 007/174] update --- freqtrade/optimize/backtesting.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 2629ed5fa..d17b7b246 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -24,6 +24,7 @@ from freqtrade.persistence import Trade from profilehooks import profile from collections import OrderedDict import timeit +from time import sleep logger = logging.getLogger(__name__) @@ -191,7 +192,14 @@ class Backtesting(object): return btr return None - @profile + def s(self): + st = timeit.default_timer() + return st + + def f(self, st): + return (timeit.default_timer() - st) + + def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -207,21 +215,31 @@ class Backtesting(object): realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ + debug_timing = False + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', False) trades = [] trade_count_lock: Dict = {} + ########################### Call out BSlap instead of using FT bslap_results: list = [] - last_bslap_results: list = [] + for pair, pair_data in processed.items(): + if debug_timing: # Start timer + fl = self.s() + ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() ticker_data.drop(ticker_data.head(1).index, inplace=True) + if debug_timing: # print time taken + flt = self.f(fl) + print("populate_buy_trend:", pair, round(flt, 10)) + st = self.st() # #dump same DFs to disk for offline testing in scratch # f_pair:str = pair @@ -234,11 +252,18 @@ class Backtesting(object): last_bslap_results = bslap_results bslap_results = last_bslap_results + bslap_pair_results + if debug_timing: # print time taken + tt = self.f(st) + print("Time to Back Slap :", pair, round(tt,10)) + print("-----------------------") + + # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. bslap_results_df = DataFrame(bslap_results) bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + ### don't use this, itll drop exit type field # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) From fb0edd71ff166a353613a1f14de42a27163ea92b Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 14:16:35 +0000 Subject: [PATCH 008/174] in tech test --- freqtrade/optimize/backtesting.py | 276 ++++++++++++++++++------------ 1 file changed, 171 insertions(+), 105 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d17b7b246..bbb52d167 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -73,6 +73,35 @@ class Backtesting(object): self.stop_loss_value = self.analyze.strategy.stoploss + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + # self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.use_backslap = True # Enable backslap + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair + + @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: """ @@ -215,7 +244,7 @@ class Backtesting(object): realistic: do we try to simulate realistic trades? (default: True) :return: DataFrame """ - debug_timing = False + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] processed = args['processed'] @@ -224,99 +253,114 @@ class Backtesting(object): trades = [] trade_count_lock: Dict = {} - ########################### Call out BSlap instead of using FT - bslap_results: list = [] + use_backslap = self.use_backslap + debug_timing = self.debug_timing_main_loop + if use_backslap: # Use Back Slap code + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in processed.items(): + if debug_timing: # Start timer + fl = self.s() + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + if debug_timing: # print time taken + flt = self.f(fl) + #print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + #call bslap - results are a list of dicts + bslap_pair_results = self.backslap_pair(ticker_data, pair) + last_bslap_results = bslap_results + bslap_results = last_bslap_results + bslap_pair_results + + if debug_timing: # print time taken + tt = self.f(st) + print("Time to BackSlap :", pair, round(tt,10)) + print("-----------------------") - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - ticker_data = self.populate_sell_trend( + ### don't use this, itll drop exit type field + # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + + return bslap_results_df + + else: # use Original Back test code + ########################## Original BT loop + + for pair, pair_data in processed.items(): + if debug_timing: # Start timer + fl = self.s() + + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run + + ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() - ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken - flt = self.f(fl) - print("populate_buy_trend:", pair, round(flt, 10)) - st = self.st() + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + ticker_data.drop(ticker_data.head(1).index, inplace=True) - #call bslap - results are a list of dicts - bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results - bslap_results = last_bslap_results + bslap_pair_results + if debug_timing: # print time taken + flt = self.f(fl) + #print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() - if debug_timing: # print time taken - tt = self.f(st) - print("Time to Back Slap :", pair, round(tt,10)) - print("-----------------------") + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) + ticker = [x for x in ticker_data.itertuples()] + + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off + + if realistic: + if lock_pair_until is not None and row.date <= lock_pair_until: + continue + if max_open_trades > 0: + # Check if max_open_trades has already been reached for the given date + if not trade_count_lock.get(row.date, 0) < max_open_trades: + continue + + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + if trade_entry: + lock_pair_until = trade_entry.close_time + trades.append(trade_entry) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date - ### don't use this, itll drop exit type field - # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + if debug_timing: # print time taken + tt = self.f(st) + print("Time to BackTest :", pair, round(tt, 10)) + print("-----------------------") - bslap_results_df = self.vector_fill_results_table(bslap_results_df) - - return bslap_results_df - - ########################### Original BT loop - # for pair, pair_data in processed.items(): - # pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - # - # ticker_data = self.populate_sell_trend( - # self.populate_buy_trend(pair_data))[headers].copy() - # - # # to avoid using data from future, we buy/sell with signal from previous candle - # ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - # ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - # - # ticker_data.drop(ticker_data.head(1).index, inplace=True) - # - # # Convert from Pandas to list for performance reasons - # # (Looping Pandas is slow.) - # ticker = [x for x in ticker_data.itertuples()] - # - # lock_pair_until = None - # for index, row in enumerate(ticker): - # if row.buy == 0 or row.sell == 1: - # continue # skip rows where no buy signal or that would immediately sell off - # - # if realistic: - # if lock_pair_until is not None and row.date <= lock_pair_until: - # continue - # if max_open_trades > 0: - # # Check if max_open_trades has already been reached for the given date - # if not trade_count_lock.get(row.date, 0) < max_open_trades: - # continue - # - # trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - # - # trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - # trade_count_lock, args) - # - # - # if trade_entry: - # lock_pair_until = trade_entry.close_time - # trades.append(trade_entry) - # else: - # # Set lock_pair_until to end of testing period if trade could not be closed - # # This happens only if the buy-signal was with the last candle - # lock_pair_until = ticker_data.iloc[-1].date - # - # return DataFrame.from_records(trades, columns=BacktestResult._fields) - ######################## Original BT loop end + return DataFrame.from_records(trades, columns=BacktestResult._fields) + ####################### Original BT loop end def vector_fill_results_table(self, bslap_results_df: DataFrame): """ @@ -333,13 +377,18 @@ class Backtesting(object): :return: bslap_results Dataframe """ import pandas as pd - debug = False + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + #fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 - # stake and fees - stake = 0.015 - # 0.05% is 0.00,05 - open_fee = 0.0000 - close_fee = 0.0000 if debug: print("Stake is,", stake, "the sum of currency to spend per trade") print("The open fee is", open_fee, "The close fee is", close_fee) @@ -420,9 +469,12 @@ class Backtesting(object): from datetime import datetime ### backslap debug wrap - debug_2loops = False # only loop twice, for faster debug - debug_timing = False # print timing for each step - debug = False # print values, to check accuracy + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy # Read Stop Loss Values and Stake stop = self.stop_loss_value @@ -451,20 +503,34 @@ class Backtesting(object): buy stop triggers and stop calculated on # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 ''' - np_buy: int = 0 - np_open: int = 1 - np_close: int = 2 - np_sell: int = 3 - np_high: int = 4 - np_low: int = 5 - np_stop: int = 6 - np_bto: int = np_close # buys_triggered_on - should be close - np_bco: int = np_open # buys calculated on - open of the next candle. - #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close - #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close - np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close - np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close - # + # np_buy: int = 0 + # np_open: int = 1 + # np_close: int = 2 + # np_sell: int = 3 + # np_high: int = 4 + # np_low: int = 5 + # np_stop: int = 6 + # np_bto: int = np_close # buys_triggered_on - should be close + # np_bco: int = np_open # buys calculated on - open of the next candle. + # #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close + # #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close + # np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close + # np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + ### End Config pair: str = pair From 5aaf454f120e75f4e228ece32ba5a396aa3446b4 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 15:48:06 +0000 Subject: [PATCH 009/174] GAS trades verified from candle data to excel by hand All pass 3 sells 1 stop loss --- freqtrade/optimize/backtesting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index bbb52d167..f254bc8a4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,10 +95,10 @@ class Backtesting(object): self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = True # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair @@ -255,6 +255,7 @@ class Backtesting(object): use_backslap = self.use_backslap debug_timing = self.debug_timing_main_loop + if use_backslap: # Use Back Slap code ########################### Call out BSlap Loop instead of Original BT code bslap_results: list = [] @@ -395,7 +396,7 @@ class Backtesting(object): if debug: from pandas import set_option set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) + set_option('display.max_columns', 10) pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) @@ -422,8 +423,7 @@ class Backtesting(object): if debug: print("\n") print(bslap_results_df[ - ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_take', 'profit_percent', 'profit_abs', - 'exit_type']]) + ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) return bslap_results_df @@ -708,8 +708,8 @@ class Backtesting(object): if debug: print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) if debug_timing: t_t = f(st) print("4-numpy", str.format('{0:.17f}', t_t)) From 4a39a754f4811aa4531e8ce0a30019d391a72979 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 15:57:15 +0000 Subject: [PATCH 010/174] Fixed: self.use_backslap = Bool on line97 If self.use_backslap = True Backslap executes If self.use_backslap = False Original Backtest Code executes --- freqtrade/optimize/backtesting.py | 57 ++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index f254bc8a4..191616c6c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -94,12 +94,12 @@ class Backtesting(object): self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap + self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging self.debug_vector = True # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap @staticmethod @@ -245,18 +245,18 @@ class Backtesting(object): :return: DataFrame """ - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - trades = [] - trade_count_lock: Dict = {} - use_backslap = self.use_backslap debug_timing = self.debug_timing_main_loop if use_backslap: # Use Back Slap code + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + trades = [] + trade_count_lock: Dict = {} + ########################### Call out BSlap Loop instead of Original BT code bslap_results: list = [] for pair, pair_data in processed.items(): @@ -304,6 +304,13 @@ class Backtesting(object): else: # use Original Back test code ########################## Original BT loop + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + trades = [] + trade_count_lock: Dict = {} + for pair, pair_data in processed.items(): if debug_timing: # Start timer fl = self.s() @@ -1035,16 +1042,28 @@ class Backtesting(object): if self.config.get('export', False): self._store_backtest_result(self.config.get('exportfilename'), results) - logger.info( - '\n====================================================== ' - 'BackSLAP REPORT' - ' =======================================================\n' - '%s', - self._generate_text_table( - data, - results + if self.use_backslap: + logger.info( + '\n====================================================== ' + 'BackSLAP REPORT' + ' =======================================================\n' + '%s', + self._generate_text_table( + data, + results + ) + ) + else: + logger.info( + '\n================================================= ' + 'BACKTEST REPORT' + ' ==================================================\n' + '%s', + self._generate_text_table( + data, + results + ) ) - ) ## TODO. Catch open trades for this report. # logger.info( From 0f3339f74f30dec2910fc4c36e401bb390fc5053 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:09:42 +0000 Subject: [PATCH 011/174] use ujson to load ticker files 30% faster from disk. --- freqtrade/optimize/__init__.py | 24 ++++++++++++++++++++---- requirements.txt | 7 +++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index e806ff2b8..90dda79e2 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -10,9 +10,16 @@ import arrow from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange +import importlib +ujson_found = importlib.util.find_spec("ujson") +if ujson_found is not None: + import ujson logger = logging.getLogger(__name__) +if ujson_found is not None: + logger.debug('Loaded UltraJson ujson in optimize.py') + def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: if not tickerlist: @@ -63,11 +70,17 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - pairdata = json.load(tickerdata) + if ujson_found is not None: + pairdata = ujson.load(tickerdata, precise_float=True) + else: + pairdata = json.load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - pairdata = json.load(tickerdata) + if ujson_found is not None: + pairdata = ujson.load(tickerdata, precise_float=True) + else: + pairdata = json.load(tickerdata) else: return None @@ -163,7 +176,10 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + if ujson_found is not None: + data = ujson.load(file, precise_float=True) + else: + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: @@ -226,4 +242,4 @@ def download_backtesting_testdata(datadir: str, logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) + misc.file_dump_json(filename, data) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3fb91888c..99b471b1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,10 @@ scikit-optimize==0.5.2 # Required for plotting data #plotly==3.0.0 + +# find first, C search in arrays +py_find_1st==1.1.1 + +#Load ticker files 30% faster +ujson==1.35 + From 059aceb582e04c2a64c5370ffb5381e7c1956385 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:12:33 +0000 Subject: [PATCH 012/174] Disabled full debug on in last commit Switched Stops to trigger on Low Switched Stops to pay stop-rate not close. --- freqtrade/optimize/backtesting.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 191616c6c..3aabfc2ad 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,17 +89,17 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - # self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = True # print overall timing per pair - works in Backtest and Backslap @staticmethod From 885a653439bd49e8d5b25d75422f2559dfc225aa Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:18:54 +0000 Subject: [PATCH 013/174] Disabled full debug on in last commit Switched Stops to trigger on Low Switched Stops to pay stop-rate not close. --- freqtrade/optimize/backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3aabfc2ad..774d4b954 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -98,7 +98,7 @@ class Backtesting(object): self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = True # Debug vector calcs self.debug_timing_main_loop = True # print overall timing per pair - works in Backtest and Backslap From 99d16e82c0a4109060abe8e5675b6231bfa2b710 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 16:30:11 +0000 Subject: [PATCH 014/174] disable time calcs output on vector displaying in debug. Excessive. --- freqtrade/optimize/backtesting.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 774d4b954..0fd21b14e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,10 +89,10 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended @@ -410,8 +410,8 @@ class Backtesting(object): # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - if debug: - print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) + # if debug: + # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) ## Spends, Takes, Profit, Absolute Profit # Buy Price From ec1960530b98a2a20d4c2dc6204c2d06765bebd2 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 17:06:06 +0000 Subject: [PATCH 015/174] Added Show trades option If true, prints trades ordered by date after summary. Useful for spotting trends. --- freqtrade/optimize/backtesting.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 0fd21b14e..85e42b91a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -94,12 +94,14 @@ class Backtesting(object): self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs - self.debug_timing_main_loop = True # print overall timing per pair - works in Backtest and Backslap + self.use_backslap = True # Enable backslap - if false Orginal code is executed. + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = True #prints trades in addition to summary report, also saves to backslap.txt @staticmethod @@ -1053,6 +1055,17 @@ class Backtesting(object): results ) ) + if self.backslap_show_trades: + TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', + 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + + def to_fwf(df, fname): + content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') + print(content) + open(fname, "w").write(content) + + DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") + else: logger.info( '\n================================================= ' From 8d5da4e6ad9ffc93b7ea0b9c6b49cbfa4406ce56 Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 17:48:11 +0000 Subject: [PATCH 016/174] changed defaults Seperated save trades and print trades options. --- freqtrade/optimize/backtesting.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 85e42b91a..43e05d70b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -101,7 +101,8 @@ class Backtesting(object): self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = True #prints trades in addition to summary report, also saves to backslap.txt + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt @staticmethod @@ -1055,17 +1056,25 @@ class Backtesting(object): results ) ) + #optional print trades if self.backslap_show_trades: TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - def to_fwf(df, fname): content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') print(content) - open(fname, "w").write(content) DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") + #optional save trades + if self.backslap_save_trades: + TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', + 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + def to_fwf(df, fname): + content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') + open(fname, "w").write(content) + DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") + else: logger.info( '\n================================================= ' From 3b0cb7bc333c980523e38abd5d70ba9c6a24113f Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 18:06:31 +0000 Subject: [PATCH 017/174] Added ujson and py_find_1st to setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index cd0574fa2..4ec541674 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,8 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', + 'ujson' + 'py_find_1st' ], include_package_data=True, zip_safe=False, From 357c8c0ba0ed902f1757844b0fffb94be45ecc3f Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 18:32:41 +0000 Subject: [PATCH 018/174] sensible defaults --- freqtrade/optimize/backtesting.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 43e05d70b..eb002a7ec 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,14 +89,14 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap + self.debug_timing = True # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap From a31391734798c2f0107b259c9e0b9cf54788c7bf Mon Sep 17 00:00:00 2001 From: creslinux Date: Mon, 16 Jul 2018 18:59:48 +0000 Subject: [PATCH 019/174] Handle a buy on the last candle We will never see this, as buy is on close which is the end of backtest e.g there is no next candle OPEN to buy at, or on --- freqtrade/optimize/backtesting.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index eb002a7ec..d6e4d1daf 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,10 +89,10 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended @@ -437,7 +437,7 @@ class Backtesting(object): return bslap_results_df - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int): + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int): import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 @@ -469,6 +469,9 @@ class Backtesting(object): if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index t_open_ind = t_open_ind + t_exit_ind # Align numpy index + if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + return t_open_ind def backslap_pair(self, ticker_data, pair): @@ -606,11 +609,12 @@ class Backtesting(object): Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. Provides: The next "buy" index after t_exit_ind If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind) + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len) if debug: print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) From baaf0a5b216d61462dbe55e3ac5b0168ece2ae16 Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 08:12:21 +0000 Subject: [PATCH 020/174] Handle when 0 trades are found in any pairs being tested. --- freqtrade/optimize/backtesting.py | 55 ++++++++++++++----------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6e4d1daf..3cdc9366b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,14 +89,14 @@ class Backtesting(object): self.np_stop: int = 6 self.np_bto: int = self.np_close # buys_triggered_on - should be close self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - #self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. + self.use_backslap = True # Enable backslap - if false Orginal code is executed. self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = True # Stages within Backslap + self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap @@ -294,13 +294,16 @@ class Backtesting(object): # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. bslap_results_df = DataFrame(bslap_results) - bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - ### don't use this, itll drop exit type field - # bslap_results_df = DataFrame(bslap_results, columns=BacktestResult._fields) + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + + bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) + else: + bslap_results_df = [] + bslap_results_df= DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - bslap_results_df = self.vector_fill_results_table(bslap_results_df) return bslap_results_df @@ -370,10 +373,12 @@ class Backtesting(object): print("Time to BackTest :", pair, round(tt, 10)) print("-----------------------") + print("trades") + return DataFrame.from_records(trades, columns=BacktestResult._fields) ####################### Original BT loop end - def vector_fill_results_table(self, bslap_results_df: DataFrame): + def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): """ The Results frame contains a number of columns that are calculable from othe columns. These are left blank till all rows are added, @@ -411,6 +416,7 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) + #Ony do Math on a dataframe that has an open; No result pairs contain the pair string only # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] # if debug: @@ -516,19 +522,7 @@ class Backtesting(object): buy stop triggers and stop calculated on # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 ''' - # np_buy: int = 0 - # np_open: int = 1 - # np_close: int = 2 - # np_sell: int = 3 - # np_high: int = 4 - # np_low: int = 5 - # np_stop: int = 6 - # np_bto: int = np_close # buys_triggered_on - should be close - # np_bco: int = np_open # buys calculated on - open of the next candle. - # #np_sto: int = np_low # stops_triggered_on - Should be low, FT uses close - # #np_sco: int = np_stop # stops_calculated_on - Should be stop, FT uses close - # np_sto: int = np_close # stops_triggered_on - Should be low, FT uses close - # np_sco: int = np_close # stops_calculated_on - Should be stop, FT uses close + ####### # Use vars set at top of backtest @@ -748,10 +742,10 @@ class Backtesting(object): open 6am 98 3 0 0 ----- ------ ------- ----- ----- -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering in 1, candle we bought at OPEN is valid. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. Buys and sells are triggered at candle close - Both with action their postions at the open of the next candle Index + 1 + Both will open their postions at the open of the next candle. i/e + 1 index Stop and buy Indexes are on the view. To map to the ticker dataframe the t_open_ind index should be summed. @@ -760,10 +754,10 @@ class Backtesting(object): t_exit_ind : Sell found in view t_open_ind : Where view was started on ticker_data - TODO: fix this frig for logig test,, case/switch/dictionary would be better... + TODO: fix this frig for logic test,, case/switch/dictionary would be better... more so when later testing many options, dynamic stop / roi etc - cludge - Im setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Im setting np_t_stop_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) ''' if debug: @@ -939,6 +933,7 @@ class Backtesting(object): opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades + if debug: print(bslap_pair_results) break From ed4bf32f2af3d6dfd1dc08aa6e7aa5de5e28031a Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 10:59:17 +0000 Subject: [PATCH 021/174] Fixed Stop closing in Index 0 when buy opening on Index 1 --- freqtrade/optimize/backtesting.py | 41 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3cdc9366b..c0259c8fc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,10 +95,10 @@ class Backtesting(object): #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = True # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap self.backslap_show_trades = False # prints trades in addition to summary report @@ -296,8 +296,10 @@ class Backtesting(object): bslap_results_df = DataFrame(bslap_results) if len(bslap_results_df) > 0: # Only post process a frame if it has a record - bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # if debug: + # print("open_time and close_time converted to datetime columns") bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) else: @@ -373,7 +375,6 @@ class Backtesting(object): print("Time to BackTest :", pair, round(tt, 10)) print("-----------------------") - print("trades") return DataFrame.from_records(trades, columns=BacktestResult._fields) ####################### Original BT loop end @@ -393,6 +394,7 @@ class Backtesting(object): :return: bslap_results Dataframe """ import pandas as pd + import numpy as np debug = self.debug_vector # stake and fees @@ -416,8 +418,6 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) - #Ony do Math on a dataframe that has an open; No result pairs contain the pair string only - # Populate duration bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] # if debug: # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) @@ -621,9 +621,9 @@ class Backtesting(object): if t_open_ind != -1: """ - 1 - Create view to search within for our open trade + 1 - Create views to search within for our open trade - The view is our search space for the next Stop or Sell + The views are our search space for the next Stop or Sell Numpy view is employed as: 1,000 faster than pandas searches Pandas cannot assure it will always return a view, it may make a slow copy. @@ -633,9 +633,12 @@ class Backtesting(object): Requires: np_bslap is our numpy array of the ticker DataFrame Requires: t_open_ind is the index row with the buy. - Provided: np_t_open_v View of array after trade. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) """ np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind +1:] if debug: print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) @@ -675,23 +678,27 @@ class Backtesting(object): where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - Requires: np_t_open_v Numpy view of ticker_data after trade open + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v[:, np_sto], + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], np_t_stop_pri, utf1st.cmp_smaller) + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind +1 + + if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind, + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind -1, ". STO is using field", np_sto, "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - print("If -1 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind + 1, np_t_open_v[np_t_stop_ind + 1]) + print("If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind -1 , np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind , np_t_open_v[np_t_stop_ind + 1]) if debug_timing: t_t = f(st) print("3-numpy", str.format('{0:.17f}', t_t)) @@ -765,7 +772,7 @@ class Backtesting(object): # cludge for logic test (-1) means it was not found, set crazy high to lose < test np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind == -1 else np_t_stop_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind # Stoploss trigger found before a sell =1 if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: From 8cea0517eb4e225553afbc8bc450be97a214ef11 Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 11:22:38 +0000 Subject: [PATCH 022/174] Added stop_stops stop_stops is an int value when number of stops in a pair reached the int the pair is stopped trading. This allows backtest to align with my pre_trade_mgt that does the same in dry and live operations --- freqtrade/optimize/backtesting.py | 32 ++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c0259c8fc..8bc6efb4d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,14 +95,16 @@ class Backtesting(object): #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + self.backslap_show_trades = True # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit @staticmethod @@ -389,7 +391,6 @@ class Backtesting(object): - Profit - trade duration - profit abs - :param bslap_results Dataframe :return: bslap_results Dataframe """ @@ -443,14 +444,19 @@ class Backtesting(object): return bslap_results_df - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int): + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. t_exit_ind is the index the last trade exited on or 0 if first time around this loop. + + stop_stops i """ # Timers, to be called if in debug def s(): @@ -478,6 +484,10 @@ class Backtesting(object): if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use t_open_ind = -1 # -1 ends the loop + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + return t_open_ind def backslap_pair(self, ticker_data, pair): @@ -564,6 +574,9 @@ class Backtesting(object): t_exit_ind = 0 # Start loop from first index t_exit_last = 0 # To test for exit + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + st = s() # Start timer for processing dataframe if debug: print('Processing:', pair) @@ -604,11 +617,13 @@ class Backtesting(object): Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len) + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) if debug: print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) @@ -971,6 +986,9 @@ class Backtesting(object): # append the dict to the list and print list bslap_pair_results.append(bslap_result) + if t_exit_type is "stop": + stop_stops_count = stop_stops_count + 1 + if debug: print("The trade dict is: \n", bslap_result) print("Trades dicts in list after append are: \n ", bslap_pair_results) From 3184c85dca50f60c263d8f7d3c4981d41a132e56 Mon Sep 17 00:00:00 2001 From: creslinux Date: Tue, 17 Jul 2018 21:33:11 +0000 Subject: [PATCH 023/174] default settings to trigger low, take stop --- freqtrade/optimize/backtesting.py | 55 +++++++++++++++++-------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8bc6efb4d..6cd271534 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -101,7 +101,7 @@ class Backtesting(object): self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = True # prints trades in addition to summary report + self.backslap_show_trades = False # prints trades in addition to summary report self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit @@ -414,33 +414,35 @@ class Backtesting(object): if debug: from pandas import set_option set_option('display.max_rows', 5000) - set_option('display.max_columns', 10) + set_option('display.max_columns', 20) pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - # if debug: - # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) ## Spends, Takes, Profit, Absolute Profit + print(bslap_results_df) # Buy Price - bslap_results_df['buy_sum'] = stake * bslap_results_df['open_rate'] - bslap_results_df['buy_fee'] = bslap_results_df['buy_sum'] * open_fee - bslap_results_df['buy_spend'] = bslap_results_df['buy_sum'] + bslap_results_df['buy_fee'] + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + # Sell price - bslap_results_df['sell_sum'] = stake * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] # profit_percent - bslap_results_df['profit_percent'] = bslap_results_df['sell_take'] / bslap_results_df['buy_spend'] - 1 + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + if debug: print("\n") print(bslap_results_df[ - ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) return bslap_results_df @@ -458,6 +460,8 @@ class Backtesting(object): stop_stops i """ + debug = self.debug + # Timers, to be called if in debug def s(): st = timeit.default_timer() @@ -486,7 +490,8 @@ class Backtesting(object): if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop t_open_ind = -1 # -1 ends the loop - print("Max stop limit ", stop_stops, "reached. Moving to next pair") + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") return t_open_ind @@ -1026,6 +1031,8 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + + ld_files = self.s() data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -1046,6 +1053,8 @@ class Backtesting(object): max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) + t_t = self.f(ld_files) + print("Load from json to file to df in mem took", t_t) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) @@ -1110,18 +1119,16 @@ class Backtesting(object): results ) ) - - ## TODO. Catch open trades for this report. - # logger.info( - # '\n=============================================== ' - # 'LEFT OPEN TRADES REPORT' - # ' ===============================================\n' - # '%s', - # self._generate_text_table( - # data, - # results.loc[results.open_at_end] - # ) - # ) + logger.info( + '\n=============================================== ' + 'LEFT OPEN TRADES REPORT' + ' ===============================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) + ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 79f931f29639e05232ba1044cebceaf2e6a3e332 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Wed, 25 Jul 2018 19:49:25 +0000 Subject: [PATCH 024/174] Update backtesting.py --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c0259c8fc..636a95b2d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -95,10 +95,10 @@ class Backtesting(object): #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = True # Main debug enable, very print heavy, enable 2 loops recommended + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = True # Debug vector calcs + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap self.backslap_show_trades = False # prints trades in addition to summary report From 482b85182ad77a517e79d98d772f93f1f2536061 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 26 Jul 2018 06:53:43 +0000 Subject: [PATCH 025/174] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ec541674..bfe01e6fe 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup(name='freqtrade', 'cachetools', 'coinmarketcap', 'scikit-optimize', - 'ujson' + 'ujson', 'py_find_1st' ], include_package_data=True, From e39ae45d2f5ff7c954b7a45201bf4a941c6c55cd Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 26 Jul 2018 18:40:45 +0000 Subject: [PATCH 026/174] Some reason did not push this... vector calcs redone. --- freqtrade/optimize/backtesting.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6cd271534..5b4487762 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -419,6 +419,12 @@ class Backtesting(object): pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] ## Spends, Takes, Profit, Absolute Profit @@ -438,6 +444,10 @@ class Backtesting(object): # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + if debug: print("\n") From 2dc3d6a6b857d558beb8b0291f3921d0670783f2 Mon Sep 17 00:00:00 2001 From: creslin <34645187+creslinux@users.noreply.github.com> Date: Thu, 26 Jul 2018 18:42:20 +0000 Subject: [PATCH 027/174] Update backtesting.py --- freqtrade/optimize/backtesting.py | 89 +++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 636a95b2d..5b4487762 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -98,11 +98,13 @@ class Backtesting(object): self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit @staticmethod @@ -389,7 +391,6 @@ class Backtesting(object): - Profit - trade duration - profit abs - :param bslap_results Dataframe :return: bslap_results Dataframe """ @@ -413,45 +414,64 @@ class Backtesting(object): if debug: from pandas import set_option set_option('display.max_rows', 5000) - set_option('display.max_columns', 10) + set_option('display.max_columns', 20) pd.set_option('display.width', 1000) pd.set_option('max_colwidth', 40) pd.set_option('precision', 12) + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - # if debug: - # print(bslap_results_df[['open_time', 'close_time', 'trade_duration']]) ## Spends, Takes, Profit, Absolute Profit + print(bslap_results_df) # Buy Price - bslap_results_df['buy_sum'] = stake * bslap_results_df['open_rate'] - bslap_results_df['buy_fee'] = bslap_results_df['buy_sum'] * open_fee - bslap_results_df['buy_spend'] = bslap_results_df['buy_sum'] + bslap_results_df['buy_fee'] + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + # Sell price - bslap_results_df['sell_sum'] = stake * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] # profit_percent - bslap_results_df['profit_percent'] = bslap_results_df['sell_take'] / bslap_results_df['buy_spend'] - 1 + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + if debug: print("\n") print(bslap_results_df[ - ['buy_sum', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) return bslap_results_df - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int): + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. t_exit_ind is the index the last trade exited on or 0 if first time around this loop. + + stop_stops i """ + debug = self.debug + # Timers, to be called if in debug def s(): st = timeit.default_timer() @@ -478,6 +498,11 @@ class Backtesting(object): if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use t_open_ind = -1 # -1 ends the loop + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + return t_open_ind def backslap_pair(self, ticker_data, pair): @@ -564,6 +589,9 @@ class Backtesting(object): t_exit_ind = 0 # Start loop from first index t_exit_last = 0 # To test for exit + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + st = s() # Start timer for processing dataframe if debug: print('Processing:', pair) @@ -604,11 +632,13 @@ class Backtesting(object): Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len) + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) if debug: print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) @@ -971,6 +1001,9 @@ class Backtesting(object): # append the dict to the list and print list bslap_pair_results.append(bslap_result) + if t_exit_type is "stop": + stop_stops_count = stop_stops_count + 1 + if debug: print("The trade dict is: \n", bslap_result) print("Trades dicts in list after append are: \n ", bslap_pair_results) @@ -1008,6 +1041,8 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) + + ld_files = self.s() data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -1028,6 +1063,8 @@ class Backtesting(object): max_open_trades = 0 preprocessed = self.tickerdata_to_dataframe(data) + t_t = self.f(ld_files) + print("Load from json to file to df in mem took", t_t) # Print timeframe min_date, max_date = self.get_timeframe(preprocessed) @@ -1092,18 +1129,16 @@ class Backtesting(object): results ) ) - - ## TODO. Catch open trades for this report. - # logger.info( - # '\n=============================================== ' - # 'LEFT OPEN TRADES REPORT' - # ' ===============================================\n' - # '%s', - # self._generate_text_table( - # data, - # results.loc[results.open_at_end] - # ) - # ) + logger.info( + '\n=============================================== ' + 'LEFT OPEN TRADES REPORT' + ' ===============================================\n' + '%s', + self._generate_text_table( + data, + results.loc[results.open_at_end] + ) + ) def setup_configuration(args: Namespace) -> Dict[str, Any]: From 0372485cf025aa3b4c7e5484d695899681094f05 Mon Sep 17 00:00:00 2001 From: creslinux Date: Thu, 26 Jul 2018 19:17:00 +0000 Subject: [PATCH 028/174] Some reason did not push this... vector calcs redone. --- freqtrade/optimize/backtesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 5b4487762..36ecc6826 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -423,12 +423,12 @@ class Backtesting(object): # csv = "cryptosher_before_debug" # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] ## Spends, Takes, Profit, Absolute Profit - print(bslap_results_df) + # print(bslap_results_df) # Buy Price bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying bslap_results_df['buy_fee'] = stake * open_fee From 1a673c6ac94e036b9f66343bce9c8d4cb237d91f Mon Sep 17 00:00:00 2001 From: Gert Date: Sat, 28 Jul 2018 14:23:18 -0700 Subject: [PATCH 029/174] working on moving backslap --- freqtrade/optimize/backslapping.py | 785 +++++++++++++++++++++++++++++ freqtrade/optimize/backtesting.py | 745 +-------------------------- 2 files changed, 810 insertions(+), 720 deletions(-) create mode 100644 freqtrade/optimize/backslapping.py diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py new file mode 100644 index 000000000..dff55648f --- /dev/null +++ b/freqtrade/optimize/backslapping.py @@ -0,0 +1,785 @@ +import timeit +from typing import Dict, Any + +from pandas import DataFrame + +from freqtrade.analyze import Analyze +from freqtrade.exchange import Exchange + + +class Backslapping: + """ + provides a quick way to evaluate strategies over a longer term of time + """ + + def __init__(self, config: Dict[str, Any], exchange = None) -> None: + """ + constructor + """ + + self.config = config + self.analyze = Analyze(self.config) + self.ticker_interval = self.analyze.strategy.ticker_interval + self.tickerdata_to_dataframe = self.analyze.tickerdata_to_dataframe + self.populate_buy_trend = self.analyze.populate_buy_trend + self.populate_sell_trend = self.analyze.populate_sell_trend + + ### + # + ### + if exchange is None: + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + else: + self.exchange = exchange + + self.fee = self.exchange.get_fee() + + self.stop_loss_value = self.analyze.strategy.stoploss + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + + def s(self): + st = timeit.default_timer() + return st + + def f(self, st): + return (timeit.default_timer() - st) + + def run(self,args): + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + trades = [] + trade_count_lock: Dict = {} + + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in processed.items(): + if self.debug_timing: # Start timer + fl = self.s() + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + if self.debug_timing: # print time taken + flt = self.f(fl) + # print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + # call bslap - results are a list of dicts + bslap_pair_results = self.backslap_pair(ticker_data, pair) + last_bslap_results = bslap_results + bslap_results = last_bslap_results + bslap_pair_results + + if self.debug_timing: # print time taken + tt = self.f(st) + print("Time to BackSlap :", pair, round(tt, 10)) + print("-----------------------") + + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # if debug: + # print("open_time and close_time converted to datetime columns") + + bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) + else: + from freqtrade.optimize.backtesting import BacktestResult + + bslap_results_df = [] + bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + + return bslap_results_df + + def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + import numpy as np + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + if debug: + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 20) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + + ## Spends, Takes, Profit, Absolute Profit + # print(bslap_results_df) + # Buy Price + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + + # Sell price + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + if debug: + print("\n") + print(bslap_results_df[ + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', + 'profit_abs', 'exit_type']]) + + return bslap_results_df + + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, + stop_stops_count: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. + + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + + stop_stops i + """ + debug = self.debug + + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + st = s() + t_open_ind: int + + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + + return t_open_ind + + def backslap_pair(self, ticker_data, pair): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st + from datetime import datetime + + ### backslap debug wrap + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy + + # Read Stop Loss Values and Stake + stop = self.stop_loss_value + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + + ### End Config + + pair: str = pair + + # ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) + if debug_2loops: + if loop == 3: + print( + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + if debug_timing: + st = s() + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) + + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if t_open_ind != -1: + + """ + 1 - Create views to search within for our open trade + + The views are our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) + """ + np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind + 1:] + + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) + + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind + 1 + + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", + np_t_stop_ind - 1, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") + + print( + "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. + + Buys and sells are triggered at candle close + Both will open their postions at the open of the next candle. i/e + 1 index + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logic test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") + + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind + + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = 'stop' # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_type = 'sell' # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully + else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = "No Exit" + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == 'stop': + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == 'sell': + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + + # Catch no exit found + if t_exit_type == "No Exit": + np_trade_exit_price = 0 + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind or t_exit_last == -1: + """ + Break loop and go on to next pair. + + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + """ + # TODO :add handing here to record none closed open trades + + if debug: + print(bslap_pair_results) + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result + bslap_result["pair"] = pair + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. + bslap_result["close_index"] = close_index + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + bslap_result["exit_type"] = t_exit_type + # append the dict to the list and print list + bslap_pair_results.append(bslap_result) + + if t_exit_type is "stop": + stop_stops_count = stop_stops_count + 1 + + if debug: + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8+trade", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 36ecc6826..b68c544f2 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -20,6 +20,7 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json +from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from profilehooks import profile from collections import OrderedDict @@ -54,6 +55,7 @@ class Backtesting(object): backtesting = Backtesting(config) backtesting.start() """ + def __init__(self, config: Dict[str, Any]) -> None: self.config = config self.analyze = Analyze(self.config) @@ -91,21 +93,22 @@ class Backtesting(object): self.np_bco: int = self.np_open # buys calculated on - open of the next candle. self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - #self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - #self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs + self.use_backslap = True # Enable backslap - if false Orginal code is executed. + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + self.backslap = Backslapping(config) @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -119,7 +122,7 @@ class Backtesting(object): for frame in data.values() ] return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ @@ -193,7 +196,6 @@ class Backtesting(object): buy_signal = sell_row.buy if self.analyze.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell): - return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), @@ -233,7 +235,6 @@ class Backtesting(object): def f(self, st): return (timeit.default_timer() - st) - def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -253,65 +254,9 @@ class Backtesting(object): use_backslap = self.use_backslap debug_timing = self.debug_timing_main_loop - if use_backslap: # Use Back Slap code - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - trades = [] - trade_count_lock: Dict = {} - - ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() - - if debug_timing: # print time taken - flt = self.f(fl) - #print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() - - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') - - #call bslap - results are a list of dicts - bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results - bslap_results = last_bslap_results + bslap_pair_results - - if debug_timing: # print time taken - tt = self.f(st) - print("Time to BackSlap :", pair, round(tt,10)) - print("-----------------------") - - - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - # if debug: - # print("open_time and close_time converted to datetime columns") - - bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) - else: - bslap_results_df = [] - bslap_results_df= DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - - - return bslap_results_df - - else: # use Original Back test code + if use_backslap: # Use Back Slap code + return self.backslap.run(args) + else: # use Original Back test code ########################## Original BT loop headers = ['date', 'buy', 'open', 'close', 'sell'] @@ -322,7 +267,7 @@ class Backtesting(object): trade_count_lock: Dict = {} for pair, pair_data in processed.items(): - if debug_timing: # Start timer + if debug_timing: # Start timer fl = self.s() pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run @@ -336,9 +281,9 @@ class Backtesting(object): ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken + if debug_timing: # print time taken flt = self.f(fl) - #print("populate_buy_trend:", pair, round(flt, 10)) + # print("populate_buy_trend:", pair, round(flt, 10)) st = self.s() # Convert from Pandas to list for performance reasons @@ -363,7 +308,6 @@ class Backtesting(object): trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], trade_count_lock, args) - if trade_entry: lock_pair_until = trade_entry.close_time trades.append(trade_entry) @@ -377,651 +321,9 @@ class Backtesting(object): print("Time to BackTest :", pair, round(tt, 10)) print("-----------------------") - return DataFrame.from_records(trades, columns=BacktestResult._fields) ####################### Original BT loop end - def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): - """ - The Results frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe - """ - import pandas as pd - import numpy as np - debug = self.debug_vector - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - #fee = 0.001 - - stake = self.config.get('stake_amount') - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - if debug: - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 20) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - # # Get before - # csv = "cryptosher_before_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - - ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) - # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending - - # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] - # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] - # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - - # # Get After - # csv="cryptosher_after_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - - if debug: - print("\n") - print(bslap_results_df[ - ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum','sell_fee', 'sell_take', 'profit_percent', 'profit_abs', 'exit_type']]) - - return bslap_results_df - - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): - import utils_find_1st as utf1st - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - - stop_stops i - """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - def f(st): - return (timeit.default_timer() - st) - - st = s() - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len -1 : # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") - - return t_open_ind - - def backslap_pair(self, ticker_data, pair): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy - - # Read Stop Loss Values and Stake - stop = self.stop_loss_value - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - def s(): - st = timeit.default_timer() - return st - def f(st): - return (timeit.default_timer() - st) - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - - - ####### - # Use vars set at top of backtest - np_buy: int = self.np_buy - np_open: int = self.np_open - np_close: int = self.np_close - np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - - ### End Config - - pair: str = pair - - #ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data - - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) - - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) - - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values - - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit - - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair - - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} - - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit - - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - if debug_timing: - st = s() - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag - - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind - - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind +1:] - - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind +1 - - - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", np_t_stop_ind -1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print("If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind -1 , np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind , np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = 'stop' # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index - t_exit_type = 'sell' # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = "No Exit" - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == 'stop': - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == 'sell': - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == "No Exit": - np_trade_exit_price = 0 - - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) - - # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - - if debug: - print(bslap_pair_results) - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is "stop": - stop_stops_count = stop_stops_count + 1 - - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - - # Send back List of trade dicts - return bslap_pair_results - def start(self) -> None: """ Run a backtesting end-to-end @@ -1099,23 +401,26 @@ class Backtesting(object): results ) ) - #optional print trades + # optional print trades if self.backslap_show_trades: TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + def to_fwf(df, fname): content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') print(content) DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - #optional save trades + # optional save trades if self.backslap_save_trades: TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) + def to_fwf(df, fname): content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') open(fname, "w").write(content) + DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") else: From ab66fe1b724cc50c883005282a08bda0f538327d Mon Sep 17 00:00:00 2001 From: Gert Date: Sat, 28 Jul 2018 19:45:33 -0700 Subject: [PATCH 030/174] prepared for tracking signals --- freqtrade/optimize/backslapping.py | 16 +++++++++------- freqtrade/optimize/backtesting.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py index 3eee5db37..b16515942 100644 --- a/freqtrade/optimize/backslapping.py +++ b/freqtrade/optimize/backslapping.py @@ -5,6 +5,7 @@ from pandas import DataFrame from freqtrade.exchange import Exchange from freqtrade.strategy import IStrategy +from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import StrategyResolver @@ -571,7 +572,7 @@ class Backslapping: # Stoploss trigger found before a sell =1 if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = 'stop' # Set Exit type (stop) + t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) np_t_exit_pri = np_sco # The price field our STOP exit will use if debug: print("Type STOP is first exit condition. " @@ -582,7 +583,7 @@ class Backslapping: # move sell onto next candle, we only look back on sell # will use the open price later. t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index - t_exit_type = 'sell' # Set Exit type (sell) + t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: print("Type SELL is first exit condition. " @@ -591,7 +592,7 @@ class Backslapping: # No stop or buy left in view - set t_exit_last -1 to handle gracefully else: t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = "No Exit" + t_exit_type = SellType.NONE np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column if debug: print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") @@ -688,16 +689,16 @@ class Backslapping: # TODO no! this is hard coded bleh fix this open np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == 'stop': + if t_exit_type == SellType.STOP_LOSS: if np_t_exit_pri == 6: np_trade_exit_price = np_t_stop_pri else: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == 'sell': + if t_exit_type == SellType.SELL_SIGNAL: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] # Catch no exit found - if t_exit_type == "No Exit": + if t_exit_type == SellType.NONE: np_trade_exit_price = 0 if debug_timing: @@ -762,10 +763,11 @@ class Backslapping: bslap_result["open_rate"] = round(np_trade_enter_price, 15) bslap_result["close_rate"] = round(np_trade_exit_price, 15) bslap_result["exit_type"] = t_exit_type + bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care # append the dict to the list and print list bslap_pair_results.append(bslap_result) - if t_exit_type is "stop": + if t_exit_type is SellType.STOP_LOSS: stop_stops_count = stop_stops_count + 1 if debug: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4ec3ac82a..8585a2628 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -455,7 +455,7 @@ class Backtesting(object): ) ) - if 'Sell Reason' in results.columns: + if 'sell_reason' in results.columns: logger.info( '\n' + ' SELL READON STATS '.center(119, '=') + From 04d5e857e2e9bc49d66055c1ec25088c45265e59 Mon Sep 17 00:00:00 2001 From: Gert Date: Tue, 31 Jul 2018 18:10:23 -0700 Subject: [PATCH 031/174] added option to easily switch between backtesting and backslapping from the commandline option --- freqtrade/arguments.py | 8 ++++++++ freqtrade/optimize/backtesting.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 022a2c739..1f6de2052 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -161,6 +161,14 @@ class Arguments(object): dest='exportfilename', metavar='PATH', ) + parser.add_argument( + '--backslap', + help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more " + "accurate results, unless you are utilizing Min/Max function in your strategy.", + required=False, + dest='backslap', + action='store_true' + ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 688b760ca..6ba6d7e3e 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -98,7 +98,13 @@ class Backtesting(object): # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - self.use_backslap = True # Enable backslap - if false Orginal code is executed. + if 'backslap' in config: + self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed. + else: + self.use_backslap = False + + logger.info("using backslap: {}".format(self.use_backslap)) + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended self.debug_timing = False # Stages within Backslap self.debug_2loops = False # Limit each pair to two loops, useful when debugging @@ -364,7 +370,6 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - ld_files = self.s() data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -374,6 +379,7 @@ class Backtesting(object): timerange=timerange ) + ld_files = self.s() if not data: logger.critical("No data found. Terminating.") return @@ -489,7 +495,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: # Ensure we do not use Exchange credentials config['exchange']['key'] = '' config['exchange']['secret'] = '' - + config['backslap'] = args.backslap if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: raise DependencyException('stake amount could not be "%s" for backtesting' % constants.UNLIMITED_STAKE_AMOUNT) From 721fb3e326e254a5afdc3f81ccaae25d5cbfcbf7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Aug 2018 10:12:57 +0200 Subject: [PATCH 032/174] remove unused profile import --- freqtrade/optimize/backtesting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6ba6d7e3e..d6de6cb0a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -23,7 +23,6 @@ from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from profilehooks import profile from collections import OrderedDict import timeit from time import sleep From bc6b80ff387ac19d1baba01175ada8d199f289c2 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 24 Aug 2018 11:59:10 +0200 Subject: [PATCH 033/174] Edge functionality drafted --- freqtrade/edge.py | 6 ++++++ freqtrade/freqtradebot.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 freqtrade/edge.py diff --git a/freqtrade/edge.py b/freqtrade/edge.py new file mode 100644 index 000000000..4795e8f4d --- /dev/null +++ b/freqtrade/edge.py @@ -0,0 +1,6 @@ +# EDGE +# WinRate and Expected Risk Reward should be calculated for all whitelisted pairs +# ASYNC: For each pair call backslap +# Save WR and ERR in Edge Dict +# Save last time updated for each pair in edge_last_update_time +# Calulate expectancy and position size and stop loss diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a2090d267..140ab2868 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -342,6 +342,9 @@ class FreqtradeBot(object): :return: True if a trade object has been created and persisted, False otherwise """ interval = self.strategy.ticker_interval + + # EDGE + # STAKE AMOUNT SHOULD COME FROM EDGE stake_amount = self._get_trade_stake_amount() if not stake_amount: @@ -362,6 +365,16 @@ class FreqtradeBot(object): if not whitelist: raise DependencyException('No currency pairs in whitelist') + + # EDGE + # WinRate and Expected Risk Reward should be calculated for all whitelisted pairs + # ASYNC: For each pair call backslap + # Save WR and ERR in Edge Dict + # Save last time updated for each pair in edge_last_update_time + # Calulate expectancy and position size and stop loss + + # whitelist = Edge.filter(whitelist) + # Pick pair based on buy signals for _pair in whitelist: thistory = self.exchange.get_candle_history(_pair, interval) From e30d23cf23c43701c16ef7bc0ad1eb2a8a77ee4f Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 14 Sep 2018 19:04:54 +0200 Subject: [PATCH 034/174] [draft] First version of edge positioning --- freqtrade/optimize/edge.py | 1023 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1023 insertions(+) create mode 100644 freqtrade/optimize/edge.py diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py new file mode 100644 index 000000000..ad37bf49b --- /dev/null +++ b/freqtrade/optimize/edge.py @@ -0,0 +1,1023 @@ +# pragma pylint: disable=missing-docstring, W0212, too-many-arguments + +""" +This module contains the backtesting logic +""" +import logging +import operator +import sys +from argparse import Namespace +from datetime import datetime, timedelta +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +import arrow +from pandas import DataFrame, to_datetime +from tabulate import tabulate +import numpy as np + +import freqtrade.optimize as optimize +from freqtrade import DependencyException, constants +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange +from freqtrade.misc import file_dump_json +from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from collections import OrderedDict +import timeit +from time import sleep + +import pdb + +logger = logging.getLogger(__name__) + +class Edge: + """ + provides a quick way to evaluate strategies over a longer term of time + """ + + def __init__(self, config: Dict[str, Any], exchange = None) -> None: + """ + constructor + """ + self.config = config + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend + + ### + # + ### + if exchange is None: + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + else: + self.exchange = exchange + + self.fee = self.exchange.get_fee() + + self.stop_loss_value = self.strategy.stoploss + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + + @staticmethod + def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum timeframe for the given backtest data + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeframe = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return min(timeframe, key=operator.itemgetter(0))[0], \ + max(timeframe, key=operator.itemgetter(1))[1] + + def s(self): + st = timeit.default_timer() + return st + + def f(self, st): + return (timeit.default_timer() - st) + + def run(self,args): + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + realistic = args.get('realistic', False) + stoploss_range = args['stoploss_range'] + trades = [] + trade_count_lock: Dict = {} + + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in processed.items(): + if self.debug_timing: # Start timer + fl = self.s() + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + if self.debug_timing: # print time taken + flt = self.f(fl) + # print("populate_buy_trend:", pair, round(flt, 10)) + st = self.s() + + # #dump same DFs to disk for offline testing in scratch + # f_pair:str = pair + # csv = f_pair.replace("/", "_") + # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv + # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') + + # call bslap - results are a list of dicts + for stoploss in stoploss_range: + bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) + + #bslap_results += self.backslap_pair(ticker_data, pair, -0.05) + # bslap_pair_results = self.backslap_pair(ticker_data, pair, -0.05) + # last_bslap_results = bslap_results + # bslap_results = last_bslap_results + bslap_pair_results + + + if self.debug_timing: # print time taken + tt = self.f(st) + print("Time to BackSlap :", pair, round(tt, 10)) + print("-----------------------") + + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) + # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) + # if debug: + # print("open_time and close_time converted to datetime columns") + + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + else: + from freqtrade.optimize.backtesting import BacktestResult + + bslap_results_df = [] + bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + + return bslap_results_df + + def vector_fill_results_table(self, bslap_results_df: DataFrame): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + import numpy as np + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + if debug: + print("Stake is,", stake, "the sum of currency to spend per trade") + print("The open fee is", open_fee, "The close fee is", close_fee) + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 20) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + # # Get before + # csv = "cryptosher_before_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + + ## Spends, Takes, Profit, Absolute Profit + # print(bslap_results_df) + # Buy Price + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + + # Sell price + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + # # Get After + # csv="cryptosher_after_debug" + # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') + + if debug: + print("\n") + print(bslap_results_df[ + ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', + 'profit_abs', 'exit_type']]) + + #pdb.set_trace() + return bslap_results_df + + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, + stop_stops_count: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. + + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + + stop_stops i + """ + debug = self.debug + + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + st = s() + t_open_ind: int + + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + + return t_open_ind + + def backslap_pair(self, ticker_data, pair, stoploss): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st + from datetime import datetime + + ### backslap debug wrap + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy + + # Read Stop Loss Values and Stake + # pdb.set_trace() + #stop = self.stop_loss_value + stop = stoploss + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + + ### End Config + + pair: str = pair + + # ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) + if debug_2loops: + if loop == 3: + print( + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + if debug_timing: + st = s() + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) + + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if t_open_ind != -1: + + """ + 1 - Create views to search within for our open trade + + The views are our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) + """ + np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind + 1:] + + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) + + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind + 1 + + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", + np_t_stop_ind - 1, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") + + print( + "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. + + Buys and sells are triggered at candle close + Both will open their postions at the open of the next candle. i/e + 1 index + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logic test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") + + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind + + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully + else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = SellType.NONE + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == SellType.STOP_LOSS: + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == SellType.SELL_SIGNAL: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + + # Catch no exit found + if t_exit_type == SellType.NONE: + np_trade_exit_price = 0 + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind or t_exit_last == -1: + """ + Break loop and go on to next pair. + + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + """ + # TODO :add handing here to record none closed open trades + + if debug: + print(bslap_pair_results) + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result + bslap_result["pair"] = pair + bslap_result["stoploss"] = stop + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. + bslap_result["close_index"] = close_index + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + bslap_result["exit_type"] = t_exit_type + bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care + # append the dict to the list and print list + bslap_pair_results.append(bslap_result) + + if t_exit_type is SellType.STOP_LOSS: + stop_stops_count = stop_stops_count + 1 + + if debug: + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8+trade", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results + + def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: + """ + This is a temporary version of edge positioning calculation. + The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The calulation will be done per pair and per strategy. + """ + + # Removing open trades from dataset + results = results[results.open_at_end == False] + ################################### + + # Removing pairs having less than min_trades_number + min_trades_number = 50 + results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) + ################################### + + + # Removing outliers (Pump&Dumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # And every value less than (standard deviation - 2*average) is out (dump) + # + # Calculating standard deviation of profits + std = results[["profit_abs"]].std() + # + # Calculating average of profits + avg = results[["profit_abs"]].mean() + # + # Removing Pumps + results = results[results.profit_abs < float(avg + 2*std)] + # + # Removing Dumps + results = results[results.profit_abs > float(avg - 2*std)] + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = 24*60 + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + + # Win Rate is the number of profitable trades + # Divided by number of trades + def winrate(x): + x = x[x > 0].count() / x.count() + return x + ############################# + + # Risk Reward Ratio + # 1 / ((loss money / losing trades) / (gained money / winning trades)) + def risk_reward_ratio(x): + x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) + return x + ############################## + + # Required Risk Reward + # (1/(winrate - 1) + def required_risk_reward(x): + x = (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + # The difference between risk reward ratio and required risk reward + # We use it as an indicator to find the most interesting pair to trade + def delta(x): + x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + + final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ + agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ + reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ + .groupby('pair').first().sort_values(by=['delta'], ascending=False) + + return final + + + def start(self) -> None: + """ + Run a backtesting end-to-end + :return: None + """ + data = {} + pairs = self.config['exchange']['pair_whitelist'] + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + if self.config.get('live'): + logger.info('Downloading data for all pairs in whitelist ...') + for pair in pairs: + data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) + else: + logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange + ) + + if not data: + logger.critical("No data found. Terminating.") + return + + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): + max_open_trades = self.config['max_open_trades'] + else: + logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') + max_open_trades = 0 + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + stoploss_range = np.arange(-0.11, -0.00, 0.01) + + # Execute backtest and print results + results = self.run( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'stoploss_range': stoploss_range, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } + ) + + logger.info( + '\n====================================================== ' + 'Edge positionning REPORT' + ' =======================================================\n' + '%s', + self._process_result( + data, + results, + stoploss_range + ) + ) + + +def setup_configuration(args: Namespace) -> Dict[str, Any]: + """ + Prepare the configuration for the backtesting + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['backslap'] = args.backslap + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: + raise DependencyException('stake amount could not be "%s" for backtesting' % + constants.UNLIMITED_STAKE_AMOUNT) + + return config + +# Initialize configuration +arguments = Arguments( + sys.argv[1:], + 'Simple High Frequency Trading Bot for crypto currencies' + ) +args = arguments.get_parsed_arg() + +config = setup_configuration(args) + +config["strategy"] = "MultiRSI" +edge = Edge(config) +edge.start() From 07ba14d1ea150366f953c9ae3e1ca4bc2bd25aec Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 15 Sep 2018 15:52:10 +0200 Subject: [PATCH 035/174] backslap bug resolved --- freqtrade/optimize/edge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index ad37bf49b..88e92f03b 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -627,7 +627,7 @@ class Edge: elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: # move sell onto next candle, we only look back on sell # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: From 88854cba2d2a7da001b37a4292121071c4a04bd7 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 15 Sep 2018 15:53:42 +0200 Subject: [PATCH 036/174] removing only pumps from dataset --- freqtrade/optimize/edge.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 88e92f03b..cfe1b3012 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -852,10 +852,9 @@ class Edge: ################################### - # Removing outliers (Pump&Dumps) from the dataset + # Removing outliers (Only Pumps) from the dataset # The method to detect outliers is to calculate standard deviation # Then every value more than (standard deviation + 2*average) is out (pump) - # And every value less than (standard deviation - 2*average) is out (dump) # # Calculating standard deviation of profits std = results[["profit_abs"]].std() @@ -865,9 +864,6 @@ class Edge: # # Removing Pumps results = results[results.profit_abs < float(avg + 2*std)] - # - # Removing Dumps - results = results[results.profit_abs > float(avg - 2*std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) From decaf6c42ea28cfcb932442533dcb94b02bf44f4 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 20 Sep 2018 16:15:53 +0200 Subject: [PATCH 037/174] =?UTF-8?q?Backslap=20bug=20on=20=E2=80=9Cstop=20l?= =?UTF-8?q?oss=20triggered=E2=80=9D=20indexes=20resolved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/optimize/backslapping.py | 24 +++++++++++++++++++++--- freqtrade/optimize/edge.py | 21 +++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py index 28b2bd460..7eba39a6e 100644 --- a/freqtrade/optimize/backslapping.py +++ b/freqtrade/optimize/backslapping.py @@ -136,7 +136,7 @@ class Backslapping: bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - + return bslap_results_df def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): @@ -280,10 +280,19 @@ class Backslapping: # debug = False # print values, to check accuracy debug_2loops = self.debug_2loops # only loop twice, for faster debug debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy + #debug = self.debug # print values, to check accuracy + debug = False + + ticker_data = ticker_data.sort_values(by=['date']) + ticker_data = ticker_data.reset_index(drop=True) + + #pandas_df = df.toPandas() + #pandas_df.to_json # Read Stop Loss Values and Stake + # pdb.set_trace() stop = self.stop_loss_value + #stop = stoploss p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price if debug: @@ -583,7 +592,7 @@ class Backslapping: elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: # move sell onto next candle, we only look back on sell # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: @@ -693,6 +702,7 @@ class Backslapping: if t_exit_type == SellType.STOP_LOSS: if np_t_exit_pri == 6: np_trade_exit_price = np_t_stop_pri + t_exit_ind = t_exit_ind + 1 else: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] if t_exit_type == SellType.SELL_SIGNAL: @@ -736,6 +746,7 @@ class Backslapping: opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades + if debug: print(bslap_pair_results) @@ -745,6 +756,12 @@ class Backslapping: Add trade to backtest looking results list of dicts Loop back to look for more trades. """ + + # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE + # removing the +1 here so prices match. + if t_exit_type == SellType.STOP_LOSS: + t_exit_ind = t_exit_ind - 1 + # Build trade dictionary ## In general if a field can be calculated later from other fields leave blank here ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time @@ -753,6 +770,7 @@ class Backslapping: close_index: int = t_exit_ind bslap_result = {} # Must have at start or we end up with a list of multiple same last result bslap_result["pair"] = pair + bslap_result["stoploss"] = stop bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index cfe1b3012..86ffb6747 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -133,6 +133,10 @@ class Edge: if self.debug_timing: # Start timer fl = self.s() + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() @@ -151,11 +155,6 @@ class Edge: for stoploss in stoploss_range: bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) - #bslap_results += self.backslap_pair(ticker_data, pair, -0.05) - # bslap_pair_results = self.backslap_pair(ticker_data, pair, -0.05) - # last_bslap_results = bslap_results - # bslap_results = last_bslap_results + bslap_pair_results - if self.debug_timing: # print time taken tt = self.f(st) @@ -627,7 +626,7 @@ class Edge: elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: # move sell onto next candle, we only look back on sell # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use if debug: @@ -737,6 +736,7 @@ class Edge: if t_exit_type == SellType.STOP_LOSS: if np_t_exit_pri == 6: np_trade_exit_price = np_t_stop_pri + t_exit_ind = t_exit_ind + 1 else: np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] if t_exit_type == SellType.SELL_SIGNAL: @@ -780,7 +780,7 @@ class Edge: opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades - + if debug: print(bslap_pair_results) break @@ -789,6 +789,12 @@ class Edge: Add trade to backtest looking results list of dicts Loop back to look for more trades. """ + + # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE + # removing the +1 here so prices match. + if t_exit_type == SellType.STOP_LOSS: + t_exit_ind = t_exit_ind - 1 + # Build trade dictionary ## In general if a field can be calculated later from other fields leave blank here ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time @@ -1014,6 +1020,5 @@ args = arguments.get_parsed_arg() config = setup_configuration(args) -config["strategy"] = "MultiRSI" edge = Edge(config) edge.start() From ef52c7b510a80b82d63a079901c93e28cf648b6e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:41:31 +0200 Subject: [PATCH 038/174] edge positioning put into package --- config.json.example | 7 + freqtrade/edge/__init__.py | 929 ++++++++++++++++++++++++++++++++ freqtrade/freqtradebot.py | 43 +- freqtrade/strategy/interface.py | 11 +- freqtrade/tests/conftest.py | 9 + 5 files changed, 980 insertions(+), 19 deletions(-) create mode 100644 freqtrade/edge/__init__.py diff --git a/config.json.example b/config.json.example index 7a0bb6b9b..801a5137c 100644 --- a/config.json.example +++ b/config.json.example @@ -50,6 +50,13 @@ "sell_profit_only": false, "ignore_roi_if_buy_signal": false }, + "edge": { + "enabled": false, + "process_throttle_secs": 1800, + "maximum_winrate_to_consider": 0.80, + "remove_pumps": true, + "minimum_delta_to_consider": 1 + }, "telegram": { "enabled": true, "token": "your_telegram_token", diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py new file mode 100644 index 000000000..9668e1a44 --- /dev/null +++ b/freqtrade/edge/__init__.py @@ -0,0 +1,929 @@ +import logging +import operator +import sys +from argparse import Namespace +from datetime import datetime, timedelta +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +import arrow +from pandas import DataFrame, to_datetime +from tabulate import tabulate +import numpy as np + +import freqtrade.optimize as optimize +from freqtrade import DependencyException, constants +from freqtrade.arguments import Arguments +from freqtrade.configuration import Configuration +from freqtrade.exchange import Exchange +from freqtrade.misc import file_dump_json +from freqtrade.persistence import Trade +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from freqtrade.optimize.backtesting import Backtesting +from collections import OrderedDict +import timeit +from time import sleep + +import pdb + +logger = logging.getLogger(__name__) + +class Edge(): + + config: Dict = {} + + def __init__(self, config: Dict[str, Any], exchange = None) -> None: + """ + constructor + """ + self.config = config + self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.ticker_interval = self.strategy.ticker_interval + self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe + self.get_timeframe = Backtesting.get_timeframe + self.populate_buy_trend = self.strategy.populate_buy_trend + self.populate_sell_trend = self.strategy.populate_sell_trend + + self._last_updated = None + self._cached_pairs = [] + self._total_capital = self.config['edge']['total_capital_in_stake_currency'] + self._allowed_risk = self.config['edge']['allowed_risk'] + + ### + # + ### + if exchange is None: + self.config['exchange']['secret'] = '' + self.config['exchange']['password'] = '' + self.config['exchange']['uid'] = '' + self.config['dry_run'] = True + self.exchange = Exchange(self.config) + else: + self.exchange = exchange + + self.fee = self.exchange.get_fee() + + self.stop_loss_value = self.strategy.stoploss + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + self.np_buy: int = 0 + self.np_open: int = 1 + self.np_close: int = 2 + self.np_sell: int = 3 + self.np_high: int = 4 + self.np_low: int = 5 + self.np_stop: int = 6 + self.np_bto: int = self.np_close # buys_triggered_on - should be close + self.np_bco: int = self.np_open # buys calculated on - open of the next candle. + self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close + self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close + # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close + # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close + + self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended + self.debug_timing = False # Stages within Backslap + self.debug_2loops = False # Limit each pair to two loops, useful when debugging + self.debug_vector = False # Debug vector calcs + self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap + + self.backslap_show_trades = False # prints trades in addition to summary report + self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt + + self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit + + + def calculate(self) -> bool: + pairs = self.config['exchange']['pair_whitelist'] + heartbeat = self.config['edge']['process_throttle_secs'] + + if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)): + return False + + data = {} + + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + + logger.info('Using local backtesting data (using whitelist in given config) ...') + + timerange = Arguments.parse_timerange(None if self.config.get( + 'timerange') is None else str(self.config.get('timerange'))) + + data = optimize.load_data( + self.config['datadir'], + pairs=pairs, + ticker_interval=self.ticker_interval, + refresh_pairs=self.config.get('refresh_pairs', False), + exchange=self.exchange, + timerange=timerange + ) + + if not data: + logger.critical("No data found. Edge is stopped ...") + return + + preprocessed = self.tickerdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days) ...', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + + + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + + # Max open trades need not be considered in Edge positioning + max_open_trades = 0 + + realistic = False + stoploss_range = np.arange(-0.11, -0.00, 0.01) + trades = [] + trade_count_lock: Dict = {} + + ########################### Call out BSlap Loop instead of Original BT code + bslap_results: list = [] + for pair, pair_data in preprocessed.items(): + + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + + ticker_data = self.populate_sell_trend( + self.populate_buy_trend(pair_data))[headers].copy() + + # call backslap - results are a list of dicts + for stoploss in stoploss_range: + bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) + + # Switch List of Trade Dicts (bslap_results) to Dataframe + # Fill missing, calculable columns, profit, duration , abs etc. + bslap_results_df = DataFrame(bslap_results) + + if len(bslap_results_df) > 0: # Only post process a frame if it has a record + bslap_results_df = self.vector_fill_results_table(bslap_results_df) + else: + from freqtrade.optimize.backtesting import BacktestResult + + bslap_results_df = [] + bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + + self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range) + self._last_updated = arrow.utcnow().timestamp + return True + + def sort_pairs(self, pairs) -> bool: + if len(self._cached_pairs) == 0: + self.calculate() + edge_sorted_pairs = [x[0] for x in self._cached_pairs] + return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] + + + def vector_fill_results_table(self, bslap_results_df: DataFrame): + """ + The Results frame contains a number of columns that are calculable + from othe columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param bslap_results Dataframe + :return: bslap_results Dataframe + """ + import pandas as pd + import numpy as np + debug = self.debug_vector + + # stake and fees + # stake = 0.015 + # 0.05% is 0.0005 + # fee = 0.001 + + stake = self.config.get('stake_amount') + fee = self.fee + open_fee = fee / 2 + close_fee = fee / 2 + + + bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] + bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + + ## Spends, Takes, Profit, Absolute Profit + # print(bslap_results_df) + # Buy Price + bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying + bslap_results_df['buy_fee'] = stake * open_fee + bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + + # Sell price + bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] + bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee + bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + # profit_percent + bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ + / bslap_results_df['buy_spend'] + # Absolute profit + bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + + + return bslap_results_df + + def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, + stop_stops_count: int): + import utils_find_1st as utf1st + """ + The purpose of this def is to return the next "buy" = 1 + after t_exit_ind. + + This function will also check is the stop limit for the pair has been reached. + if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. + + t_exit_ind is the index the last trade exited on + or 0 if first time around this loop. + + stop_stops i + """ + debug = self.debug + + # Timers, to be called if in debug + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + st = s() + t_open_ind: int + + """ + Create a view on our buy index starting after last trade exit + Search for next buy + """ + np_buy_arr_v = np_buy_arr[t_exit_ind:] + t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) + + ''' + If -1 is returned no buy has been found, preserve the value + ''' + if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index + t_open_ind = t_open_ind + t_exit_ind # Align numpy index + + if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use + t_open_ind = -1 # -1 ends the loop + + if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop + t_open_ind = -1 # -1 ends the loop + if debug: + print("Max stop limit ", stop_stops, "reached. Moving to next pair") + + return t_open_ind + + def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: + """ + This is a temporary version of edge positioning calculation. + The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The calulation will be done per pair and per strategy. + """ + + # Removing open trades from dataset + results = results[results.open_at_end == False] + ################################### + + # Removing pairs having less than min_trades_number + min_trades_number = 50 + results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) + ################################### + + + # Removing outliers (Only Pumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # + # Calculating standard deviation of profits + std = results[["profit_abs"]].std() + # + # Calculating average of profits + avg = results[["profit_abs"]].mean() + # + # Removing Pumps + results = results[results.profit_abs < float(avg + 2*std)] + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = 24*60 + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + + # Win Rate is the number of profitable trades + # Divided by number of trades + def winrate(x): + x = x[x > 0].count() / x.count() + return x + ############################# + + # Risk Reward Ratio + # 1 / ((loss money / losing trades) / (gained money / winning trades)) + def risk_reward_ratio(x): + x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) + return x + ############################## + + # Required Risk Reward + # (1/(winrate - 1) + def required_risk_reward(x): + x = (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + # The difference between risk reward ratio and required risk reward + # We use it as an indicator to find the most interesting pair to trade + def delta(x): + x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + return x + ############################## + + + final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ + agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ + reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ + .groupby('pair').first().sort_values(by=['delta'], ascending=False) + + # Returning an array of pairs in order of "delta" + return final.reset_index().values + + def backslap_pair(self, ticker_data, pair, stoploss): + import pandas as pd + import numpy as np + import timeit + import utils_find_1st as utf1st + from datetime import datetime + + ### backslap debug wrap + # debug_2loops = False # only loop twice, for faster debug + # debug_timing = False # print timing for each step + # debug = False # print values, to check accuracy + debug_2loops = self.debug_2loops # only loop twice, for faster debug + debug_timing = self.debug_timing # print timing for each step + debug = self.debug # print values, to check accuracy + + # Read Stop Loss Values and Stake + # pdb.set_trace() + #stop = self.stop_loss_value + stop = stoploss + p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + + if debug: + print("Stop is ", stop, "value from stragey file") + print("p_stop is", p_stop, "value used to multiply to entry price") + + if debug: + from pandas import set_option + set_option('display.max_rows', 5000) + set_option('display.max_columns', 8) + pd.set_option('display.width', 1000) + pd.set_option('max_colwidth', 40) + pd.set_option('precision', 12) + + def s(): + st = timeit.default_timer() + return st + + def f(st): + return (timeit.default_timer() - st) + + #### backslap config + ''' + Numpy arrays are used for 100x speed up + We requires setting Int values for + buy stop triggers and stop calculated on + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 + ''' + + ####### + # Use vars set at top of backtest + np_buy: int = self.np_buy + np_open: int = self.np_open + np_close: int = self.np_close + np_sell: int = self.np_sell + np_high: int = self.np_high + np_low: int = self.np_low + np_stop: int = self.np_stop + np_bto: int = self.np_bto # buys_triggered_on - should be close + np_bco: int = self.np_bco # buys calculated on - open of the next candle. + np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close + np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + + ### End Config + + pair: str = pair + + # ticker_data: DataFrame = ticker_dfs[t_file] + bslap: DataFrame = ticker_data + + # Build a single dimension numpy array from "buy" index for faster search + # (500x faster than pandas) + np_buy_arr = bslap['buy'].values + np_buy_arr_len: int = len(np_buy_arr) + + # use numpy array for faster searches in loop, 20x faster than pandas + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + + # Build a numpy list of date-times. + # We use these when building the trade + # The rationale is to address a value from a pandas cell is thousands of + # times more expensive. Processing time went X25 when trying to use any data from pandas + np_bslap_dates = bslap['date'].values + + loop: int = 0 # how many time around the loop + t_exit_ind = 0 # Start loop from first index + t_exit_last = 0 # To test for exit + + stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at + stop_stops_count = 0 # stop counter per pair + + st = s() # Start timer for processing dataframe + if debug: + print('Processing:', pair) + + # Results will be stored in a list of dicts + bslap_pair_results: list = [] + bslap_result: dict = {} + + while t_exit_ind < np_buy_arr_len: + loop = loop + 1 + if debug or debug_timing: + print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) + if debug_2loops: + if loop == 3: + print( + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") + break + ''' + Dev phases + Phase 1 + 1) Manage buy, sell, stop enter/exit + a) Find first buy index + b) Discover first stop and sell hit after buy index + c) Chose first instance as trade exit + + Phase 2 + 2) Manage dynamic Stop and ROI Exit + a) Create trade slice from 1 + b) search within trade slice for dynamice stop hit + c) search within trade slice for ROI hit + ''' + + if debug_timing: + st = s() + ''' + 0 - Find next buy entry + Finds index for first (buy = 1) flag + + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" + Required: t_exit_ind - Either 0, first loop. Or The index we last exited on + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: stop_stop_counts - count of stops hit in the pair + Provides: The next "buy" index after t_exit_ind + + If -1 is returned no buy has been found in remainder of array, skip to exit loop + ''' + t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) + + if debug: + print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) + print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") + if debug_timing: + t_t = f(st) + print("0-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if t_open_ind != -1: + + """ + 1 - Create views to search within for our open trade + + The views are our search space for the next Stop or Sell + Numpy view is employed as: + 1,000 faster than pandas searches + Pandas cannot assure it will always return a view, it may make a slow copy. + + The view contains columns: + buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + + Requires: np_bslap is our numpy array of the ticker DataFrame + Requires: t_open_ind is the index row with the buy. + Provides: np_t_open_v View of array after buy. + Provides: np_t_open_v_stop View of array after buy +1 + (Stop will search in here to prevent stopping in the past) + """ + np_t_open_v = np_bslap[t_open_ind:] + np_t_open_v_stop = np_bslap[t_open_ind + 1:] + + if debug: + print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) + print("Numpy View: Buy - Open - Close - Sell - High - Low") + print("Row 0", np_t_open_v[0]) + print("Row 1", np_t_open_v[1], ) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 2 - Calculate our stop-loss price + + As stop is based on buy price of our trade + - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle + - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. + This is as we only see the CLOSE after it has happened. + The back test assumption is we have bought at first available price, the OPEN + + Requires: np_bslap - is our numpy array of the ticker DataFrame + Requires: t_open_ind - is the index row with the first buy. + Requires: p_stop - is the stop rate, ie. 0.99 is -1% + Provides: np_t_stop_pri - The value stop-loss will be triggered on + ''' + np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) + + if debug: + print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) + if debug_timing: + t_t = f(st) + print("2-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 3 - Find candle STO is under Stop-Loss After Trade opened. + + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri + + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) + Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" + Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop + Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss + ''' + np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], + np_t_stop_pri, + utf1st.cmp_smaller) + + # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. + np_t_stop_ind = np_t_stop_ind + 1 + + if debug: + print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", + np_t_stop_ind - 1, + ". STO is using field", np_sto, + "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") + + print( + "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") + print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) + print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) + if debug_timing: + t_t = f(st) + print("3-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 4 - Find first sell index after trade open + + First index in the view np_t_open_v where ['sell'] = 1 + + Requires: np_t_open_v - view of ticker_data from buy onwards + Requires: no_sell - integer '3', the buy column in the array + Provides: np_t_sell_ind index of view where first sell=1 after buy + ''' + # Use numpy array for faster search for sell + # Sell uses column 3. + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # Numpy searches 25-35x quicker than pandas on this data + + np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], + 1, utf1st.cmp_equal) + if debug: + print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) + print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") + print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) + print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) + if debug_timing: + t_t = f(st) + print("4-numpy", str.format('{0:.17f}', t_t)) + st = s() + + ''' + 5 - Determine which was hit first a stop or sell + To then use as exit index price-field (sell on buy, stop on stop) + + STOP takes priority over SELL as would be 'in candle' from tick data + Sell would use Open from Next candle. + So in a draw Stop would be hit first on ticker data in live + + Validity of when types of trades may be executed can be summarised as: + + Tick View + index index Buy Sell open low close high Stop price + open 2am 94 -1 0 0 ----- ------ ------ ----- ----- + open 3am 95 0 1 0 ----- ------ trg buy ----- ----- + open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out + open 5am 97 2 0 0 Exit ------ ------- ----- ----- + open 6am 98 3 0 0 ----- ------ ------- ----- ----- + + -1 means not found till end of view i.e no valid Stop found. Exclude from match. + Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. + + Buys and sells are triggered at candle close + Both will open their postions at the open of the next candle. i/e + 1 index + + Stop and buy Indexes are on the view. To map to the ticker dataframe + the t_open_ind index should be summed. + + np_t_stop_ind: Stop Found index in view + t_exit_ind : Sell found in view + t_open_ind : Where view was started on ticker_data + + TODO: fix this frig for logic test,, case/switch/dictionary would be better... + more so when later testing many options, dynamic stop / roi etc + cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) + cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) + + ''' + if debug: + print("\n(5) numpy debug\nStop or Sell Logic Processing") + + # cludge for logic test (-1) means it was not found, set crazy high to lose < test + np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind + np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind + + # Stoploss trigger found before a sell =1 + if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: + t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index + t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) + np_t_exit_pri = np_sco # The price field our STOP exit will use + if debug: + print("Type STOP is first exit condition. " + "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) + + # Buy = 1 found before a stoploss triggered + elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: + # move sell onto next candle, we only look back on sell + # will use the open price later. + t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index + t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) + np_t_exit_pri = np_open # The price field our SELL exit will use + if debug: + print("Type SELL is first exit condition. " + "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) + + # No stop or buy left in view - set t_exit_last -1 to handle gracefully + else: + t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. + t_exit_type = SellType.NONE + np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column + if debug: + print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") + + # TODO: fix having to cludge/uncludge this .. + # Undo cludge + np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind + np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind + + if debug_timing: + t_t = f(st) + print("5-logic", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + ''' + Print out the buys, stops, sells + Include Line before and after to for easy + Human verification + ''' + # Combine the np_t_stop_pri value to bslap dataframe to make debug + # life easy. This is the current stop price based on buy price_ + # This is slow but don't care about performance in debug + # + # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 + # as there is no data column in the numpy array. + bslap['np_stop_pri'] = np_t_stop_pri + + # Buy + print("\n\nDATAFRAME DEBUG =================== BUY ", pair) + print("Numpy Array BUY Index is:", 0) + print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") + print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) + op_is = t_open_ind - 1 # Print open index start, line before + op_if = t_open_ind + 3 # Print open index finish, line after + print(bslap.iloc[op_is:op_if], "\n") + + # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) + if np_t_stop_ind < 0: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("No STOPS were found until the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== STOP ", pair) + print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) + df_stop_index = (t_open_ind + np_t_stop_ind) + + print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") + print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", + df_stop_index, ": \n", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), + "is less than", str.format('{0:.17f}', np_t_stop_pri)) + + print("A stoploss exit will be calculated at rate:", + str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) + + print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, + ": As live STOPs are not linked to O-C times") + + st_is = df_stop_index - 1 # Print stop index start, line before + st_if = df_stop_index + 2 # Print stop index finish, line after + print(bslap.iloc[st_is:st_if], "\n") + + # Sell + if np_t_sell_ind < 0: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("No SELLS were found till the end of ticker data file\n") + else: + print("DATAFRAME DEBUG =================== SELL ", pair) + print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) + df_sell_index = (t_open_ind + np_t_sell_ind) + + print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") + print("First Sell Index after Trade open is in candle", df_sell_index) + print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", + df_sell_index + 1) + sl_is = df_sell_index - 1 # Print sell index start, line before + sl_if = df_sell_index + 3 # Print sell index finish, line after + print(bslap.iloc[sl_is:sl_if], "\n") + + # Chosen Exit (stop or sell) + + print("DATAFRAME DEBUG =================== EXIT ", pair) + print("Exit type is :", t_exit_type) + print("trade exit price field is", np_t_exit_pri, "\n") + + if debug_timing: + t_t = f(st) + print("6-depra", str.format('{0:.17f}', t_t)) + st = s() + + ## use numpy view "np_t_open_v" for speed. Columns are + # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 + # exception is 6 which is use the stop value. + + # TODO no! this is hard coded bleh fix this open + np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] + if t_exit_type == SellType.STOP_LOSS: + if np_t_exit_pri == 6: + np_trade_exit_price = np_t_stop_pri + t_exit_ind = t_exit_ind + 1 + else: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + if t_exit_type == SellType.SELL_SIGNAL: + np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] + + # Catch no exit found + if t_exit_type == SellType.NONE: + np_trade_exit_price = 0 + + if debug_timing: + t_t = f(st) + print("7-numpy", str.format('{0:.17f}', t_t)) + st = s() + + if debug: + print("//////////////////////////////////////////////") + print("+++++++++++++++++++++++++++++++++ Trade Enter ") + print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) + print("--------------------------------- Trade Exit ") + print("Trade Exit Type is ", t_exit_type) + print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) + print("//////////////////////////////////////////////") + + else: # no buys were found, step 0 returned -1 + # Gracefully exit the loop + t_exit_last == -1 + if debug: + print("\n(E) No buys were found in remaining ticker file. Exiting", pair) + + # Loop control - catch no closed trades. + if debug: + print("---------------------------------------- end of loop", loop, + " Dataframe Exit Index is: ", t_exit_ind) + print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) + + if t_exit_last >= t_exit_ind or t_exit_last == -1: + """ + Break loop and go on to next pair. + + When last trade exit equals index of last exit, there is no + opportunity to close any more trades. + """ + # TODO :add handing here to record none closed open trades + + if debug: + print(bslap_pair_results) + break + else: + """ + Add trade to backtest looking results list of dicts + Loop back to look for more trades. + """ + + # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE + # removing the +1 here so prices match. + if t_exit_type == SellType.STOP_LOSS: + t_exit_ind = t_exit_ind - 1 + + # Build trade dictionary + ## In general if a field can be calculated later from other fields leave blank here + ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time + + # create a new dict + close_index: int = t_exit_ind + bslap_result = {} # Must have at start or we end up with a list of multiple same last result + bslap_result["pair"] = pair + bslap_result["stoploss"] = stop + bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower + bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower + bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. + bslap_result["close_index"] = close_index + bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete + bslap_result["open_at_end"] = False + bslap_result["open_rate"] = round(np_trade_enter_price, 15) + bslap_result["close_rate"] = round(np_trade_exit_price, 15) + bslap_result["exit_type"] = t_exit_type + bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care + # append the dict to the list and print list + bslap_pair_results.append(bslap_result) + + if t_exit_type is SellType.STOP_LOSS: + stop_stops_count = stop_stops_count + 1 + + if debug: + print("The trade dict is: \n", bslap_result) + print("Trades dicts in list after append are: \n ", bslap_pair_results) + + """ + Loop back to start. t_exit_last becomes where loop + will seek to open new trades from. + Push index on 1 to not open on close + """ + t_exit_last = t_exit_ind + 1 + + if debug_timing: + t_t = f(st) + print("8+trade", str.format('{0:.17f}', t_t)) + + # Send back List of trade dicts + return bslap_pair_results + + def stake_amount(self, pair: str) -> str: + info = [x for x in self._cached_pairs if x[0] == pair][0] + stoploss = info[1] + + allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + + return (allowed_dollars_at_risk / symbol_strategy_stop_loss) + + ### stake amount is the same as position size + ### calculate position size + # print("\n~~~~~~~~ Position Size ~~~~~~~~") + # print("bid price is ", bid_price) + # print("stop trigger is ", stop_trigger_price) + + # allowed_dollars_at_risk = total_capital * allowed_risk + # print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) + + # position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) + # print("position_size in dollars", round(position_size, 5 )) + + # buy_amount = position_size / bid_price + # print("amount of tokens to buy ", round(buy_amount,5)) + + # check_risk = (buy_amount * (bid_price - stop_trigger_price)) + # print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") + + def stoploss(self, pair: str) -> float: + info = [x for x in self._cached_pairs if x[0] == pair][0] + return info[1] \ No newline at end of file diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d34807090..a0abc8665 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,6 +17,7 @@ from cachetools import TTLCache, cached from freqtrade import (DependencyException, OperationalException, TemporaryError, __version__, constants, persistence) from freqtrade.exchange import Exchange +from freqtrade.edge import Edge from freqtrade.persistence import Trade from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -24,6 +25,8 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.exchange.exchange_helpers import order_book_to_dataframe +import pdb + logger = logging.getLogger(__name__) @@ -54,6 +57,7 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) + self.edge = Edge(self.config, self.exchange) self._init_modules() def _init_modules(self) -> None: @@ -185,6 +189,10 @@ class FreqtradeBot(object): # Refreshing candles self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Calculating Edge positiong + if self.config['edge']['enabled']: + self.edge.calculate() + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() @@ -307,13 +315,17 @@ class FreqtradeBot(object): return used_rate - def _get_trade_stake_amount(self) -> Optional[float]: + def _get_trade_stake_amount(self, pair="") -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency :return: float: Stake Amount """ - stake_amount = self.config['stake_amount'] + if self.config['edge']['enabled']: + stake_amount = self.edge.stake_amount(pair) + else: + stake_amount = self.config['stake_amount'] + avaliable_amount = self.exchange.get_balance(self.config['stake_currency']) if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: @@ -372,17 +384,8 @@ class FreqtradeBot(object): """ interval = self.strategy.ticker_interval - # EDGE - # STAKE AMOUNT SHOULD COME FROM EDGE - stake_amount = self._get_trade_stake_amount() + logger.info('Checking buy signals to create a new trade: ...') - if not stake_amount: - return False - - logger.info( - 'Checking buy signals to create a new trade with stake_amount: %f ...', - stake_amount - ) whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) # Remove currently opened and latest pairs from whitelist @@ -396,9 +399,13 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals + if self.config['edge']['enabled']: + whitelist = self.edge.sort_pairs(whitelist) + for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: + stake_amount = self._get_trade_stake_amount(_pair) bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ @@ -436,11 +443,12 @@ class FreqtradeBot(object): """ pair_s = pair.replace('_', '/') pair_url = self.exchange.get_pair_detail_url(pair) - stake_currency = self.config['stake_currency'] - fiat_currency = self.config.get('fiat_display_currency', None) + fiat_currency = self.config.get('fiat_display_currency', None) + # Calculate amount buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) + stake_currency = self.config['stake_currency'] min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: @@ -622,7 +630,12 @@ class FreqtradeBot(object): return False def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if (self.config['edge']['enabled']): + stoploss = self.edge.stoploss(trade.pair) + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + else: + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) logger.info('excuted sell') diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6afa4161b..13fe01a70 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool) -> SellCheckTuple: + sell: bool, force_stoploss=0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. @@ -211,7 +211,7 @@ class IStrategy(ABC): """ current_profit = trade.calc_profit_percent(rate) stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit) + current_profit=current_profit, force_stoploss=force_stoploss) if stoplossflag.sell_flag: return stoplossflag @@ -237,7 +237,7 @@ class IStrategy(ABC): return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, - current_profit: float) -> SellCheckTuple: + current_profit: float, force_stoploss: float) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not @@ -246,7 +246,10 @@ class IStrategy(ABC): trailing_stop = self.config.get('trailing_stop', False) - trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + if force_stoploss == 0: + trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) + else: + trade.adjust_stop_loss(trade.open_rate, force_stoploss, initial=True) # evaluate if the stoploss was hit if self.stoploss is not None and trade.stop_loss >= current_rate: diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index af9062cab..14e8a151d 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -127,6 +127,15 @@ def default_conf(): "NEO/BTC" ] }, + "edge": { + "enabled": False, + "process_throttle_secs": 1800, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "maximum_winrate_to_consider": 0.80, + "remove_pumps": True, + "minimum_delta_to_consider": 1 + }, "telegram": { "enabled": True, "token": "token", From 4746aea05c79724dfe255bd9574226b6f9454f66 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:42:04 +0200 Subject: [PATCH 039/174] test file for edge (will be removed) --- freqtrade/optimize/edge.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py index 86ffb6747..efafb8d77 100644 --- a/freqtrade/optimize/edge.py +++ b/freqtrade/optimize/edge.py @@ -912,6 +912,7 @@ class Edge: reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ .groupby('pair').first().sort_values(by=['delta'], ascending=False) + pdb.set_trace() return final @@ -1022,3 +1023,15 @@ config = setup_configuration(args) edge = Edge(config) edge.start() + +allowed_dollars_at_risk = total_capital * allowed_risk +print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) + +position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) +print("position_size in dollars", round(position_size, 5 )) + +buy_amount = position_size / bid_price +print("amount of tokens to buy ", round(buy_amount,5)) + +check_risk = (buy_amount * (bid_price - stop_trigger_price)) +print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") \ No newline at end of file From 2d432bfa953dd2d35f2d95edb7580be355266f38 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:54:37 +0200 Subject: [PATCH 040/174] backtesting rollbacked to develop branch --- freqtrade/optimize/backtesting.py | 442 ++++++++++-------------------- 1 file changed, 141 insertions(+), 301 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c539d0154..d0b70afc7 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,11 +6,13 @@ This module contains the backtesting logic import logging import operator from argparse import Namespace +from copy import deepcopy from datetime import datetime, timedelta +from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow -from pandas import DataFrame, to_datetime +from pandas import DataFrame from tabulate import tabulate import freqtrade.optimize as optimize @@ -19,15 +21,9 @@ from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange from freqtrade.misc import file_dump_json -from freqtrade.optimize.backslapping import Backslapping from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import OrderedDict -import timeit -from time import sleep - -import pdb logger = logging.getLogger(__name__) @@ -61,11 +57,6 @@ class Backtesting(object): def __init__(self, config: Dict[str, Any]) -> None: self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.advise_buy = self.strategy.advise_buy - self.advise_sell = self.strategy.advise_sell # Reset keys for backtesting self.config['exchange']['key'] = '' @@ -73,51 +64,35 @@ class Backtesting(object): self.config['exchange']['password'] = '' self.config['exchange']['uid'] = '' self.config['dry_run'] = True + self.strategylist: List[IStrategy] = [] + if self.config.get('strategy_list', None): + # Force one interval + self.ticker_interval = str(self.config.get('ticker_interval')) + for strat in list(self.config['strategy_list']): + stratconf = deepcopy(self.config) + stratconf['strategy'] = strat + self.strategylist.append(StrategyResolver(stratconf).strategy) + + else: + # only one strategy + strat = StrategyResolver(self.config).strategy + + self.strategylist.append(StrategyResolver(self.config).strategy) + # Load one strategy + self._set_strategy(self.strategylist[0]) + self.exchange = Exchange(self.config) self.fee = self.exchange.get_fee() - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - if 'backslap' in config: - self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed. - else: - self.use_backslap = False - - logger.info("using backslap: {}".format(self.use_backslap)) - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - self.backslap = Backslapping(config) + def _set_strategy(self, strategy): + """ + Load strategy into backtesting + """ + self.strategy = strategy + self.ticker_interval = self.config.get('ticker_interval') + self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe + self.advise_buy = strategy.advise_buy + self.advise_sell = strategy.advise_sell @staticmethod def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: @@ -131,7 +106,7 @@ class Backtesting(object): for frame in data.values() ] return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] + max(timeframe, key=operator.itemgetter(1))[1] def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str: """ @@ -142,13 +117,10 @@ class Backtesting(object): floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') tabular_data = [] - # headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - # 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'total loss ab', 'total profit ab', 'Risk Reward Ratio', 'Win Rate'] headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'RRR', 'Win Rate %', 'Required RR'] + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] for pair in data: result = results[results.pair == pair] - win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None tabular_data.append([ pair, len(result.index), @@ -158,12 +130,7 @@ class Backtesting(object): str(timedelta( minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', len(result[result.profit_abs > 0]), - len(result[result.profit_abs < 0]), - # result[result.profit_abs < 0]['profit_abs'].sum(), - # result[result.profit_abs > 0]['profit_abs'].sum(), - abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))), - win_rate * 100 if win_rate else "nan", - ((1 / win_rate) - 1) if win_rate else "nan" + len(result[result.profit_abs < 0]) ]) # Append Total @@ -180,88 +147,42 @@ class Backtesting(object): ]) return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") - - def _generate_text_table_edge_positioning(self, data: Dict[str, Dict], results: DataFrame) -> str: - """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. - The calulation will be done per pair and per strategy. - """ - - tabular_data = [] - headers = ['Number of trades', 'RRR', 'Win Rate %', 'Required RR'] - - ### - # The algorithm should be: - # 1) Removing outliers from dataframe. i.e. all profit_percent which are outside (mean -+ (2 * (standard deviation))). - # 2) Removing pairs with less than X trades (X defined in config). - # 3) Calculating RRR and WR. - # 4) Removing pairs for which WR and RRR are not in an acceptable range (e.x. WR > 95%). - # 5) Sorting the result based on the delta between required RR and RRR. - - # Here we assume initial data in order to calculate position size. - # these values will be replaced by exchange info or config - for pair in data: - result = results[results.pair == pair] - - # WinRate is calculated as follows: (Number of profitable trades) / (Total Trades) - win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None - - # Risk Reward Ratio is calculated as follows: 1 / ((total loss on losing trades / number of losing trades) / (total gain on profitable trades / number of winning trades)) - risk_reward_ratio = abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))) - - # Required Reward Ratio is (1 / WinRate) - 1 - required_risk_reward = ((1 / win_rate) - 1) if win_rate else None - - #pdb.set_trace() - - tabular_data.append([ - pair, - len(result.index), - risk_reward_ratio, - win_rate * 100 if win_rate else "nan", - required_risk_reward - ]) - - # for pair in data: - # result = results[results.pair == pair] - # win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None - # tabular_data.append([ - # pair, - # #len(result.index), - # #result.profit_percent.mean() * 100.0, - # #result.profit_percent.sum() * 100.0, - # #result.profit_abs.sum(), - # str(timedelta( - # minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', - # len(result[result.profit_abs > 0]), - # len(result[result.profit_abs < 0]), - # # result[result.profit_abs < 0]['profit_abs'].sum(), - # # result[result.profit_abs > 0]['profit_abs'].sum(), - # abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))), - # win_rate * 100 if win_rate else "nan", - # ((1 / win_rate) - 1) if win_rate else "nan" - # ]) - - #return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") - return tabulate(tabular_data, headers=headers, tablefmt="pipe") - - - - def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: """ Generate small table outlining Backtest results """ - tabular_data = [] headers = ['Sell Reason', 'Count'] for reason, count in results['sell_reason'].value_counts().iteritems(): - tabular_data.append([reason.value, count]) + tabular_data.append([reason.value, count]) return tabulate(tabular_data, headers=headers, tablefmt="pipe") - def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: + def _generate_text_table_strategy(self, all_results: dict) -> str: + """ + Generate summary table per strategy + """ + stake_currency = str(self.config.get('stake_currency')) + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', + 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss'] + for strategy, results in all_results.items(): + tabular_data.append([ + strategy, + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs < 0]) + ]) + return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") + + def _store_backtest_result(self, recordfilename: str, results: DataFrame, + strategyname: Optional[str] = None) -> None: records = [(t.pair, t.profit_percent, t.open_time.timestamp(), t.close_time.timestamp(), t.open_index - 1, t.trade_duration, @@ -269,6 +190,11 @@ class Backtesting(object): for index, t in results.iterrows()] if records: + if strategyname: + # Inject strategyname to filename + recname = Path(recordfilename) + recordfilename = str(Path.joinpath( + recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix)) logger.info('Dumping backtest results to %s', recordfilename) file_dump_json(recordfilename, records) @@ -297,13 +223,14 @@ class Backtesting(object): sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell_row.sell) if sell.sell_flag: + return BacktestResult(pair=pair, profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=False, @@ -320,7 +247,7 @@ class Backtesting(object): open_time=buy_row.date, close_time=sell_row.date, trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), + sell_row.date - buy_row.date).total_seconds() // 60), open_index=buy_row.Index, close_index=sell_row.Index, open_at_end=True, @@ -333,13 +260,6 @@ class Backtesting(object): return btr return None - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - def backtest(self, args: Dict) -> DataFrame: """ Implements backtesting functionality @@ -355,50 +275,32 @@ class Backtesting(object): position_stacking: do we allow position stacking? (default: False) :return: DataFrame """ + headers = ['date', 'buy', 'open', 'close', 'sell'] + processed = args['processed'] + max_open_trades = args.get('max_open_trades', 0) + position_stacking = args.get('position_stacking', False) + trades = [] + trade_count_lock: Dict = {} + for pair, pair_data in processed.items(): + pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - use_backslap = self.use_backslap - debug_timing = self.debug_timing_main_loop - - if use_backslap: # Use Back Slap code - return self.backslap.run(args) - else: # use Original Back test code - ########################## Original BT loop - - headers = ['date', 'buy', 'open', 'close', 'sell'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - position_stacking = args.get('position_stacking', False) - trades = [] - trade_count_lock: Dict = {} - - for pair, pair_data in processed.items(): - if debug_timing: # Start timer - fl = self.s() - - pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run - - ticker_data = self.advise_sell( + ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + # to avoid using data from future, we buy/sell with signal from previous candle + ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) + ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) - ticker_data.drop(ticker_data.head(1).index, inplace=True) + ticker_data.drop(ticker_data.head(1).index, inplace=True) - if debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() + # Convert from Pandas to list for performance reasons + # (Looping Pandas is slow.) + ticker = [x for x in ticker_data.itertuples()] - # Convert from Pandas to list for performance reasons - # (Looping Pandas is slow.) - ticker = [x for x in ticker_data.itertuples()] - - lock_pair_until = None - for index, row in enumerate(ticker): - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off + lock_pair_until = None + for index, row in enumerate(ticker): + if row.buy == 0 or row.sell == 1: + continue # skip rows where no buy signal or that would immediately sell off if not position_stacking: if lock_pair_until is not None and row.date <= lock_pair_until: @@ -408,26 +310,20 @@ class Backtesting(object): if not trade_count_lock.get(row.date, 0) < max_open_trades: continue - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 + trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], - trade_count_lock, args) + trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:], + trade_count_lock, args) - if trade_entry: - lock_pair_until = trade_entry.close_time - trades.append(trade_entry) - else: - # Set lock_pair_until to end of testing period if trade could not be closed - # This happens only if the buy-signal was with the last candle - lock_pair_until = ticker_data.iloc[-1].date + if trade_entry: + lock_pair_until = trade_entry.close_time + trades.append(trade_entry) + else: + # Set lock_pair_until to end of testing period if trade could not be closed + # This happens only if the buy-signal was with the last candle + lock_pair_until = ticker_data.iloc[-1].date - if debug_timing: # print time taken - tt = self.f(st) - print("Time to BackTest :", pair, round(tt, 10)) - print("-----------------------") - - return DataFrame.from_records(trades, columns=BacktestResult._fields) - ####################### Original BT loop end + return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: """ @@ -448,7 +344,6 @@ class Backtesting(object): timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - data = optimize.load_data( self.config['datadir'], pairs=pairs, @@ -458,7 +353,6 @@ class Backtesting(object): timerange=timerange ) - ld_files = self.s() if not data: logger.critical("No data found. Terminating.") return @@ -468,109 +362,55 @@ class Backtesting(object): else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + all_results = {} - preprocessed = self.tickerdata_to_dataframe(data) - t_t = self.f(ld_files) - print("Load from json to file to df in mem took", t_t) + for strat in self.strategylist: + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) + self._set_strategy(strat) - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - - # Execute backtest and print results - results = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - if self.config.get('export', False): - self._store_backtest_result(self.config.get('exportfilename'), results) - - if self.use_backslap: - # logger.info( - # '\n====================================================== ' - # 'BackSLAP REPORT' - # ' =======================================================\n' - # '%s', - # self._generate_text_table( - # data, - # results - # ) - # ) + # need to reprocess data every time to populate signals + preprocessed = self.tickerdata_to_dataframe(data) + # Print timeframe + min_date, max_date = self.get_timeframe(preprocessed) logger.info( - '\n====================================================== ' - 'Edge positionning REPORT' - ' =======================================================\n' - '%s', - self._generate_text_table_edge_positioning( - data, - results - ) - ) - # optional print trades - if self.backslap_show_trades: - TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', - 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - - def to_fwf(df, fname): - content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') - print(content) - - DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - - # optional save trades - if self.backslap_save_trades: - TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs', - 'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1) - - def to_fwf(df, fname): - content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') - open(fname, "w").write(content) - - DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") - - else: - logger.info( - '\n================================================= ' - 'BACKTEST REPORT' - ' ==================================================\n' - '%s', - self._generate_text_table( - data, - results - ) + 'Measuring data from %s up to %s (%s days)..', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days ) - if 'sell_reason' in results.columns: - logger.info( - '\n' + - ' SELL READON STATS '.center(119, '=') + - '\n%s \n', - self._generate_text_table_sell_reason(data, results) - + # Execute backtest and print results + all_results[self.strategy.get_strategy_name()] = self.backtest( + { + 'stake_amount': self.config.get('stake_amount'), + 'processed': preprocessed, + 'max_open_trades': max_open_trades, + 'position_stacking': self.config.get('position_stacking', False), + } ) - else: - logger.info("no sell reasons available!") - logger.info( - '\n' + - ' LEFT OPEN TRADES REPORT '.center(119, '=') + - '\n%s', - self._generate_text_table( - data, - results.loc[results.open_at_end] - ) - ) + for strategy, results in all_results.items(): + + if self.config.get('export', False): + self._store_backtest_result(self.config['exportfilename'], results, + strategy if len(self.strategylist) > 1 else None) + + print(f"Result for strategy {strategy}") + print(' BACKTESTING REPORT '.center(119, '=')) + print(self._generate_text_table(data, results)) + + print(' SELL REASON STATS '.center(119, '=')) + print(self._generate_text_table_sell_reason(data, results)) + + print(' LEFT OPEN TRADES REPORT '.center(119, '=')) + print(self._generate_text_table(data, results.loc[results.open_at_end])) + print() + if len(all_results) > 1: + # Print Strategy summary table + print(' Strategy Summary '.center(119, '=')) + print(self._generate_text_table_strategy(all_results)) + print('\nFor more details, please look at the detail tables above') def setup_configuration(args: Namespace) -> Dict[str, Any]: @@ -585,7 +425,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]: # Ensure we do not use Exchange credentials config['exchange']['key'] = '' config['exchange']['secret'] = '' - config['backslap'] = args.backslap + if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: raise DependencyException('stake amount could not be "%s" for backtesting' % constants.UNLIMITED_STAKE_AMOUNT) @@ -605,4 +445,4 @@ def start(args: Namespace) -> None: # Initialize backtesting object backtesting = Backtesting(config) - backtesting.start() \ No newline at end of file + backtesting.start() From 74979943baaa3d870c64e35eb59194533bdcd32b Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:57:29 +0200 Subject: [PATCH 041/174] backslap removed from arguments --- freqtrade/arguments.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/freqtrade/arguments.py b/freqtrade/arguments.py index 4f47849d8..501c1784f 100644 --- a/freqtrade/arguments.py +++ b/freqtrade/arguments.py @@ -171,14 +171,6 @@ class Arguments(object): dest='exportfilename', metavar='PATH', ) - parser.add_argument( - '--backslap', - help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more " - "accurate results, unless you are utilizing Min/Max function in your strategy.", - required=False, - dest='backslap', - action='store_true' - ) @staticmethod def optimizer_shared_options(parser: argparse.ArgumentParser) -> None: @@ -262,7 +254,6 @@ class Arguments(object): self.optimizer_shared_options(hyperopt_cmd) self.hyperopt_options(hyperopt_cmd) - @staticmethod def parse_timerange(text: Optional[str]) -> TimeRange: """ From 4bd956d5b1da5f4ab7ed1d3a3e5051c10875a073 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 17:58:20 +0200 Subject: [PATCH 042/174] test file removed --- freqtrade/edge.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 freqtrade/edge.py diff --git a/freqtrade/edge.py b/freqtrade/edge.py deleted file mode 100644 index 4795e8f4d..000000000 --- a/freqtrade/edge.py +++ /dev/null @@ -1,6 +0,0 @@ -# EDGE -# WinRate and Expected Risk Reward should be calculated for all whitelisted pairs -# ASYNC: For each pair call backslap -# Save WR and ERR in Edge Dict -# Save last time updated for each pair in edge_last_update_time -# Calulate expectancy and position size and stop loss From 4fd037f83fb92ecb05f501eb2622b0394fbd05a1 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 18:00:37 +0200 Subject: [PATCH 043/174] removing pdb --- freqtrade/freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a0abc8665..febdcea40 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -25,7 +25,6 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.exchange.exchange_helpers import order_book_to_dataframe -import pdb logger = logging.getLogger(__name__) From 61095db071e81d4e7a8845f95b499acc1e76d8fa Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:36:26 +0200 Subject: [PATCH 044/174] edge config enriched --- config.json.example | 7 +++++-- freqtrade/tests/conftest.py | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index 801a5137c..175a537f1 100644 --- a/config.json.example +++ b/config.json.example @@ -53,9 +53,12 @@ "edge": { "enabled": false, "process_throttle_secs": 1800, - "maximum_winrate_to_consider": 0.80, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "maximum_winrate": 0.80, + "min_trade_number": 15, "remove_pumps": true, - "minimum_delta_to_consider": 1 + "minimum_delta": 1 }, "telegram": { "enabled": true, diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 14e8a151d..09c746df2 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -130,11 +130,10 @@ def default_conf(): "edge": { "enabled": False, "process_throttle_secs": 1800, - "total_capital_in_stake_currency": 0.5, - "allowed_risk": 0.01, - "maximum_winrate_to_consider": 0.80, + "maximum_winrate": 0.80, + "min_trade_number": 15, "remove_pumps": True, - "minimum_delta_to_consider": 1 + "minimum_delta": 1 }, "telegram": { "enabled": True, From 3e3ed947cc67d0093de093029db79275e2250e24 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:46:18 +0200 Subject: [PATCH 045/174] =?UTF-8?q?added=20=E2=80=9Cmax=5Ftrade=5Fduration?= =?UTF-8?q?=E2=80=9D=20config=20+=20using=20=E2=80=9Cremove=5Fdumps?= =?UTF-8?q?=E2=80=9D=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json.example | 3 ++- freqtrade/edge/__init__.py | 30 ++++++++---------------------- freqtrade/tests/conftest.py | 3 +++ 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/config.json.example b/config.json.example index 175a537f1..f90b4f470 100644 --- a/config.json.example +++ b/config.json.example @@ -55,8 +55,9 @@ "process_throttle_secs": 1800, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, - "maximum_winrate": 0.80, + "maximum_winrate": 0.80, "min_trade_number": 15, + "max_trade_duration_minute": 1440, "remove_pumps": true, "minimum_delta": 1 }, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9668e1a44..46cdaaf32 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -43,11 +43,13 @@ class Edge(): self.get_timeframe = Backtesting.get_timeframe self.populate_buy_trend = self.strategy.populate_buy_trend self.populate_sell_trend = self.strategy.populate_sell_trend + + self.edge_config = self.config.get('edge', {}) self._last_updated = None self._cached_pairs = [] - self._total_capital = self.config['edge']['total_capital_in_stake_currency'] - self._allowed_risk = self.config['edge']['allowed_risk'] + self._total_capital = self.edge_config['total_capital_in_stake_currency'] + self._allowed_risk = self.edge_config['allowed_risk'] ### # @@ -303,7 +305,7 @@ class Edge(): ################################### # Removing pairs having less than min_trades_number - min_trades_number = 50 + min_trades_number = self.edge_config.get('min_trade_number', 15) results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) ################################### @@ -319,11 +321,12 @@ class Edge(): avg = results[["profit_abs"]].mean() # # Removing Pumps - results = results[results.profit_abs < float(avg + 2*std)] + if self.edge_config.get('remove_pumps', True): + results = results[results.profit_abs < float(avg + 2*std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = 24*60 + max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) results = results[results.trade_duration < max_trade_duration] ####################################################################### @@ -906,23 +909,6 @@ class Edge(): return (allowed_dollars_at_risk / symbol_strategy_stop_loss) - ### stake amount is the same as position size - ### calculate position size - # print("\n~~~~~~~~ Position Size ~~~~~~~~") - # print("bid price is ", bid_price) - # print("stop trigger is ", stop_trigger_price) - - # allowed_dollars_at_risk = total_capital * allowed_risk - # print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) - - # position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) - # print("position_size in dollars", round(position_size, 5 )) - - # buy_amount = position_size / bid_price - # print("amount of tokens to buy ", round(buy_amount,5)) - - # check_risk = (buy_amount * (bid_price - stop_trigger_price)) - # print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") def stoploss(self, pair: str) -> float: info = [x for x in self._cached_pairs if x[0] == pair][0] diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 09c746df2..8a3f71f51 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -130,8 +130,11 @@ def default_conf(): "edge": { "enabled": False, "process_throttle_secs": 1800, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, "maximum_winrate": 0.80, "min_trade_number": 15, + "max_trade_duration_minute": 1440, "remove_pumps": True, "minimum_delta": 1 }, From 3b925e46be86e38c0208b73f0f8f1555cd6c720e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:48:27 +0200 Subject: [PATCH 046/174] removing default pair value of _get_trade_stake_amount --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index febdcea40..7ac4f5522 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -314,7 +314,7 @@ class FreqtradeBot(object): return used_rate - def _get_trade_stake_amount(self, pair="") -> Optional[float]: + def _get_trade_stake_amount(self, pair) -> Optional[float]: """ Check if stake amount can be fulfilled with the available balance for the stake currency From fbc77c1f28d677d089812e22b03f8add571240de Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:55:36 +0200 Subject: [PATCH 047/174] moving stake_currency line back to its initial place --- freqtrade/freqtradebot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7ac4f5522..182a42819 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -442,13 +442,12 @@ class FreqtradeBot(object): """ pair_s = pair.replace('_', '/') pair_url = self.exchange.get_pair_detail_url(pair) - + stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) # Calculate amount buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) - stake_currency = self.config['stake_currency'] - + min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( From e1ca80734d1f3609e0e2cd9117ca45b9afa2dec7 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:58:37 +0200 Subject: [PATCH 048/174] removing unnecessary ujson import --- freqtrade/optimize/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 646f893e6..61710944b 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -17,9 +17,6 @@ from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange import importlib -ujson_found = importlib.util.find_spec("ujson") -if ujson_found is not None: - import ujson logger = logging.getLogger(__name__) From 66b1eac1db03c2c07ef3cb7206a6e5253ee60447 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 21:59:35 +0200 Subject: [PATCH 049/174] removing unnecessary ujson import --- freqtrade/optimize/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 646f893e6..aeb56811e 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -17,16 +17,9 @@ from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange import importlib -ujson_found = importlib.util.find_spec("ujson") -if ujson_found is not None: - import ujson logger = logging.getLogger(__name__) -if ujson_found is not None: - logger.debug('Loaded UltraJson ujson in optimize.py') - - def json_load(data): """Try to load data with ujson""" if _UJSON: From d6d3dfdcc2c22fb256596b5f9e2ef429586331cf Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 22:06:09 +0200 Subject: [PATCH 050/174] =?UTF-8?q?removing=20=E2=80=9Cif=20ujson=5Ffound?= =?UTF-8?q?=20is=20not=20None:=E2=80=9D=20as=20=E2=80=9Cjson=E2=80=9D=20re?= =?UTF-8?q?fers=20to=20=E2=80=9Cujson=E2=80=9D=20if=20it=20exists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/optimize/__init__.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index aeb56811e..7e44b3bac 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -77,17 +77,11 @@ def load_tickerdata_file( if os.path.isfile(gzipfile): logger.debug('Loading ticker data from file %s', gzipfile) with gzip.open(gzipfile) as tickerdata: - if ujson_found is not None: - pairdata = ujson.load(tickerdata, precise_float=True) - else: - pairdata = json.load(tickerdata) + pairdata = json.load(tickerdata) elif os.path.isfile(file): logger.debug('Loading ticker data from file %s', file) with open(file) as tickerdata: - if ujson_found is not None: - pairdata = ujson.load(tickerdata, precise_float=True) - else: - pairdata = json.load(tickerdata) + pairdata = json.load(tickerdata) else: return None @@ -183,10 +177,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - if ujson_found is not None: - data = ujson.load(file, precise_float=True) - else: - data = json.load(file) + data = json.load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: From cf37093e5a267dc0a47349e84047a20d2811b75a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 21 Sep 2018 22:07:12 +0200 Subject: [PATCH 051/174] empty dict default removed --- freqtrade/strategy/default_strategy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/strategy/default_strategy.py b/freqtrade/strategy/default_strategy.py index 323a7d95f..f1646779b 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/freqtrade/strategy/default_strategy.py @@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy): # Optimal ticker interval for the strategy ticker_interval = '5m' - def populate_indicators(self, dataframe: DataFrame, metadata: dict = {}) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Adds several different TA indicators to the given DataFrame @@ -199,7 +199,7 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict = {}) -> DataFrame: + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the buy signal for the given dataframe :param dataframe: DataFrame @@ -221,7 +221,7 @@ class DefaultStrategy(IStrategy): return dataframe - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict = {}) -> DataFrame: + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame From f1b4e4b36caf33f72128d05f29ca927b6f4bd746 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 22 Sep 2018 15:43:41 +0200 Subject: [PATCH 052/174] =?UTF-8?q?stop=20loss=20range=20=E2=80=9Cstart,?= =?UTF-8?q?=20end,=20step=E2=80=9D=20configurable=20for=20Edge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.json.example | 3 +++ freqtrade/edge/__init__.py | 8 ++++++-- freqtrade/tests/conftest.py | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/config.json.example b/config.json.example index f90b4f470..83e9e0d38 100644 --- a/config.json.example +++ b/config.json.example @@ -55,6 +55,9 @@ "process_throttle_secs": 1800, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.001, "maximum_winrate": 0.80, "min_trade_number": 15, "max_trade_duration_minute": 1440, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 46cdaaf32..f18e84f3f 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -148,7 +148,11 @@ class Edge(): max_open_trades = 0 realistic = False - stoploss_range = np.arange(-0.11, -0.00, 0.01) + stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) trades = [] trade_count_lock: Dict = {} @@ -165,7 +169,7 @@ class Edge(): # call backslap - results are a list of dicts for stoploss in stoploss_range: - bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) + bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 6)) # Switch List of Trade Dicts (bslap_results) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8a3f71f51..bece82436 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -132,6 +132,9 @@ def default_conf(): "process_throttle_secs": 1800, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.001, "maximum_winrate": 0.80, "min_trade_number": 15, "max_trade_duration_minute": 1440, From 29459d7d30b8e2019d033740d0b1200163b2c397 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 23 Sep 2018 04:51:53 +0200 Subject: [PATCH 053/174] import libraries organized. --- freqtrade/edge/__init__.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f18e84f3f..a9d049ee4 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -6,11 +6,13 @@ from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple import arrow -from pandas import DataFrame, to_datetime -from tabulate import tabulate -import numpy as np +from pandas import DataFrame, to_datetime +import pandas as pd + +from tabulate import tabulate import freqtrade.optimize as optimize +from freqtrade.optimize.backtesting import BacktestResult from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments from freqtrade.configuration import Configuration @@ -21,10 +23,12 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting from collections import OrderedDict -import timeit -from time import sleep -import pdb +import numpy as np +import timeit +import utils_find_1st as utf1st +from time import sleep +from pandas import set_option logger = logging.getLogger(__name__) @@ -178,8 +182,6 @@ class Edge(): if len(bslap_results_df) > 0: # Only post process a frame if it has a record bslap_results_df = self.vector_fill_results_table(bslap_results_df) else: - from freqtrade.optimize.backtesting import BacktestResult - bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) @@ -207,8 +209,6 @@ class Edge(): :param bslap_results Dataframe :return: bslap_results Dataframe """ - import pandas as pd - import numpy as np debug = self.debug_vector # stake and fees @@ -247,7 +247,6 @@ class Edge(): def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, stop_stops_count: int): - import utils_find_1st as utf1st """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. @@ -373,12 +372,6 @@ class Edge(): return final.reset_index().values def backslap_pair(self, ticker_data, pair, stoploss): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - ### backslap debug wrap # debug_2loops = False # only loop twice, for faster debug # debug_timing = False # print timing for each step @@ -388,7 +381,6 @@ class Edge(): debug = self.debug # print values, to check accuracy # Read Stop Loss Values and Stake - # pdb.set_trace() #stop = self.stop_loss_value stop = stoploss p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price @@ -398,7 +390,6 @@ class Edge(): print("p_stop is", p_stop, "value used to multiply to entry price") if debug: - from pandas import set_option set_option('display.max_rows', 5000) set_option('display.max_columns', 8) pd.set_option('display.width', 1000) From a26131cea359822d9290a46ec3d6f0017648ffab Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 14:21:37 +0200 Subject: [PATCH 054/174] .travis: install future before requirements.txt --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 981eedcf8..c81c96460 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy +- pip install future - pip install -r requirements.txt - pip install -e . jobs: From 136678351765ef8921405277dabb6d075dedcdf2 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 14:28:16 +0200 Subject: [PATCH 055/174] Dockerfile: installing future before requirements.txt --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2506665ab..0e90f0720 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ WORKDIR /freqtrade # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ + && pip install future \ && pip install -r requirements.txt --no-cache-dir # Install and execute From 303eefda76e04f9c9b714d1875a406dd1ec52cb3 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 14:55:49 +0200 Subject: [PATCH 056/174] =?UTF-8?q?test=5Fget=5Ftrade=5Fstake=5Famount=5Fu?= =?UTF-8?q?nlimited=5Famount=20fixed:=20=E2=80=9Cpair=E2=80=9D=20argument?= =?UTF-8?q?=20added=20to=20=5Fget=5Ftrade=5Fstake=5Famount?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/tests/test_freqtradebot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 5e982f11a..c246cc858 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -229,25 +229,25 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] / conf['max_open_trades'] # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' freqtrade.create_trade() - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('LTC/BTC') assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) # create 2 trades, order amount should be None freqtrade.create_trade() - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('XRP/BTC') assert result is None # set max_open_trades = None, so do not trade conf['max_open_trades'] = 0 freqtrade = FreqtradeBot(conf) - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('NEO/BTC') assert result is None From 76dd7549636fca2f8e17f82270e9efc2a7d3bb58 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 15:02:50 +0200 Subject: [PATCH 057/174] =?UTF-8?q?test=5Fget=5Ftrade=5Fstake=5Famount=20a?= =?UTF-8?q?nd=20test=5Fget=5Ftrade=5Fstake=5Famount=5Fno=5Fstake=5Famount?= =?UTF-8?q?=20fixed:=20=E2=80=9Cpair=E2=80=9D=20arg=20added=20to=20=5Fget?= =?UTF-8?q?=5Ftrade=5Fstake=5Famount?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/tests/test_freqtradebot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c246cc858..4d5039449 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -183,7 +183,7 @@ def test_get_trade_stake_amount(default_conf, ticker, limit_buy_order, fee, mock freqtrade = FreqtradeBot(default_conf) - result = freqtrade._get_trade_stake_amount() + result = freqtrade._get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] @@ -201,7 +201,7 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, freqtrade = FreqtradeBot(default_conf) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade._get_trade_stake_amount() + freqtrade._get_trade_stake_amount('ETH/BTC') def test_get_trade_stake_amount_unlimited_amount(default_conf, @@ -499,7 +499,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) assert freqtrade.create_trade() is False - assert freqtrade._get_trade_stake_amount() is None + assert freqtrade._get_trade_stake_amount('ETH/BTC') is None def test_create_trade_no_pairs(default_conf, ticker, limit_buy_order, fee, markets, mocker) -> None: From 308428644b2f7fcb8a241f68bb09b3730e8309f9 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 15:27:26 +0200 Subject: [PATCH 058/174] test_process_trade_creation log message changed: in reality the buy signal is actually triggered --- freqtrade/freqtradebot.py | 4 +--- freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 182a42819..e3cc669d9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -382,9 +382,6 @@ class FreqtradeBot(object): :return: True if a trade object has been created and persisted, False otherwise """ interval = self.strategy.ticker_interval - - logger.info('Checking buy signals to create a new trade: ...') - whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist']) # Remove currently opened and latest pairs from whitelist @@ -405,6 +402,7 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) + logger.info('Buy signal found: about create a new trade with stake_amount: %f ...', stake_amount) bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 4d5039449..79a09fde3 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -598,7 +598,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, assert trade.amount == 90.99181073703367 assert log_has( - 'Checking buy signals to create a new trade with stake_amount: 0.001000 ...', + 'Buy signal found: about create a new trade with stake_amount: 0.001000 ...', caplog.record_tuples ) From 027ec4d98eda93679e929655ad3688e9dcde0cd2 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 15:47:07 +0200 Subject: [PATCH 059/174] test_sell_profit_only_enable_loss and test_create_trade_limit_reached fixed --- freqtrade/freqtradebot.py | 2 ++ freqtrade/tests/test_freqtradebot.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e3cc669d9..60d286af1 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -402,6 +402,8 @@ class FreqtradeBot(object): (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: stake_amount = self._get_trade_stake_amount(_pair) + if not stake_amount: + return False logger.info('Buy signal found: about create a new trade with stake_amount: %f ...', stake_amount) bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 79a09fde3..b12ec1018 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -1423,7 +1423,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, market freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = \ - lambda current_rate, trade, current_time, current_profit: SellCheckTuple( + lambda current_rate, trade, current_time, force_stoploss, current_profit: SellCheckTuple( sell_flag=False, sell_type=SellType.NONE) freqtrade.create_trade() From a806dd45f27a8f023d148221f81937e4970f6b4e Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 16:02:29 +0200 Subject: [PATCH 060/174] lost in branches ! typo for some magical unknown reasons --- freqtrade/optimize/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 7e44b3bac..967227805 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -20,6 +20,7 @@ import importlib logger = logging.getLogger(__name__) + def json_load(data): """Try to load data with ujson""" if _UJSON: @@ -177,7 +178,7 @@ def load_cached_data_for_updating(filename: str, # read the cached file if os.path.isfile(filename): with open(filename, "rt") as file: - data = json.load(file) + data = json_load(file) # remove the last item, because we are not sure if it is correct # it could be fetched when the candle was incompleted if data: @@ -242,4 +243,4 @@ def download_backtesting_testdata(datadir: str, logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - misc.file_dump_json(filename, data) \ No newline at end of file + misc.file_dump_json(filename, data) From e8716f16ad6b868cef60d149128ec91f195770c1 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 17:47:50 +0200 Subject: [PATCH 061/174] calculating expectancy and sort pairs accordingly instead of delta --- freqtrade/edge/__init__.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a9d049ee4..77350feed 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -355,20 +355,23 @@ class Edge(): return x ############################## - # The difference between risk reward ratio and required risk reward - # We use it as an indicator to find the most interesting pair to trade - def delta(x): - x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + # Expectancy + # Tells you the interest percentage you should hope + # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 + def expectancy(x): + average_win = float(x[x > 0].sum() / x[x > 0].count()) + average_loss = float(abs(x[x < 0].sum() / x[x < 0].count())) + winrate = float(x[x > 0].count()/x.count()) + x = ((1 + average_win/average_loss) * winrate) - 1 return x ############################## - final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ - reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ - .groupby('pair').first().sort_values(by=['delta'], ascending=False) + agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ + reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ + .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) - # Returning an array of pairs in order of "delta" + # Returning an array of pairs in order of "expectancy" return final.reset_index().values def backslap_pair(self, ticker_data, pair, stoploss): From 40d73de357812868b982cd55aba09d677e050609 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 24 Sep 2018 19:22:30 +0200 Subject: [PATCH 062/174] refactoring backslap (round one) --- freqtrade/edge/__init__.py | 370 ++++++------------------------------- 1 file changed, 52 insertions(+), 318 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 77350feed..7a45b7b78 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,42 +1,33 @@ +# pragma pylint: disable=W0603 +""" Edge positioning package """ import logging -import operator -import sys -from argparse import Namespace -from datetime import datetime, timedelta -from typing import Any, Dict, List, NamedTuple, Optional, Tuple - +from typing import Any, Dict import arrow -from pandas import DataFrame, to_datetime +from pandas import DataFrame import pandas as pd -from tabulate import tabulate import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult -from freqtrade import DependencyException, constants from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration from freqtrade.exchange import Exchange -from freqtrade.misc import file_dump_json -from freqtrade.persistence import Trade from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting -from collections import OrderedDict import numpy as np import timeit import utils_find_1st as utf1st -from time import sleep -from pandas import set_option +import pdb logger = logging.getLogger(__name__) + class Edge(): config: Dict = {} - def __init__(self, config: Dict[str, Any], exchange = None) -> None: + def __init__(self, config: Dict[str, Any], exchange=None) -> None: """ constructor """ @@ -47,7 +38,7 @@ class Edge(): self.get_timeframe = Backtesting.get_timeframe self.populate_buy_trend = self.strategy.populate_buy_trend self.populate_sell_trend = self.strategy.populate_sell_trend - + self.edge_config = self.config.get('edge', {}) self._last_updated = None @@ -103,7 +94,6 @@ class Edge(): self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - def calculate(self) -> bool: pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] @@ -115,9 +105,8 @@ class Edge(): logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - logger.info('Using local backtesting data (using whitelist in given config) ...') - + #TODO: add "timerange" to Edge config timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -133,7 +122,7 @@ class Edge(): if not data: logger.critical("No data found. Edge is stopped ...") return - + preprocessed = self.tickerdata_to_dataframe(data) # Print timeframe @@ -144,27 +133,17 @@ class Edge(): max_date.isoformat(), (max_date - min_date).days ) - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - # Max open trades need not be considered in Edge positioning - max_open_trades = 0 - - realistic = False stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) - trades = [] - trade_count_lock: Dict = {} ########################### Call out BSlap Loop instead of Original BT code bslap_results: list = [] for pair, pair_data in preprocessed.items(): - - # Sorting dataframe by date and reset index + # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) @@ -184,18 +163,17 @@ class Edge(): else: bslap_results_df = [] bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - + self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range) self._last_updated = arrow.utcnow().timestamp return True - + def sort_pairs(self, pairs) -> bool: if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] - - + def vector_fill_results_table(self, bslap_results_df: DataFrame): """ The Results frame contains a number of columns that are calculable @@ -209,7 +187,6 @@ class Edge(): :param bslap_results Dataframe :return: bslap_results Dataframe """ - debug = self.debug_vector # stake and fees # stake = 0.015 @@ -221,7 +198,6 @@ class Edge(): open_fee = fee / 2 close_fee = fee / 2 - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) @@ -242,7 +218,6 @@ class Edge(): # Absolute profit bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - return bslap_results_df def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, @@ -250,26 +225,12 @@ class Edge(): """ The purpose of this def is to return the next "buy" = 1 after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. + This function will also check is the stop limit for the pair has been reached. if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - t_exit_ind is the index the last trade exited on or 0 if first time around this loop. - stop_stops i """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - st = s() t_open_ind: int """ @@ -290,17 +251,15 @@ class Edge(): if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") return t_open_ind def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. - The calulation will be done per pair and per strategy. + This is a temporary version of edge positioning calculation. + The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The calulation will be done per pair and per strategy. """ # Removing open trades from dataset @@ -312,8 +271,7 @@ class Edge(): results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) ################################### - - # Removing outliers (Only Pumps) from the dataset + # Removing outliers (Only Pumps) from the dataset # The method to detect outliers is to calculate standard deviation # Then every value more than (standard deviation + 2*average) is out (pump) # @@ -333,7 +291,6 @@ class Edge(): results = results[results.trade_duration < max_trade_duration] ####################################################################### - # Win Rate is the number of profitable trades # Divided by number of trades def winrate(x): @@ -354,7 +311,11 @@ class Edge(): x = (1/(x[x > 0].count()/x.count()) -1) return x ############################## - + + def delta(x): + x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) + return x + # Expectancy # Tells you the interest percentage you should hope # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 @@ -365,47 +326,20 @@ class Edge(): x = ((1 + average_win/average_loss) * winrate) - 1 return x ############################## - + final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ + agg([winrate, risk_reward_ratio, required_risk_reward, expectancy, delta]).\ reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) - + # Returning an array of pairs in order of "expectancy" return final.reset_index().values def backslap_pair(self, ticker_data, pair, stoploss): - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy - # Read Stop Loss Values and Stake - #stop = self.stop_loss_value stop = stoploss p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - #### backslap config ''' Numpy arrays are used for 100x speed up @@ -416,22 +350,12 @@ class Edge(): ####### # Use vars set at top of backtest - np_buy: int = self.np_buy np_open: int = self.np_open - np_close: int = self.np_close np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close np_bco: int = self.np_bco # buys calculated on - open of the next candle. np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - ### End Config - - pair: str = pair - # ticker_data: DataFrame = ticker_dfs[t_file] bslap: DataFrame = ticker_data @@ -457,23 +381,12 @@ class Edge(): stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at stop_stops_count = 0 # stop counter per pair - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - # Results will be stored in a list of dicts bslap_pair_results: list = [] bslap_result: dict = {} while t_exit_ind < np_buy_arr_len: loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print( - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break ''' Dev phases Phase 1 @@ -481,7 +394,7 @@ class Edge(): a) Find first buy index b) Discover first stop and sell hit after buy index c) Chose first instance as trade exit - + Phase 2 2) Manage dynamic Stop and ROI Exit a) Create trade slice from 1 @@ -489,72 +402,50 @@ class Edge(): c) search within trade slice for ROI hit ''' - if debug_timing: - st = s() ''' 0 - Find next buy entry Finds index for first (buy = 1) flag - + Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair + Requires: np_buy_arr_len - length of pair array. + Requires: stops_stops - number of stops allowed before stop trading a pair Requires: stop_stop_counts - count of stops hit in the pair Provides: The next "buy" index after t_exit_ind - + If -1 is returned no buy has been found in remainder of array, skip to exit loop ''' t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - if t_open_ind != -1: """ 1 - Create views to search within for our open trade - The views are our search space for the next Stop or Sell Numpy view is employed as: 1,000 faster than pandas searches Pandas cannot assure it will always return a view, it may make a slow copy. - The view contains columns: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - + Requires: np_bslap is our numpy array of the ticker DataFrame Requires: t_open_ind is the index row with the buy. Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 + Provides: np_t_open_v_stop View of array after buy +1 (Stop will search in here to prevent stopping in the past) """ np_t_open_v = np_bslap[t_open_ind:] np_t_open_v_stop = np_bslap[t_open_ind + 1:] - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - ''' 2 - Calculate our stop-loss price - + As stop is based on buy price of our trade - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. This is as we only see the CLOSE after it has happened. The back test assumption is we have bought at first available price, the OPEN - + Requires: np_bslap - is our numpy array of the ticker DataFrame Requires: t_open_ind - is the index row with the first buy. Requires: p_stop - is the stop rate, ie. 0.99 is -1% @@ -562,18 +453,11 @@ class Edge(): ''' np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - ''' 3 - Find candle STO is under Stop-Loss After Trade opened. - + where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - + Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop @@ -586,26 +470,11 @@ class Edge(): # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. np_t_stop_ind = np_t_stop_ind + 1 - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", - np_t_stop_ind - 1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print( - "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - ''' 4 - Find first sell index after trade open - + First index in the view np_t_open_v where ['sell'] = 1 - + Requires: np_t_open_v - view of ticker_data from buy onwards Requires: no_sell - integer '3', the buy column in the array Provides: np_t_sell_ind index of view where first sell=1 after buy @@ -617,26 +486,17 @@ class Edge(): np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() ''' 5 - Determine which was hit first a stop or sell To then use as exit index price-field (sell on buy, stop on stop) - + STOP takes priority over SELL as would be 'in candle' from tick data Sell would use Open from Next candle. So in a draw Stop would be hit first on ticker data in live - + Validity of when types of trades may be executed can be summarised as: - + Tick View index index Buy Sell open low close high Stop price open 2am 94 -1 0 0 ----- ------ ------ ----- ----- @@ -644,28 +504,26 @@ class Edge(): open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out open 5am 97 2 0 0 Exit ------ ------- ----- ----- open 6am 98 3 0 0 ----- ------ ------- ----- ----- - + -1 means not found till end of view i.e no valid Stop found. Exclude from match. Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - + Buys and sells are triggered at candle close Both will open their postions at the open of the next candle. i/e + 1 index - + Stop and buy Indexes are on the view. To map to the ticker dataframe the t_open_ind index should be summed. - + np_t_stop_ind: Stop Found index in view t_exit_ind : Sell found in view t_open_ind : Where view was started on ticker_data - + TODO: fix this frig for logic test,, case/switch/dictionary would be better... more so when later testing many options, dynamic stop / roi etc cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - + ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") # cludge for logic test (-1) means it was not found, set crazy high to lose < test np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind @@ -676,9 +534,6 @@ class Edge(): t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) # Buy = 1 found before a stoploss triggered elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: @@ -687,104 +542,18 @@ class Edge(): t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) # No stop or buy left in view - set t_exit_last -1 to handle gracefully else: t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. t_exit_type = SellType.NONE np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") # TODO: fix having to cludge/uncludge this .. # Undo cludge np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - ## use numpy view "np_t_open_v" for speed. Columns are # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 # exception is 6 which is use the stop value. @@ -804,43 +573,19 @@ class Edge(): if t_exit_type == SellType.NONE: np_trade_exit_price = 0 - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - else: # no buys were found, step 0 returned -1 # Gracefully exit the loop t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - if t_exit_last >= t_exit_ind or t_exit_last == -1: """ Break loop and go on to next pair. - + When last trade exit equals index of last exit, there is no opportunity to close any more trades. """ # TODO :add handing here to record none closed open trades - - if debug: - print(bslap_pair_results) break else: """ @@ -880,10 +625,6 @@ class Edge(): if t_exit_type is SellType.STOP_LOSS: stop_stops_count = stop_stops_count + 1 - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - """ Loop back to start. t_exit_last becomes where loop will seek to open new trades from. @@ -891,23 +632,16 @@ class Edge(): """ t_exit_last = t_exit_ind + 1 - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - # Send back List of trade dicts return bslap_pair_results def stake_amount(self, pair: str) -> str: info = [x for x in self._cached_pairs if x[0] == pair][0] stoploss = info[1] - allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + return position_size - return (allowed_dollars_at_risk / symbol_strategy_stop_loss) - - def stoploss(self, pair: str) -> float: info = [x for x in self._cached_pairs if x[0] == pair][0] return info[1] \ No newline at end of file From 87df4e455663e0ac64030ada8bc6fd70d6b7dcbd Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 15:20:53 +0200 Subject: [PATCH 063/174] refactoring backslap (round 2) --- freqtrade/edge/__init__.py | 509 +++----------- freqtrade/optimize/backslapping.py | 808 ---------------------- freqtrade/optimize/edge.py | 1037 ---------------------------- 3 files changed, 109 insertions(+), 2245 deletions(-) delete mode 100644 freqtrade/optimize/backslapping.py delete mode 100644 freqtrade/optimize/edge.py diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7a45b7b78..7a0b715ab 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -16,9 +16,7 @@ from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting import numpy as np -import timeit import utils_find_1st as utf1st -import pdb logger = logging.getLogger(__name__) @@ -60,40 +58,6 @@ class Edge(): self.fee = self.exchange.get_fee() - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - def calculate(self) -> bool: pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] @@ -102,7 +66,6 @@ class Edge(): return False data = {} - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') @@ -141,7 +104,7 @@ class Edge(): stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] + trades: list = [] for pair, pair_data in preprocessed.items(): # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) @@ -150,33 +113,44 @@ class Edge(): ticker_data = self.populate_sell_trend( self.populate_buy_trend(pair_data))[headers].copy() - # call backslap - results are a list of dicts - for stoploss in stoploss_range: - bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 6)) + trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) + - # Switch List of Trade Dicts (bslap_results) to Dataframe + # Switch List of Trade Dicts (trades) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) + trades_df = DataFrame(trades) - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - bslap_results_df = self.vector_fill_results_table(bslap_results_df) + if len(trades_df) > 0: # Only post process a frame if it has a record + trades_df = self._fill_calculable_fields(trades_df) else: - bslap_results_df = [] - bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) + trades_df = [] + trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) - self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range) + + self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp return True + def stake_amount(self, pair: str) -> str: + info = [x for x in self._cached_pairs if x[0] == pair][0] + stoploss = info[1] + allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) + position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) + return position_size + + def stoploss(self, pair: str) -> float: + info = [x for x in self._cached_pairs if x[0] == pair][0] + return info[1] + def sort_pairs(self, pairs) -> bool: if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] - def vector_fill_results_table(self, bslap_results_df: DataFrame): + def _fill_calculable_fields(self, result: DataFrame): """ - The Results frame contains a number of columns that are calculable + The result frame contains a number of columns that are calculable from othe columns. These are left blank till all rows are added, to be populated in single vector calls. @@ -184,8 +158,8 @@ class Edge(): - Profit - trade duration - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe + :param result Dataframe + :return: result Dataframe """ # stake and fees @@ -198,63 +172,30 @@ class Edge(): open_fee = fee / 2 close_fee = fee / 2 - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + result['trade_duration'] = result['close_time'] - result['open_time'] + result['trade_duration'] = result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) + # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending + result['buy_vol'] = stake / result['open_rate'] # How many target are we buying + result['buy_fee'] = stake * open_fee + result['buy_spend'] = stake + result['buy_fee'] # How much we're spending # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] + result['sell_sum'] = result['buy_vol'] * result['close_rate'] + result['sell_fee'] = result['sell_sum'] * close_fee + result['sell_take'] = result['sell_sum'] - result['sell_fee'] + # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) \ + / result['buy_spend'] # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] + result['profit_abs'] = result['sell_take'] - result['buy_spend'] - return bslap_results_df + return result - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, - stop_stops_count: int): - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - stop_stops i - """ - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - - return t_open_ind - - def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: + def _process_expectancy(self, results: DataFrame) -> str: """ This is a temporary version of edge positioning calculation. The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and @@ -262,10 +203,6 @@ class Edge(): The calulation will be done per pair and per strategy. """ - # Removing open trades from dataset - results = results[results.open_at_end == False] - ################################### - # Removing pairs having less than min_trades_number min_trades_number = self.edge_config.get('min_trade_number', 15) results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) @@ -335,313 +272,85 @@ class Edge(): # Returning an array of pairs in order of "expectancy" return final.reset_index().values - def backslap_pair(self, ticker_data, pair, stoploss): - # Read Stop Loss Values and Stake - stop = stoploss - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price + def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): + buy_column = ticker_data['buy'].values + sell_column = ticker_data['sell'].values + date_column = ticker_data['date'].values + ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values + + result: list = [] + for stoploss in stoploss_range: + result += self._detect_stop_and_sell_points(buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair) - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' + return result - ####### - # Use vars set at top of backtest - np_open: int = self.np_open - np_sell: int = self.np_sell - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close + def _detect_stop_and_sell_points(self, buy_column, sell_column, date_column, ohlc_columns, stoploss, pair, start_point=0): + result: list = [] + open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) + #open_trade_index = np.argmax(buy_column == 1) - # ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data + # return empty if we don't find trade entry (i.e. buy==1) + if open_trade_index == -1: + return [] - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) + stop_price_percentage = stoploss + 1 + open_price = ohlc_columns[open_trade_index + 1, 0] + stop_price = (open_price * stop_price_percentage) - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) + # Searching for the index where stoploss is hit + stop_index = utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values + # If we don't find it then we assume stop_index will be far in future (infinite number) + if stop_index == -1: + stop_index = float('inf') - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit + #stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair + # Searching for the index where sell is hit + sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} + # If we don't find it then we assume sell_index will be far in future (infinite number) + if sell_index == -1: + sell_index = float('inf') - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit + #sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' + # Check if we don't find any stop or sell point (in that case trade remains open) + # It is not interesting for Edge to consider it so we simply ignore the trade + # And stop iterating as the party is over + if stop_index == sell_index == float('inf'): + return [] - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag + if stop_index <= sell_index: + exit_index = open_trade_index + stop_index + 1 + exit_type = SellType.STOP_LOSS + exit_price = stop_price + elif stop_index > sell_index: + exit_index = open_trade_index + sell_index + 1 + exit_type = SellType.SELL_SIGNAL + exit_price = ohlc_columns[open_trade_index + sell_index + 1, 0] - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind + trade = {} + trade["pair"] = pair + trade["stoploss"] = stoploss + trade["profit_percent"] = "" # To be 1 vector calculation across trades when loop complete + trade["profit_abs"] = "" # To be 1 vector calculation across trades when loop complete + trade["open_time"] = date_column[open_trade_index] + trade["close_time"] = date_column[exit_index] + trade["open_index"] = start_point + open_trade_index + 1 # +1 as we buy on next. + trade["close_index"] = start_point + exit_index + trade["trade_duration"] = "" # To be 1 vector calculation across trades when loop complete + trade["open_rate"] = round(open_price, 15) + trade["close_rate"] = round(exit_price, 15) + trade["exit_type"] = exit_type + result.append(trade) - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind + 1:] - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind + 1 - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index - t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = SellType.NONE - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == SellType.STOP_LOSS: - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - t_exit_ind = t_exit_ind + 1 - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == SellType.SELL_SIGNAL: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == SellType.NONE: - np_trade_exit_price = 0 - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - - # Loop control - catch no closed trades. - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - - # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE - # removing the +1 here so prices match. - if t_exit_type == SellType.STOP_LOSS: - t_exit_ind = t_exit_ind - 1 - - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["stoploss"] = stop - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is SellType.STOP_LOSS: - stop_stops_count = stop_stops_count + 1 - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - # Send back List of trade dicts - return bslap_pair_results - - def stake_amount(self, pair: str) -> str: - info = [x for x in self._cached_pairs if x[0] == pair][0] - stoploss = info[1] - allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) - position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) - return position_size - - def stoploss(self, pair: str) -> float: - info = [x for x in self._cached_pairs if x[0] == pair][0] - return info[1] \ No newline at end of file + return result + self._detect_stop_and_sell_points( + buy_column[exit_index:], + sell_column[exit_index:], + date_column[exit_index:], + ohlc_columns[exit_index:], + stoploss, + pair, + (start_point + exit_index) + ) diff --git a/freqtrade/optimize/backslapping.py b/freqtrade/optimize/backslapping.py deleted file mode 100644 index 7eba39a6e..000000000 --- a/freqtrade/optimize/backslapping.py +++ /dev/null @@ -1,808 +0,0 @@ -import timeit -from typing import Dict, Any - -from pandas import DataFrame - -from freqtrade.exchange import Exchange -from freqtrade.strategy import IStrategy -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import StrategyResolver -import pdb - -class Backslapping: - """ - provides a quick way to evaluate strategies over a longer term of time - """ - - def __init__(self, config: Dict[str, Any], exchange = None) -> None: - """ - constructor - """ - - self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend - - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange - - self.fee = self.exchange.get_fee() - - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - - def run(self,args): - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - trades = [] - trade_count_lock: Dict = {} - - ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] - for pair, pair_data in processed.items(): - if self.debug_timing: # Start timer - fl = self.s() - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() - - if self.debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() - - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') - - # call bslap - results are a list of dicts - bslap_pair_results = self.backslap_pair(ticker_data, pair) - last_bslap_results = bslap_results - bslap_results = last_bslap_results + bslap_pair_results - - if self.debug_timing: # print time taken - tt = self.f(st) - print("Time to BackSlap :", pair, round(tt, 10)) - print("-----------------------") - - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - # if debug: - # print("open_time and close_time converted to datetime columns") - - bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair) - else: - from freqtrade.optimize.backtesting import BacktestResult - - bslap_results_df = [] - bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - - return bslap_results_df - - def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str): - """ - The Results frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe - """ - import pandas as pd - import numpy as np - debug = self.debug_vector - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - stake = self.config.get('stake_amount') - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - if debug: - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 20) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - # # Get before - # csv = "cryptosher_before_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) - - ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) - # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending - - # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] - # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] - # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - - # # Get After - # csv="cryptosher_after_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - if debug: - print("\n") - print(bslap_results_df[ - ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', - 'profit_abs', 'exit_type']]) - - #pdb.set_trace() - return bslap_results_df - - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, - stop_stops_count: int): - import utils_find_1st as utf1st - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - - stop_stops i - """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - st = s() - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") - - return t_open_ind - - def backslap_pair(self, ticker_data, pair): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - #debug = self.debug # print values, to check accuracy - debug = False - - ticker_data = ticker_data.sort_values(by=['date']) - ticker_data = ticker_data.reset_index(drop=True) - - #pandas_df = df.toPandas() - #pandas_df.to_json - - # Read Stop Loss Values and Stake - # pdb.set_trace() - stop = self.stop_loss_value - #stop = stoploss - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - - ####### - # Use vars set at top of backtest - np_buy: int = self.np_buy - np_open: int = self.np_open - np_close: int = self.np_close - np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - - ### End Config - - pair: str = pair - - # ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data - - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) - - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) - - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values - - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit - - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair - - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} - - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print( - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit - - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - if debug_timing: - st = s() - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag - - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind - - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind + 1:] - - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind + 1 - - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", - np_t_stop_ind - 1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print( - "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index - t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = SellType.NONE - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == SellType.STOP_LOSS: - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - t_exit_ind = t_exit_ind + 1 - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == SellType.SELL_SIGNAL: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == SellType.NONE: - np_trade_exit_price = 0 - - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) - - # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - - - if debug: - print(bslap_pair_results) - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - - # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE - # removing the +1 here so prices match. - if t_exit_type == SellType.STOP_LOSS: - t_exit_ind = t_exit_ind - 1 - - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["stoploss"] = stop - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is SellType.STOP_LOSS: - stop_stops_count = stop_stops_count + 1 - - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - - # Send back List of trade dicts - return bslap_pair_results \ No newline at end of file diff --git a/freqtrade/optimize/edge.py b/freqtrade/optimize/edge.py deleted file mode 100644 index efafb8d77..000000000 --- a/freqtrade/optimize/edge.py +++ /dev/null @@ -1,1037 +0,0 @@ -# pragma pylint: disable=missing-docstring, W0212, too-many-arguments - -""" -This module contains the backtesting logic -""" -import logging -import operator -import sys -from argparse import Namespace -from datetime import datetime, timedelta -from typing import Any, Dict, List, NamedTuple, Optional, Tuple - -import arrow -from pandas import DataFrame, to_datetime -from tabulate import tabulate -import numpy as np - -import freqtrade.optimize as optimize -from freqtrade import DependencyException, constants -from freqtrade.arguments import Arguments -from freqtrade.configuration import Configuration -from freqtrade.exchange import Exchange -from freqtrade.misc import file_dump_json -from freqtrade.persistence import Trade -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import OrderedDict -import timeit -from time import sleep - -import pdb - -logger = logging.getLogger(__name__) - -class Edge: - """ - provides a quick way to evaluate strategies over a longer term of time - """ - - def __init__(self, config: Dict[str, Any], exchange = None) -> None: - """ - constructor - """ - self.config = config - self.strategy: IStrategy = StrategyResolver(self.config).strategy - self.ticker_interval = self.strategy.ticker_interval - self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend - - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange - - self.fee = self.exchange.get_fee() - - self.stop_loss_value = self.strategy.stoploss - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - self.np_buy: int = 0 - self.np_open: int = 1 - self.np_close: int = 2 - self.np_sell: int = 3 - self.np_high: int = 4 - self.np_low: int = 5 - self.np_stop: int = 6 - self.np_bto: int = self.np_close # buys_triggered_on - should be close - self.np_bco: int = self.np_open # buys calculated on - open of the next candle. - self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close - self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close - # self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close - # self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close - - self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended - self.debug_timing = False # Stages within Backslap - self.debug_2loops = False # Limit each pair to two loops, useful when debugging - self.debug_vector = False # Debug vector calcs - self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap - - self.backslap_show_trades = False # prints trades in addition to summary report - self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt - - self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit - - @staticmethod - def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum timeframe for the given backtest data - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeframe = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return min(timeframe, key=operator.itemgetter(0))[0], \ - max(timeframe, key=operator.itemgetter(1))[1] - - def s(self): - st = timeit.default_timer() - return st - - def f(self, st): - return (timeit.default_timer() - st) - - def run(self,args): - - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - processed = args['processed'] - max_open_trades = args.get('max_open_trades', 0) - realistic = args.get('realistic', False) - stoploss_range = args['stoploss_range'] - trades = [] - trade_count_lock: Dict = {} - - ########################### Call out BSlap Loop instead of Original BT code - bslap_results: list = [] - for pair, pair_data in processed.items(): - if self.debug_timing: # Start timer - fl = self.s() - - # Sorting dataframe by date and reset index - pair_data = pair_data.sort_values(by=['date']) - pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() - - if self.debug_timing: # print time taken - flt = self.f(fl) - # print("populate_buy_trend:", pair, round(flt, 10)) - st = self.s() - - # #dump same DFs to disk for offline testing in scratch - # f_pair:str = pair - # csv = f_pair.replace("/", "_") - # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv - # ticker_data.to_csv(csv, sep='\t', encoding='utf-8') - - # call bslap - results are a list of dicts - for stoploss in stoploss_range: - bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3)) - - - if self.debug_timing: # print time taken - tt = self.f(st) - print("Time to BackSlap :", pair, round(tt, 10)) - print("-----------------------") - - # Switch List of Trade Dicts (bslap_results) to Dataframe - # Fill missing, calculable columns, profit, duration , abs etc. - bslap_results_df = DataFrame(bslap_results) - - if len(bslap_results_df) > 0: # Only post process a frame if it has a record - # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time']) - # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time']) - # if debug: - # print("open_time and close_time converted to datetime columns") - - bslap_results_df = self.vector_fill_results_table(bslap_results_df) - else: - from freqtrade.optimize.backtesting import BacktestResult - - bslap_results_df = [] - bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields) - - return bslap_results_df - - def vector_fill_results_table(self, bslap_results_df: DataFrame): - """ - The Results frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param bslap_results Dataframe - :return: bslap_results Dataframe - """ - import pandas as pd - import numpy as np - debug = self.debug_vector - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - stake = self.config.get('stake_amount') - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - if debug: - print("Stake is,", stake, "the sum of currency to spend per trade") - print("The open fee is", open_fee, "The close fee is", close_fee) - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 20) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - # # Get before - # csv = "cryptosher_before_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time'] - bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) - - ## Spends, Takes, Profit, Absolute Profit - # print(bslap_results_df) - # Buy Price - bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying - bslap_results_df['buy_fee'] = stake * open_fee - bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending - - # Sell price - bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate'] - bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee - bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee'] - # profit_percent - bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \ - / bslap_results_df['buy_spend'] - # Absolute profit - bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend'] - - # # Get After - # csv="cryptosher_after_debug" - # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8') - - if debug: - print("\n") - print(bslap_results_df[ - ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent', - 'profit_abs', 'exit_type']]) - - #pdb.set_trace() - return bslap_results_df - - def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int, - stop_stops_count: int): - import utils_find_1st as utf1st - """ - The purpose of this def is to return the next "buy" = 1 - after t_exit_ind. - - This function will also check is the stop limit for the pair has been reached. - if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit. - - t_exit_ind is the index the last trade exited on - or 0 if first time around this loop. - - stop_stops i - """ - debug = self.debug - - # Timers, to be called if in debug - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - st = s() - t_open_ind: int - - """ - Create a view on our buy index starting after last trade exit - Search for next buy - """ - np_buy_arr_v = np_buy_arr[t_exit_ind:] - t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal) - - ''' - If -1 is returned no buy has been found, preserve the value - ''' - if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index - t_open_ind = t_open_ind + t_exit_ind # Align numpy index - - if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use - t_open_ind = -1 # -1 ends the loop - - if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop - t_open_ind = -1 # -1 ends the loop - if debug: - print("Max stop limit ", stop_stops, "reached. Moving to next pair") - - return t_open_ind - - def backslap_pair(self, ticker_data, pair, stoploss): - import pandas as pd - import numpy as np - import timeit - import utils_find_1st as utf1st - from datetime import datetime - - ### backslap debug wrap - # debug_2loops = False # only loop twice, for faster debug - # debug_timing = False # print timing for each step - # debug = False # print values, to check accuracy - debug_2loops = self.debug_2loops # only loop twice, for faster debug - debug_timing = self.debug_timing # print timing for each step - debug = self.debug # print values, to check accuracy - - # Read Stop Loss Values and Stake - # pdb.set_trace() - #stop = self.stop_loss_value - stop = stoploss - p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price - - if debug: - print("Stop is ", stop, "value from stragey file") - print("p_stop is", p_stop, "value used to multiply to entry price") - - if debug: - from pandas import set_option - set_option('display.max_rows', 5000) - set_option('display.max_columns', 8) - pd.set_option('display.width', 1000) - pd.set_option('max_colwidth', 40) - pd.set_option('precision', 12) - - def s(): - st = timeit.default_timer() - return st - - def f(st): - return (timeit.default_timer() - st) - - #### backslap config - ''' - Numpy arrays are used for 100x speed up - We requires setting Int values for - buy stop triggers and stop calculated on - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 - ''' - - ####### - # Use vars set at top of backtest - np_buy: int = self.np_buy - np_open: int = self.np_open - np_close: int = self.np_close - np_sell: int = self.np_sell - np_high: int = self.np_high - np_low: int = self.np_low - np_stop: int = self.np_stop - np_bto: int = self.np_bto # buys_triggered_on - should be close - np_bco: int = self.np_bco # buys calculated on - open of the next candle. - np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close - np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close - - ### End Config - - pair: str = pair - - # ticker_data: DataFrame = ticker_dfs[t_file] - bslap: DataFrame = ticker_data - - # Build a single dimension numpy array from "buy" index for faster search - # (500x faster than pandas) - np_buy_arr = bslap['buy'].values - np_buy_arr_len: int = len(np_buy_arr) - - # use numpy array for faster searches in loop, 20x faster than pandas - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']]) - - # Build a numpy list of date-times. - # We use these when building the trade - # The rationale is to address a value from a pandas cell is thousands of - # times more expensive. Processing time went X25 when trying to use any data from pandas - np_bslap_dates = bslap['date'].values - - loop: int = 0 # how many time around the loop - t_exit_ind = 0 # Start loop from first index - t_exit_last = 0 # To test for exit - - stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at - stop_stops_count = 0 # stop counter per pair - - st = s() # Start timer for processing dataframe - if debug: - print('Processing:', pair) - - # Results will be stored in a list of dicts - bslap_pair_results: list = [] - bslap_result: dict = {} - - while t_exit_ind < np_buy_arr_len: - loop = loop + 1 - if debug or debug_timing: - print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair) - if debug_2loops: - if loop == 3: - print( - "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking") - break - ''' - Dev phases - Phase 1 - 1) Manage buy, sell, stop enter/exit - a) Find first buy index - b) Discover first stop and sell hit after buy index - c) Chose first instance as trade exit - - Phase 2 - 2) Manage dynamic Stop and ROI Exit - a) Create trade slice from 1 - b) search within trade slice for dynamice stop hit - c) search within trade slice for ROI hit - ''' - - if debug_timing: - st = s() - ''' - 0 - Find next buy entry - Finds index for first (buy = 1) flag - - Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1" - Required: t_exit_ind - Either 0, first loop. Or The index we last exited on - Requires: np_buy_arr_len - length of pair array. - Requires: stops_stops - number of stops allowed before stop trading a pair - Requires: stop_stop_counts - count of stops hit in the pair - Provides: The next "buy" index after t_exit_ind - - If -1 is returned no buy has been found in remainder of array, skip to exit loop - ''' - t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count) - - if debug: - print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind) - print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop") - if debug_timing: - t_t = f(st) - print("0-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if t_open_ind != -1: - - """ - 1 - Create views to search within for our open trade - - The views are our search space for the next Stop or Sell - Numpy view is employed as: - 1,000 faster than pandas searches - Pandas cannot assure it will always return a view, it may make a slow copy. - - The view contains columns: - buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - - Requires: np_bslap is our numpy array of the ticker DataFrame - Requires: t_open_ind is the index row with the buy. - Provides: np_t_open_v View of array after buy. - Provides: np_t_open_v_stop View of array after buy +1 - (Stop will search in here to prevent stopping in the past) - """ - np_t_open_v = np_bslap[t_open_ind:] - np_t_open_v_stop = np_bslap[t_open_ind + 1:] - - if debug: - print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind) - print("Numpy View: Buy - Open - Close - Sell - High - Low") - print("Row 0", np_t_open_v[0]) - print("Row 1", np_t_open_v[1], ) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 2 - Calculate our stop-loss price - - As stop is based on buy price of our trade - - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle - - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle. - This is as we only see the CLOSE after it has happened. - The back test assumption is we have bought at first available price, the OPEN - - Requires: np_bslap - is our numpy array of the ticker DataFrame - Requires: t_open_ind - is the index row with the first buy. - Requires: p_stop - is the stop rate, ie. 0.99 is -1% - Provides: np_t_stop_pri - The value stop-loss will be triggered on - ''' - np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop) - - if debug: - print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri) - if debug_timing: - t_t = f(st) - print("2-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 3 - Find candle STO is under Stop-Loss After Trade opened. - - where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri - - Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened) - Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close" - Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop - Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss - ''' - np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto], - np_t_stop_pri, - utf1st.cmp_smaller) - - # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out. - np_t_stop_ind = np_t_stop_ind + 1 - - if debug: - print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is", - np_t_stop_ind - 1, - ". STO is using field", np_sto, - "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n") - - print( - "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage") - print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind]) - print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1]) - if debug_timing: - t_t = f(st) - print("3-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 4 - Find first sell index after trade open - - First index in the view np_t_open_v where ['sell'] = 1 - - Requires: np_t_open_v - view of ticker_data from buy onwards - Requires: no_sell - integer '3', the buy column in the array - Provides: np_t_sell_ind index of view where first sell=1 after buy - ''' - # Use numpy array for faster search for sell - # Sell uses column 3. - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # Numpy searches 25-35x quicker than pandas on this data - - np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell], - 1, utf1st.cmp_equal) - if debug: - print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind) - print("If 0 or less is returned there is no sell found to end of view, then next lines garbage") - print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind]) - print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1]) - if debug_timing: - t_t = f(st) - print("4-numpy", str.format('{0:.17f}', t_t)) - st = s() - - ''' - 5 - Determine which was hit first a stop or sell - To then use as exit index price-field (sell on buy, stop on stop) - - STOP takes priority over SELL as would be 'in candle' from tick data - Sell would use Open from Next candle. - So in a draw Stop would be hit first on ticker data in live - - Validity of when types of trades may be executed can be summarised as: - - Tick View - index index Buy Sell open low close high Stop price - open 2am 94 -1 0 0 ----- ------ ------ ----- ----- - open 3am 95 0 1 0 ----- ------ trg buy ----- ----- - open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out - open 5am 97 2 0 0 Exit ------ ------- ----- ----- - open 6am 98 3 0 0 ----- ------ ------- ----- ----- - - -1 means not found till end of view i.e no valid Stop found. Exclude from match. - Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid. - - Buys and sells are triggered at candle close - Both will open their postions at the open of the next candle. i/e + 1 index - - Stop and buy Indexes are on the view. To map to the ticker dataframe - the t_open_ind index should be summed. - - np_t_stop_ind: Stop Found index in view - t_exit_ind : Sell found in view - t_open_ind : Where view was started on ticker_data - - TODO: fix this frig for logic test,, case/switch/dictionary would be better... - more so when later testing many options, dynamic stop / roi etc - cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found) - cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found) - - ''' - if debug: - print("\n(5) numpy debug\nStop or Sell Logic Processing") - - # cludge for logic test (-1) means it was not found, set crazy high to lose < test - np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind - np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind - - # Stoploss trigger found before a sell =1 - if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind: - t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index - t_exit_type = SellType.STOP_LOSS # Set Exit type (stop) - np_t_exit_pri = np_sco # The price field our STOP exit will use - if debug: - print("Type STOP is first exit condition. " - "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind) - - # Buy = 1 found before a stoploss triggered - elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind: - # move sell onto next candle, we only look back on sell - # will use the open price later. - t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index - t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell) - np_t_exit_pri = np_open # The price field our SELL exit will use - if debug: - print("Type SELL is first exit condition. " - "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind) - - # No stop or buy left in view - set t_exit_last -1 to handle gracefully - else: - t_exit_last: int = -1 # Signal loop to exit, no buys or sells found. - t_exit_type = SellType.NONE - np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column - if debug: - print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit") - - # TODO: fix having to cludge/uncludge this .. - # Undo cludge - np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind - np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind - - if debug_timing: - t_t = f(st) - print("5-logic", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - ''' - Print out the buys, stops, sells - Include Line before and after to for easy - Human verification - ''' - # Combine the np_t_stop_pri value to bslap dataframe to make debug - # life easy. This is the current stop price based on buy price_ - # This is slow but don't care about performance in debug - # - # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1 - # as there is no data column in the numpy array. - bslap['np_stop_pri'] = np_t_stop_pri - - # Buy - print("\n\nDATAFRAME DEBUG =================== BUY ", pair) - print("Numpy Array BUY Index is:", 0) - print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n") - print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1) - op_is = t_open_ind - 1 # Print open index start, line before - op_if = t_open_ind + 3 # Print open index finish, line after - print(bslap.iloc[op_is:op_if], "\n") - - # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On) - if np_t_stop_ind < 0: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("No STOPS were found until the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== STOP ", pair) - print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind) - df_stop_index = (t_open_ind + np_t_stop_ind) - - print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n") - print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ", - df_stop_index, ": \n", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]), - "is less than", str.format('{0:.17f}', np_t_stop_pri)) - - print("A stoploss exit will be calculated at rate:", - str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1])) - - print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index, - ": As live STOPs are not linked to O-C times") - - st_is = df_stop_index - 1 # Print stop index start, line before - st_if = df_stop_index + 2 # Print stop index finish, line after - print(bslap.iloc[st_is:st_if], "\n") - - # Sell - if np_t_sell_ind < 0: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("No SELLS were found till the end of ticker data file\n") - else: - print("DATAFRAME DEBUG =================== SELL ", pair) - print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind) - df_sell_index = (t_open_ind + np_t_sell_ind) - - print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n") - print("First Sell Index after Trade open is in candle", df_sell_index) - print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle", - df_sell_index + 1) - sl_is = df_sell_index - 1 # Print sell index start, line before - sl_if = df_sell_index + 3 # Print sell index finish, line after - print(bslap.iloc[sl_is:sl_if], "\n") - - # Chosen Exit (stop or sell) - - print("DATAFRAME DEBUG =================== EXIT ", pair) - print("Exit type is :", t_exit_type) - print("trade exit price field is", np_t_exit_pri, "\n") - - if debug_timing: - t_t = f(st) - print("6-depra", str.format('{0:.17f}', t_t)) - st = s() - - ## use numpy view "np_t_open_v" for speed. Columns are - # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - # exception is 6 which is use the stop value. - - # TODO no! this is hard coded bleh fix this open - np_trade_enter_price = np_bslap[t_open_ind + 1, np_open] - if t_exit_type == SellType.STOP_LOSS: - if np_t_exit_pri == 6: - np_trade_exit_price = np_t_stop_pri - t_exit_ind = t_exit_ind + 1 - else: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - if t_exit_type == SellType.SELL_SIGNAL: - np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri] - - # Catch no exit found - if t_exit_type == SellType.NONE: - np_trade_exit_price = 0 - - if debug_timing: - t_t = f(st) - print("7-numpy", str.format('{0:.17f}', t_t)) - st = s() - - if debug: - print("//////////////////////////////////////////////") - print("+++++++++++++++++++++++++++++++++ Trade Enter ") - print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price)) - print("--------------------------------- Trade Exit ") - print("Trade Exit Type is ", t_exit_type) - print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price)) - print("//////////////////////////////////////////////") - - else: # no buys were found, step 0 returned -1 - # Gracefully exit the loop - t_exit_last == -1 - if debug: - print("\n(E) No buys were found in remaining ticker file. Exiting", pair) - - # Loop control - catch no closed trades. - if debug: - print("---------------------------------------- end of loop", loop, - " Dataframe Exit Index is: ", t_exit_ind) - print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind) - - if t_exit_last >= t_exit_ind or t_exit_last == -1: - """ - Break loop and go on to next pair. - - When last trade exit equals index of last exit, there is no - opportunity to close any more trades. - """ - # TODO :add handing here to record none closed open trades - - if debug: - print(bslap_pair_results) - break - else: - """ - Add trade to backtest looking results list of dicts - Loop back to look for more trades. - """ - - # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE - # removing the +1 here so prices match. - if t_exit_type == SellType.STOP_LOSS: - t_exit_ind = t_exit_ind - 1 - - # Build trade dictionary - ## In general if a field can be calculated later from other fields leave blank here - ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time - - # create a new dict - close_index: int = t_exit_ind - bslap_result = {} # Must have at start or we end up with a list of multiple same last result - bslap_result["pair"] = pair - bslap_result["stoploss"] = stop - bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower - bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower - bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next. - bslap_result["close_index"] = close_index - bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete - bslap_result["open_at_end"] = False - bslap_result["open_rate"] = round(np_trade_enter_price, 15) - bslap_result["close_rate"] = round(np_trade_exit_price, 15) - bslap_result["exit_type"] = t_exit_type - bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care - # append the dict to the list and print list - bslap_pair_results.append(bslap_result) - - if t_exit_type is SellType.STOP_LOSS: - stop_stops_count = stop_stops_count + 1 - - if debug: - print("The trade dict is: \n", bslap_result) - print("Trades dicts in list after append are: \n ", bslap_pair_results) - - """ - Loop back to start. t_exit_last becomes where loop - will seek to open new trades from. - Push index on 1 to not open on close - """ - t_exit_last = t_exit_ind + 1 - - if debug_timing: - t_t = f(st) - print("8+trade", str.format('{0:.17f}', t_t)) - - # Send back List of trade dicts - return bslap_pair_results - - def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str: - """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. - The calulation will be done per pair and per strategy. - """ - - # Removing open trades from dataset - results = results[results.open_at_end == False] - ################################### - - # Removing pairs having less than min_trades_number - min_trades_number = 50 - results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) - ################################### - - - # Removing outliers (Only Pumps) from the dataset - # The method to detect outliers is to calculate standard deviation - # Then every value more than (standard deviation + 2*average) is out (pump) - # - # Calculating standard deviation of profits - std = results[["profit_abs"]].std() - # - # Calculating average of profits - avg = results[["profit_abs"]].mean() - # - # Removing Pumps - results = results[results.profit_abs < float(avg + 2*std)] - ########################################################################## - - # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = 24*60 - results = results[results.trade_duration < max_trade_duration] - ####################################################################### - - - # Win Rate is the number of profitable trades - # Divided by number of trades - def winrate(x): - x = x[x > 0].count() / x.count() - return x - ############################# - - # Risk Reward Ratio - # 1 / ((loss money / losing trades) / (gained money / winning trades)) - def risk_reward_ratio(x): - x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) - return x - ############################## - - # Required Risk Reward - # (1/(winrate - 1) - def required_risk_reward(x): - x = (1/(x[x > 0].count()/x.count()) -1) - return x - ############################## - - # The difference between risk reward ratio and required risk reward - # We use it as an indicator to find the most interesting pair to trade - def delta(x): - x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) - return x - ############################## - - - final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\ - reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\ - .groupby('pair').first().sort_values(by=['delta'], ascending=False) - - pdb.set_trace() - return final - - - def start(self) -> None: - """ - Run a backtesting end-to-end - :return: None - """ - data = {} - pairs = self.config['exchange']['pair_whitelist'] - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - - if self.config.get('live'): - logger.info('Downloading data for all pairs in whitelist ...') - for pair in pairs: - data[pair] = self.exchange.get_ticker_history(pair, self.ticker_interval) - else: - logger.info('Using local backtesting data (using whitelist in given config) ...') - - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) - - data = optimize.load_data( - self.config['datadir'], - pairs=pairs, - ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - exchange=self.exchange, - timerange=timerange - ) - - if not data: - logger.critical("No data found. Terminating.") - return - - # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - max_open_trades = self.config['max_open_trades'] - else: - logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 - - preprocessed = self.tickerdata_to_dataframe(data) - - # Print timeframe - min_date, max_date = self.get_timeframe(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days)..', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - - stoploss_range = np.arange(-0.11, -0.00, 0.01) - - # Execute backtest and print results - results = self.run( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'stoploss_range': stoploss_range, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - } - ) - - logger.info( - '\n====================================================== ' - 'Edge positionning REPORT' - ' =======================================================\n' - '%s', - self._process_result( - data, - results, - stoploss_range - ) - ) - - -def setup_configuration(args: Namespace) -> Dict[str, Any]: - """ - Prepare the configuration for the backtesting - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['backslap'] = args.backslap - if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: - raise DependencyException('stake amount could not be "%s" for backtesting' % - constants.UNLIMITED_STAKE_AMOUNT) - - return config - -# Initialize configuration -arguments = Arguments( - sys.argv[1:], - 'Simple High Frequency Trading Bot for crypto currencies' - ) -args = arguments.get_parsed_arg() - -config = setup_configuration(args) - -edge = Edge(config) -edge.start() - -allowed_dollars_at_risk = total_capital * allowed_risk -print("allowed capital at risk ", round(allowed_dollars_at_risk, 5)) - -position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss) -print("position_size in dollars", round(position_size, 5 )) - -buy_amount = position_size / bid_price -print("amount of tokens to buy ", round(buy_amount,5)) - -check_risk = (buy_amount * (bid_price - stop_trigger_price)) -print("check risk capital ", round(check_risk, 5), "** Should not be more than allowed capital at risk") \ No newline at end of file From fcf837bfda77192924417e351ab64c987fd5b4a1 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 16:03:51 +0200 Subject: [PATCH 064/174] refactoring variable declaration --- freqtrade/edge/__init__.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7a0b715ab..7cf2e62de 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -40,7 +40,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._last_updated = None - self._cached_pairs = [] + self._cached_pairs : list = [] self._total_capital = self.edge_config['total_capital_in_stake_currency'] self._allowed_risk = self.edge_config['allowed_risk'] @@ -65,7 +65,7 @@ class Edge(): if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)): return False - data = {} + data: Dict[str, Any] = {} logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') @@ -84,7 +84,7 @@ class Edge(): if not data: logger.critical("No data found. Edge is stopped ...") - return + return False preprocessed = self.tickerdata_to_dataframe(data) @@ -142,7 +142,7 @@ class Edge(): info = [x for x in self._cached_pairs if x[0] == pair][0] return info[1] - def sort_pairs(self, pairs) -> bool: + def sort_pairs(self, pairs) -> list: if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] @@ -195,7 +195,7 @@ class Edge(): return result - def _process_expectancy(self, results: DataFrame) -> str: + def _process_expectancy(self, results: DataFrame) -> list: """ This is a temporary version of edge positioning calculation. The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and @@ -330,19 +330,20 @@ class Edge(): exit_type = SellType.SELL_SIGNAL exit_price = ohlc_columns[open_trade_index + sell_index + 1, 0] - trade = {} - trade["pair"] = pair - trade["stoploss"] = stoploss - trade["profit_percent"] = "" # To be 1 vector calculation across trades when loop complete - trade["profit_abs"] = "" # To be 1 vector calculation across trades when loop complete - trade["open_time"] = date_column[open_trade_index] - trade["close_time"] = date_column[exit_index] - trade["open_index"] = start_point + open_trade_index + 1 # +1 as we buy on next. - trade["close_index"] = start_point + exit_index - trade["trade_duration"] = "" # To be 1 vector calculation across trades when loop complete - trade["open_rate"] = round(open_price, 15) - trade["close_rate"] = round(exit_price, 15) - trade["exit_type"] = exit_type + trade = {'pair': pair, + 'stoploss': stoploss, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': date_column[open_trade_index], + 'close_time': date_column[exit_index], + 'open_index': start_point + open_trade_index + 1, + 'close_index': start_point + exit_index, + 'trade_duration': '', + 'open_rate': round(open_price, 15), + 'close_rate': round(exit_price, 15), + 'exit_type': exit_type + } + result.append(trade) return result + self._detect_stop_and_sell_points( From 75ba6578a31e65bc25ba780fd826e2d55cd9785d Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 16:36:41 +0200 Subject: [PATCH 065/174] unused library + trailing whitespaces removed. --- freqtrade/freqtradebot.py | 17 ++++++++++++----- freqtrade/optimize/__init__.py | 1 - freqtrade/strategy/interface.py | 6 ++++-- freqtrade/tests/conftest.py | 2 +- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 60d286af1..be02db9a8 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -404,7 +404,12 @@ class FreqtradeBot(object): stake_amount = self._get_trade_stake_amount(_pair) if not stake_amount: return False - logger.info('Buy signal found: about create a new trade with stake_amount: %f ...', stake_amount) + + logger.info( + 'Buy signal found: about create a new trade with stake_amount: %f ...', + stake_amount + ) + bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ get('check_depth_of_market', {}) if (bidstrat_check_depth_of_market.get('enabled', False)) and\ @@ -444,10 +449,10 @@ class FreqtradeBot(object): pair_url = self.exchange.get_pair_detail_url(pair) stake_currency = self.config['stake_currency'] fiat_currency = self.config.get('fiat_display_currency', None) - + # Calculate amount buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair)) - + min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( @@ -630,9 +635,11 @@ class FreqtradeBot(object): def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if (self.config['edge']['enabled']): stoploss = self.edge.stoploss(trade.pair) - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + should_sell = \ + self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) else: - should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + should_sell = \ + self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 967227805..74c842427 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -16,7 +16,6 @@ import arrow from freqtrade import misc, constants, OperationalException from freqtrade.exchange import Exchange from freqtrade.arguments import TimeRange -import importlib logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 13fe01a70..73bf2313d 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -210,8 +210,10 @@ class IStrategy(ABC): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit, force_stoploss=force_stoploss) + stoplossflag = \ + self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, + current_profit=current_profit, force_stoploss=force_stoploss) + if stoplossflag.sell_flag: return stoplossflag diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index f8f7729eb..99c90d00a 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -137,7 +137,7 @@ def default_conf(): "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.001, - "maximum_winrate": 0.80, + "maximum_winrate": 0.80, "min_trade_number": 15, "max_trade_duration_minute": 1440, "remove_pumps": True, From 0594deafc6df2c82fb1792ce21074be2d53e5a94 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 16:50:17 +0200 Subject: [PATCH 066/174] removing whitespaces and long lines --- freqtrade/edge/__init__.py | 67 ++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7cf2e62de..8990c6b35 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -5,7 +5,6 @@ from typing import Any, Dict import arrow from pandas import DataFrame -import pandas as pd import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult @@ -40,7 +39,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._last_updated = None - self._cached_pairs : list = [] + self._cached_pairs: list = [] self._total_capital = self.edge_config['total_capital_in_stake_currency'] self._allowed_risk = self.edge_config['allowed_risk'] @@ -62,14 +61,15 @@ class Edge(): pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] - if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)): + if (self._last_updated is not None) and \ + (self._last_updated + heartbeat > arrow.utcnow().timestamp): return False data: Dict[str, Any] = {} logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') - #TODO: add "timerange" to Edge config + # TODO: add "timerange" to Edge config timerange = Arguments.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -103,7 +103,6 @@ class Edge(): stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) - ########################### Call out BSlap Loop instead of Original BT code trades: list = [] for pair, pair_data in preprocessed.items(): # Sorting dataframe by date and reset index @@ -114,7 +113,6 @@ class Edge(): self.populate_buy_trend(pair_data))[headers].copy() trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) - # Switch List of Trade Dicts (trades) to Dataframe # Fill missing, calculable columns, profit, duration , abs etc. @@ -126,7 +124,6 @@ class Edge(): trades_df = [] trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) - self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp return True @@ -146,7 +143,7 @@ class Edge(): if len(self._cached_pairs) == 0: self.calculate() edge_sorted_pairs = [x[0] for x in self._cached_pairs] - return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])] + return [x for _, x in sorted(zip(edge_sorted_pairs, pairs), key=lambda pair: pair[0])] def _fill_calculable_fields(self, result: DataFrame): """ @@ -173,9 +170,11 @@ class Edge(): close_fee = fee / 2 result['trade_duration'] = result['close_time'] - result['open_time'] - result['trade_duration'] = result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) - ## Spends, Takes, Profit, Absolute Profit + result['trade_duration'] = \ + result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + + # Spends, Takes, Profit, Absolute Profit # Buy Price result['buy_vol'] = stake / result['open_rate'] # How many target are we buying @@ -188,8 +187,9 @@ class Edge(): result['sell_take'] = result['sell_sum'] - result['sell_fee'] # profit_percent - result['profit_percent'] = (result['sell_take'] - result['buy_spend']) \ - / result['buy_spend'] + result['profit_percent'] = \ + (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + # Absolute profit result['profit_abs'] = result['sell_take'] - result['buy_spend'] @@ -198,8 +198,10 @@ class Edge(): def _process_expectancy(self, results: DataFrame) -> list: """ This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) and keep it in a storage. + The function will be eventually moved to a plugin called Edge in order + to calculate necessary WR, RRR and + other indictaors related to money management periodically (each X minutes) + and keep it in a storage. The calulation will be done per pair and per strategy. """ @@ -238,21 +240,17 @@ class Edge(): # Risk Reward Ratio # 1 / ((loss money / losing trades) / (gained money / winning trades)) def risk_reward_ratio(x): - x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) + x = abs(1 / ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) return x ############################## # Required Risk Reward # (1/(winrate - 1) def required_risk_reward(x): - x = (1/(x[x > 0].count()/x.count()) -1) + x = (1 / (x[x > 0].count() / x.count()) - 1) return x ############################## - def delta(x): - x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1) - return x - # Expectancy # Tells you the interest percentage you should hope # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 @@ -265,7 +263,7 @@ class Edge(): ############################## final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, expectancy, delta]).\ + agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) @@ -277,17 +275,29 @@ class Edge(): sell_column = ticker_data['sell'].values date_column = ticker_data['date'].values ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values - + result: list = [] for stoploss in stoploss_range: - result += self._detect_stop_and_sell_points(buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair) + result += self._detect_stop_and_sell_points( + buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair + ) return result - def _detect_stop_and_sell_points(self, buy_column, sell_column, date_column, ohlc_columns, stoploss, pair, start_point=0): + def _detect_stop_and_sell_points( + self, + buy_column, + sell_column, + date_column, + ohlc_columns, + stoploss, + pair, + start_point=0 + ): + result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - #open_trade_index = np.argmax(buy_column == 1) + # open_trade_index = np.argmax(buy_column == 1) # return empty if we don't find trade entry (i.e. buy==1) if open_trade_index == -1: @@ -298,13 +308,14 @@ class Edge(): stop_price = (open_price * stop_price_percentage) # Searching for the index where stoploss is hit - stop_index = utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) + stop_index = \ + utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) # If we don't find it then we assume stop_index will be far in future (infinite number) if stop_index == -1: stop_index = float('inf') - #stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) + # stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) # Searching for the index where sell is hit sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) @@ -313,7 +324,7 @@ class Edge(): if sell_index == -1: sell_index = float('inf') - #sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) + # sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) # Check if we don't find any stop or sell point (in that case trade remains open) # It is not interesting for Edge to consider it so we simply ignore the trade From 24364a56ea7dfcbef99eb50be69a334fff7fa43a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 17:03:10 +0200 Subject: [PATCH 067/174] keeping mypy happy --- freqtrade/edge/__init__.py | 10 +++++----- freqtrade/freqtradebot.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 8990c6b35..1f2633dde 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -33,8 +33,8 @@ class Edge(): self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.get_timeframe = Backtesting.get_timeframe - self.populate_buy_trend = self.strategy.populate_buy_trend - self.populate_sell_trend = self.strategy.populate_sell_trend + self.advise_sell = self.strategy.advise_sell + self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) @@ -108,9 +108,9 @@ class Edge(): # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.populate_sell_trend( - self.populate_buy_trend(pair_data))[headers].copy() + + ticker_data = self.advise_sell( + self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index be02db9a8..77e2b4915 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -338,11 +338,11 @@ class FreqtradeBot(object): if avaliable_amount < stake_amount: raise DependencyException( 'Available balance(%f %s) is lower than stake amount(%f %s)' % ( - avaliable_amount, self.config['stake_currency'], - stake_amount, self.config['stake_currency']) + float(avaliable_amount), self.config['stake_currency'], + float(stake_amount), self.config['stake_currency']) ) - return stake_amount + return float(stake_amount) def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: markets = self.exchange.get_markets() From 25d6ed319a871fbf749a6ec2c5b30fa6280ac4cf Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 26 Sep 2018 17:09:20 +0200 Subject: [PATCH 068/174] whitespace removed --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 1f2633dde..369b90ce9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -108,7 +108,7 @@ class Edge(): # Sorting dataframe by date and reset index pair_data = pair_data.sort_values(by=['date']) pair_data = pair_data.reset_index(drop=True) - + ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() From 21f5a94ecaf5dca251bc44efe53c58c5f0a5028c Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 27 Sep 2018 12:23:46 +0200 Subject: [PATCH 069/174] using autopep8 for formatting file --- freqtrade/edge/__init__.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 369b90ce9..e525b224e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -4,6 +4,8 @@ import logging from typing import Any, Dict import arrow +import numpy as np +import utils_find_1st as utf1st from pandas import DataFrame import freqtrade.optimize as optimize @@ -14,8 +16,6 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting -import numpy as np -import utils_find_1st as utf1st logger = logging.getLogger(__name__) @@ -61,8 +61,8 @@ class Edge(): pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.config['edge']['process_throttle_secs'] - if (self._last_updated is not None) and \ - (self._last_updated + heartbeat > arrow.utcnow().timestamp): + if (self._last_updated is not None) and ( + self._last_updated + heartbeat > arrow.utcnow().timestamp): return False data: Dict[str, Any] = {} @@ -78,6 +78,7 @@ class Edge(): pairs=pairs, ticker_interval=self.ticker_interval, refresh_pairs=self.config.get('refresh_pairs', False), + # refresh_pairs=True, exchange=self.exchange, timerange=timerange ) @@ -171,8 +172,8 @@ class Edge(): result['trade_duration'] = result['close_time'] - result['open_time'] - result['trade_duration'] = \ - result['trade_duration'].map(lambda x: int(x.total_seconds() / 60)) + result['trade_duration'] = result['trade_duration'].map( + lambda x: int(x.total_seconds() / 60)) # Spends, Takes, Profit, Absolute Profit @@ -187,8 +188,7 @@ class Edge(): result['sell_take'] = result['sell_sum'] - result['sell_fee'] # profit_percent - result['profit_percent'] = \ - (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] # Absolute profit result['profit_abs'] = result['sell_take'] - result['buy_spend'] @@ -222,7 +222,7 @@ class Edge(): # # Removing Pumps if self.edge_config.get('remove_pumps', True): - results = results[results.profit_abs < float(avg + 2*std)] + results = results[results.profit_abs < float(avg + 2 * std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) @@ -257,8 +257,8 @@ class Edge(): def expectancy(x): average_win = float(x[x > 0].sum() / x[x > 0].count()) average_loss = float(abs(x[x < 0].sum() / x[x < 0].count())) - winrate = float(x[x > 0].count()/x.count()) - x = ((1 + average_win/average_loss) * winrate) - 1 + winrate = float(x[x > 0].count() / x.count()) + x = ((1 + average_win / average_loss) * winrate) - 1 return x ############################## @@ -280,7 +280,7 @@ class Edge(): for stoploss in stoploss_range: result += self._detect_stop_and_sell_points( buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair - ) + ) return result @@ -292,8 +292,7 @@ class Edge(): ohlc_columns, stoploss, pair, - start_point=0 - ): + start_point=0): result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) @@ -308,8 +307,8 @@ class Edge(): stop_price = (open_price * stop_price_percentage) # Searching for the index where stoploss is hit - stop_index = \ - utf1st.find_1st(ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) + stop_index = utf1st.find_1st( + ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) # If we don't find it then we assume stop_index will be far in future (infinite number) if stop_index == -1: From 96a0fc88cbbcd446aa4db0f31d2fbb8ee48bd9b5 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 14:19:22 +0200 Subject: [PATCH 070/174] Moving Edge before refresh_pairs see comments on edge (line 129) --- freqtrade/edge/__init__.py | 16 ++++++++++++++-- freqtrade/freqtradebot.py | 9 ++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e525b224e..7d915a984 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -77,8 +77,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=self.config.get('refresh_pairs', False), - # refresh_pairs=True, + refresh_pairs=True, exchange=self.exchange, timerange=timerange ) @@ -127,6 +126,19 @@ class Edge(): self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp + + # Not a nice hack but probably simplest solution: + # When backtest load data it loads the delta between disk and exchange + # The problem is that exchange consider that recent. it is but it is incomplete (c.f. _async_get_candle_history) + # So it causes get_signal to exit cause incomplete ticker_hist + # A patch to that would be update _pairs_last_refresh_time of exchange so it will download again all pairs + # Another solution is to add new data to klines instead of reassigning it: + # self.klines[pair].update(data) instead of self.klines[pair] = data in exchange package. + # But that means indexing timestamp and having a verification so that + # there is no empty range between two timestaps (recently added and last + # one) + self.exchange._pairs_last_refresh_time = {} + return True def stake_amount(self, pair: str) -> str: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cc9343659..9ac0fd02f 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -185,13 +185,16 @@ class FreqtradeBot(object): final_list = sanitized_list[:nb_assets] if nb_assets else sanitized_list self.config['exchange']['pair_whitelist'] = final_list - # Refreshing candles - self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) - # Calculating Edge positiong + # Should be called before refresh_tickers + # Otherwise it will override cached klines in exchange + # with delta value (klines only from last refresh_pairs) if self.config['edge']['enabled']: self.edge.calculate() + # Refreshing candles + self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval) + # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From e822d5d721874eec94192a36fd598c56b9a26f50 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 14:23:39 +0200 Subject: [PATCH 071/174] upgrading py_first_1st to 1.1.2: ez_setup.py removed --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b36fa5d3..d09167155 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,8 +26,7 @@ scikit-optimize==0.5.2 #plotly==3.1.1 # find first, C search in arrays -py_find_1st==1.1.1 +py_find_1st==1.1.2 #Load ticker files 30% faster ujson==1.35 - From f15825e3a772a02194f9a509a0ca8bbac0fe31f2 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 14:28:05 +0200 Subject: [PATCH 072/174] long line broken to two --- freqtrade/edge/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 7d915a984..f9549cd90 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -129,9 +129,11 @@ class Edge(): # Not a nice hack but probably simplest solution: # When backtest load data it loads the delta between disk and exchange - # The problem is that exchange consider that recent. it is but it is incomplete (c.f. _async_get_candle_history) + # The problem is that exchange consider that recent. + # it is but it is incomplete (c.f. _async_get_candle_history) # So it causes get_signal to exit cause incomplete ticker_hist - # A patch to that would be update _pairs_last_refresh_time of exchange so it will download again all pairs + # A patch to that would be update _pairs_last_refresh_time of exchange + # so it will download again all pairs # Another solution is to add new data to klines instead of reassigning it: # self.klines[pair].update(data) instead of self.klines[pair] = data in exchange package. # But that means indexing timestamp and having a verification so that From c8d06e2b0e4db96e87f09a31a0ea07fe6dcfa900 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 16:40:34 +0200 Subject: [PATCH 073/174] filter pairs according to expectancy + bug at the end of array resolved --- config.json.example | 2 ++ freqtrade/edge/__init__.py | 18 +++++++++++------- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/conftest.py | 2 ++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/config.json.example b/config.json.example index 83e9e0d38..2c1ef4fd0 100644 --- a/config.json.example +++ b/config.json.example @@ -53,12 +53,14 @@ "edge": { "enabled": false, "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.001, "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, "min_trade_number": 15, "max_trade_duration_minute": 1440, "remove_pumps": true, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f9549cd90..9789f2991 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -154,11 +154,14 @@ class Edge(): info = [x for x in self._cached_pairs if x[0] == pair][0] return info[1] - def sort_pairs(self, pairs) -> list: - if len(self._cached_pairs) == 0: - self.calculate() - edge_sorted_pairs = [x[0] for x in self._cached_pairs] - return [x for _, x in sorted(zip(edge_sorted_pairs, pairs), key=lambda pair: pair[0])] + def filter(self, pairs) -> list: + # Filtering pairs acccording to the expectancy + filtered_expectancy: list = [] + filtered_expectancy = [x[0] for x in self._cached_pairs if x[5] > float(self.edge_config.get('minimum_expectancy', 0.2))] + + # Only return pairs which are included in "pairs" argument list + final = [x for x in filtered_expectancy if x in pairs] + return final def _fill_calculable_fields(self, result: DataFrame): """ @@ -312,8 +315,9 @@ class Edge(): open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) # open_trade_index = np.argmax(buy_column == 1) - # return empty if we don't find trade entry (i.e. buy==1) - if open_trade_index == -1: + # return empty if we don't find trade entry (i.e. buy==1) or + # we find a buy but at the of array + if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: return [] stop_price_percentage = stoploss + 1 diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 9ac0fd02f..299954e49 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -399,7 +399,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals if self.config['edge']['enabled']: - whitelist = self.edge.sort_pairs(whitelist) + whitelist = self.edge.filter(whitelist) for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 99c90d00a..c72cc5a40 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -132,12 +132,14 @@ def default_conf(): "edge": { "enabled": False, "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.001, "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, "min_trade_number": 15, "max_trade_duration_minute": 1440, "remove_pumps": True, From cff83d3e6f2ff31ab7a4f4c17adfc6c2e05f7f82 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 28 Sep 2018 16:46:42 +0200 Subject: [PATCH 074/174] bloody autopep8 again --- freqtrade/edge/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 9789f2991..d675d1659 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -157,7 +157,10 @@ class Edge(): def filter(self, pairs) -> list: # Filtering pairs acccording to the expectancy filtered_expectancy: list = [] - filtered_expectancy = [x[0] for x in self._cached_pairs if x[5] > float(self.edge_config.get('minimum_expectancy', 0.2))] + filtered_expectancy = [ + x[0] for x in self._cached_pairs if x[5] > float( + self.edge_config.get( + 'minimum_expectancy', 0.2))] # Only return pairs which are included in "pairs" argument list final = [x for x in filtered_expectancy if x in pairs] From 2a9ca9a3dc43adf14cca2861ca04ea295d25142f Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:09:08 +0200 Subject: [PATCH 075/174] Removing future from travis and dockerfile --- .travis.yml | 1 - Dockerfile | 1 - 2 files changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c81c96460..981eedcf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ install: - ./install_ta-lib.sh - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - pip install --upgrade flake8 coveralls pytest-random-order pytest-asyncio mypy -- pip install future - pip install -r requirements.txt - pip install -e . jobs: diff --git a/Dockerfile b/Dockerfile index 0e90f0720..2506665ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,6 @@ WORKDIR /freqtrade # Install dependencies COPY requirements.txt /freqtrade/ RUN pip install numpy --no-cache-dir \ - && pip install future \ && pip install -r requirements.txt --no-cache-dir # Install and execute From aa1948750f16ec1d748c09117913b23e8077ab23 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:11:48 +0200 Subject: [PATCH 076/174] removing unnecessary constructor docstring --- freqtrade/edge/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index d675d1659..c52700d44 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -23,11 +23,9 @@ logger = logging.getLogger(__name__) class Edge(): config: Dict = {} + _cached_pairs: list = [] def __init__(self, config: Dict[str, Any], exchange=None) -> None: - """ - constructor - """ self.config = config self.strategy: IStrategy = StrategyResolver(self.config).strategy self.ticker_interval = self.strategy.ticker_interval From 114fd7feef0d21c579e109a1d8be2a3b992e7ffa Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:21:40 +0200 Subject: [PATCH 077/174] declaring local variables. using get for configuration --- freqtrade/edge/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index c52700d44..f07c34427 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -23,7 +23,12 @@ logger = logging.getLogger(__name__) class Edge(): config: Dict = {} - _cached_pairs: list = [] + _last_updated: int # Timestamp of pairs last updated time + _cached_pairs: list = [] # Keeps an array of + # [pair, winrate, risk reward ratio, required risk reward, expectancy] + + _total_capital: float + _allowed_risk: float def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config @@ -35,11 +40,10 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - self._last_updated = None self._cached_pairs: list = [] - self._total_capital = self.edge_config['total_capital_in_stake_currency'] - self._allowed_risk = self.edge_config['allowed_risk'] + self._total_capital = self.edge_config.get('total_capital_in_stake_currency') + self._allowed_risk = self.edge_config.get('allowed_risk') ### # From f72fb0ad04faa56b6a53f1e8f382bfeba7f05017 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:29:33 +0200 Subject: [PATCH 078/174] =?UTF-8?q?exchange=20=E2=80=9CNone=E2=80=9D=20con?= =?UTF-8?q?dition=20removed=20as=20Edge=20is=20after=20Exchange=20anyway?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f07c34427..a452e3751 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -32,6 +32,7 @@ class Edge(): def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config + self.exchange = exchange self.strategy: IStrategy = StrategyResolver(self.config).strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe @@ -45,23 +46,11 @@ class Edge(): self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') - ### - # - ### - if exchange is None: - self.config['exchange']['secret'] = '' - self.config['exchange']['password'] = '' - self.config['exchange']['uid'] = '' - self.config['dry_run'] = True - self.exchange = Exchange(self.config) - else: - self.exchange = exchange - self.fee = self.exchange.get_fee() def calculate(self) -> bool: pairs = self.config['exchange']['pair_whitelist'] - heartbeat = self.config['edge']['process_throttle_secs'] + heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated is not None) and ( self._last_updated + heartbeat > arrow.utcnow().timestamp): @@ -318,7 +307,6 @@ class Edge(): result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - # open_trade_index = np.argmax(buy_column == 1) # return empty if we don't find trade entry (i.e. buy==1) or # we find a buy but at the of array @@ -337,8 +325,6 @@ class Edge(): if stop_index == -1: stop_index = float('inf') - # stop_index = np.argmax((ohlc_columns[open_trade_index + 1:, 2] < stop_price) == True) - # Searching for the index where sell is hit sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) @@ -346,8 +332,6 @@ class Edge(): if sell_index == -1: sell_index = float('inf') - # sell_index = np.argmax(sell_column[open_trade_index + 1:] == 1) - # Check if we don't find any stop or sell point (in that case trade remains open) # It is not interesting for Edge to consider it so we simply ignore the trade # And stop iterating as the party is over @@ -379,6 +363,8 @@ class Edge(): result.append(trade) + # Calling again the same function recursively but giving + # it a view of exit_index till the end of array return result + self._detect_stop_and_sell_points( buy_column[exit_index:], sell_column[exit_index:], From ad666ac65c6a5bd1f5cd5a1357dbf9903e176b34 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:33:18 +0200 Subject: [PATCH 079/174] autopep8 corrected --- freqtrade/freqtradebot.py | 7 +++---- freqtrade/strategy/interface.py | 9 ++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 299954e49..6981dc6df 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -638,11 +638,10 @@ class FreqtradeBot(object): def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: if (self.config['edge']['enabled']): stoploss = self.edge.stoploss(trade.pair) - should_sell = \ - self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + should_sell = self.strategy.should_sell( + trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) else: - should_sell = \ - self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) + should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 73bf2313d..bd42f6c7e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -210,9 +210,12 @@ class IStrategy(ABC): :return: True if trade should be sold, False otherwise """ current_profit = trade.calc_profit_percent(rate) - stoplossflag = \ - self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date, - current_profit=current_profit, force_stoploss=force_stoploss) + stoplossflag = self.stop_loss_reached( + current_rate=rate, + trade=trade, + current_time=date, + current_profit=current_profit, + force_stoploss=force_stoploss) if stoplossflag.sell_flag: return stoplossflag From 2056b6f5f1b95d443cca9878a0abb58a8171b646 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:35:27 +0200 Subject: [PATCH 080/174] no need to initialize a variable with None --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a452e3751..3ab0f302a 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -41,7 +41,6 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - self._last_updated = None self._cached_pairs: list = [] self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') From 8b3631d1ac3b490456dd77d8a2e1ee828408db97 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:49:27 +0200 Subject: [PATCH 081/174] =?UTF-8?q?make=20=E2=80=9Cif=20condition=E2=80=9D?= =?UTF-8?q?=20more=20readable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/strategy/interface.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index bd42f6c7e..4d613265a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -251,10 +251,8 @@ class IStrategy(ABC): trailing_stop = self.config.get('trailing_stop', False) - if force_stoploss == 0: - trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True) - else: - trade.adjust_stop_loss(trade.open_rate, force_stoploss, initial=True) + trade.adjust_stop_loss(trade.open_rate, force_stoploss if force_stoploss + else self.stoploss, initial=True) # evaluate if the stoploss was hit if self.stoploss is not None and trade.stop_loss >= current_rate: From f306abb3ee19fb17b42b462d5b05ae284f0a9289 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 1 Oct 2018 17:52:07 +0200 Subject: [PATCH 082/174] No need for Exchange class in Edge --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 3ab0f302a..029f2eb43 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -11,7 +11,6 @@ from pandas import DataFrame import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult from freqtrade.arguments import Arguments -from freqtrade.exchange import Exchange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting From a6c2e40bd43ac0b4be94031edacace028ab95c26 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:49:49 +0200 Subject: [PATCH 083/174] moving time range to initializer as we have to calculate it once --- freqtrade/edge/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 029f2eb43..d2f0ff4a2 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -11,6 +11,7 @@ from pandas import DataFrame import freqtrade.optimize as optimize from freqtrade.optimize.backtesting import BacktestResult from freqtrade.arguments import Arguments +from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting @@ -28,6 +29,8 @@ class Edge(): _total_capital: float _allowed_risk: float + _since_number_of_days: int + _timerange: TimeRange def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config @@ -43,6 +46,11 @@ class Edge(): self._cached_pairs: list = [] self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') + self._since_number_of_days = self.edge_config.get('since_number_of_days', 14) + self._last_updated = 0 + + self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( + days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() @@ -50,17 +58,13 @@ class Edge(): pairs = self.config['exchange']['pair_whitelist'] heartbeat = self.edge_config.get('process_throttle_secs') - if (self._last_updated is not None) and ( + if (self._last_updated > 0) and ( self._last_updated + heartbeat > arrow.utcnow().timestamp): return False data: Dict[str, Any] = {} logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using stake_amount: %s ...', self.config['stake_amount']) logger.info('Using local backtesting data (using whitelist in given config) ...') - # TODO: add "timerange" to Edge config - timerange = Arguments.parse_timerange(None if self.config.get( - 'timerange') is None else str(self.config.get('timerange'))) data = optimize.load_data( self.config['datadir'], @@ -68,7 +72,7 @@ class Edge(): ticker_interval=self.ticker_interval, refresh_pairs=True, exchange=self.exchange, - timerange=timerange + timerange=self._timerange ) if not data: @@ -332,7 +336,7 @@ class Edge(): # Check if we don't find any stop or sell point (in that case trade remains open) # It is not interesting for Edge to consider it so we simply ignore the trade - # And stop iterating as the party is over + # And stop iterating there is no more entry if stop_index == sell_index == float('inf'): return [] @@ -343,7 +347,7 @@ class Edge(): elif stop_index > sell_index: exit_index = open_trade_index + sell_index + 1 exit_type = SellType.SELL_SIGNAL - exit_price = ohlc_columns[open_trade_index + sell_index + 1, 0] + exit_price = ohlc_columns[exit_index, 0] trade = {'pair': pair, 'stoploss': stoploss, From 11c3b3fdb90a4d95d2e07d38e8ecd25b2d1fec3d Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:53:16 +0200 Subject: [PATCH 084/174] trade_df unnecessary type removed --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index d2f0ff4a2..3d85e2286 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -114,7 +114,6 @@ class Edge(): if len(trades_df) > 0: # Only post process a frame if it has a record trades_df = self._fill_calculable_fields(trades_df) else: - trades_df = [] trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) self._cached_pairs = self._process_expectancy(trades_df) From e4fc298bd651ae1df588af3c9e684fefa4262e1b Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:53:59 +0200 Subject: [PATCH 085/174] typo corrected --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 3d85e2286..90dd0d1ba 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -161,7 +161,7 @@ class Edge(): def _fill_calculable_fields(self, result: DataFrame): """ The result frame contains a number of columns that are calculable - from othe columns. These are left blank till all rows are added, + from other columns. These are left blank till all rows are added, to be populated in single vector calls. Columns to be populated are: From d634a034550077297fe7a1c2493041723997287f Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 11:55:14 +0200 Subject: [PATCH 086/174] adding DataFrame type --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 90dd0d1ba..8e9d5861a 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -158,7 +158,7 @@ class Edge(): final = [x for x in filtered_expectancy if x in pairs] return final - def _fill_calculable_fields(self, result: DataFrame): + def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: """ The result frame contains a number of columns that are calculable from other columns. These are left blank till all rows are added, From 9c4fdc1bc59d72938fa64097347b67a38806be26 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:15:54 +0200 Subject: [PATCH 087/174] initializing Edge in Freqtradebot only if it is enabled --- freqtrade/freqtradebot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6981dc6df..ad3d7f5db 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -56,7 +56,11 @@ class FreqtradeBot(object): self.rpc: RPCManager = RPCManager(self) self.persistence = None self.exchange = Exchange(self.config) - self.edge = Edge(self.config, self.exchange) + + # Initializing Edge only if enabled + if self.config.get('edge', {}).get('enabled', False): + self.edge = Edge(self.config, self.exchange) + self._init_modules() def _init_modules(self) -> None: @@ -189,7 +193,7 @@ class FreqtradeBot(object): # Should be called before refresh_tickers # Otherwise it will override cached klines in exchange # with delta value (klines only from last refresh_pairs) - if self.config['edge']['enabled']: + if self.config.get('edge', {}).get('enabled', False): self.edge.calculate() # Refreshing candles From 3b57aef168d16a2a15c5db67ee24bee420ac84e7 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:16:09 +0200 Subject: [PATCH 088/174] config name refactored --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 8e9d5861a..ecc7544a9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -46,7 +46,7 @@ class Edge(): self._cached_pairs: list = [] self._total_capital = self.edge_config.get('total_capital_in_stake_currency') self._allowed_risk = self.edge_config.get('allowed_risk') - self._since_number_of_days = self.edge_config.get('since_number_of_days', 14) + self._since_number_of_days = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated = 0 self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( From 26b3c3f7a8f8068443d3adaa29b31d340bc55fe7 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:20:30 +0200 Subject: [PATCH 089/174] removing unnecessary typing --- freqtrade/freqtradebot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ad3d7f5db..be4b35a20 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -345,8 +345,8 @@ class FreqtradeBot(object): if avaliable_amount < stake_amount: raise DependencyException( 'Available balance(%f %s) is lower than stake amount(%f %s)' % ( - float(avaliable_amount), self.config['stake_currency'], - float(stake_amount), self.config['stake_currency']) + avaliable_amount, self.config['stake_currency'], + stake_amount, self.config['stake_currency']) ) return float(stake_amount) From 8741a63783c11ec3035a81ea4ba3b40ad15a38c3 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:20:48 +0200 Subject: [PATCH 090/174] return type of stake_amount set to float --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index ecc7544a9..edcd66337 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -135,7 +135,7 @@ class Edge(): return True - def stake_amount(self, pair: str) -> str: + def stake_amount(self, pair: str) -> float: info = [x for x in self._cached_pairs if x[0] == pair][0] stoploss = info[1] allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) From 23f8980973343dc95748ff3f7fc71b7640f7b48e Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 12:42:59 +0200 Subject: [PATCH 091/174] edge config added to CONF_SCHEMA and config_full.json.example --- config_full.json.example | 15 +++++++++++++++ freqtrade/constants.py | 21 ++++++++++++++++++++- freqtrade/tests/conftest.py | 5 ++--- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 7083bada6..8541d984b 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -55,6 +55,21 @@ ], "outdated_offset": 5 }, + "edge": { + "enabled": false, + "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, + "min_trade_number": 15, + "max_trade_duration_minute": 1440, + "remove_pumps": true + }, "experimental": { "use_sell_signal": false, "sell_profit_only": false, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index eadfa6eba..5a03f81cd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -37,7 +37,7 @@ SUPPORTED_FIAT = [ "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" - ] +] # Required json-schema for user specified config CONF_SCHEMA = { @@ -102,6 +102,7 @@ CONF_SCHEMA = { } }, 'exchange': {'$ref': '#/definitions/exchange'}, + 'edge': {'$ref': '#/definitions/edge'}, 'experimental': { 'type': 'object', 'properties': { @@ -167,6 +168,24 @@ CONF_SCHEMA = { 'outdated_offset': {'type': 'integer', 'minimum': 1} }, 'required': ['name', 'key', 'secret', 'pair_whitelist'] + }, + 'edge': { + 'type': 'object', + 'properties': { + "enabled": {'type': 'boolean'}, + "process_throttle_secs": {'type': 'integer', 'minimum': 600}, + "calculate_since_number_of_days": {'type': 'integer'}, + "total_capital_in_stake_currency": {'type': 'number'}, + "allowed_risk": {'type': 'number'}, + "stoploss_range_min": {'type': 'number'}, + "stoploss_range_max": {'type': 'number'}, + "stoploss_range_step": {'type': 'number'}, + "maximum_winrate": {'type': 'number'}, + "minimum_expectancy": {'type': 'number'}, + "min_trade_number": {'type': 'number'}, + "max_trade_duration_minute": {'type': 'integer'}, + "remove_pumps": {'type': 'boolean'} + } } }, 'anyOf': [ diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index c72cc5a40..f6f067a41 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -137,13 +137,12 @@ def default_conf(): "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, - "stoploss_range_step": -0.001, + "stoploss_range_step": -0.01, "maximum_winrate": 0.80, "minimum_expectancy": 0.20, "min_trade_number": 15, "max_trade_duration_minute": 1440, - "remove_pumps": True, - "minimum_delta": 1 + "remove_pumps": True }, "telegram": { "enabled": True, From 697493bd017445339eaacfc26258a2ed2520cc69 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 16:07:33 +0200 Subject: [PATCH 092/174] test cases for Edge package drafted --- freqtrade/tests/edge/test_edge.py | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 freqtrade/tests/edge/test_edge.py diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py new file mode 100644 index 000000000..57f026304 --- /dev/null +++ b/freqtrade/tests/edge/test_edge.py @@ -0,0 +1,52 @@ +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument + +import json +import math +import random +from typing import List +from unittest.mock import MagicMock + +import numpy as np +import pandas as pd +import pytest +from arrow import Arrow + +from freqtrade import DependencyException, constants, optimize +from freqtrade.arguments import Arguments, TimeRange +from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, + start) +from freqtrade.tests.conftest import log_has, patch_exchange, get_patched_exchange +from freqtrade.strategy.interface import SellType +from freqtrade.strategy.default_strategy import DefaultStrategy + +from freqtrade.exchange import Exchange +from freqtrade.freqtradebot import FreqtradeBot + +from freqtrade.edge import Edge + + +# Cases to be tested: +# 1) Three complete trades within dataframe (with sell or buy hit for all) +# 2) Two open trades but one without sell/buy hit +# 3) Two complete trades and one which should not be considered as it happend while +# there was an already open trade on pair +# 4) Three complete trades with buy=1 on the last frame +# 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss +# 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss +# 7) Candle drops 4% recovers to 1% entry criteria are met candle drops +# 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% + + +def test_filter(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value=[ + ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], + ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], + ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] + ] + )) + + pairs = ['A/B', 'C/D', 'E/F', 'G/H'] + assert(edge.filter(pairs) == ['E/F', 'C/D']) From a364a1e40d654269312f38141500f9183cfa4f6f Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 16:32:57 +0200 Subject: [PATCH 093/174] Edge package test cases drafted --- freqtrade/tests/edge/test_edge.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 57f026304..ba9b71d46 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,27 +1,4 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument - -import json -import math -import random -from typing import List -from unittest.mock import MagicMock - -import numpy as np -import pandas as pd -import pytest -from arrow import Arrow - -from freqtrade import DependencyException, constants, optimize -from freqtrade.arguments import Arguments, TimeRange -from freqtrade.optimize.backtesting import (Backtesting, setup_configuration, - start) -from freqtrade.tests.conftest import log_has, patch_exchange, get_patched_exchange -from freqtrade.strategy.interface import SellType -from freqtrade.strategy.default_strategy import DefaultStrategy - -from freqtrade.exchange import Exchange -from freqtrade.freqtradebot import FreqtradeBot - +from freqtrade.tests.conftest import log_has, get_patched_exchange from freqtrade.edge import Edge From de20e142a0461ad3d0c32cc99684fdf4d0de795b Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 2 Oct 2018 18:05:24 +0200 Subject: [PATCH 094/174] added 9 use cased for testing Edge --- freqtrade/tests/edge/test_edge.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index ba9b71d46..c95cd07e5 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -3,16 +3,18 @@ from freqtrade.edge import Edge # Cases to be tested: -# 1) Three complete trades within dataframe (with sell or buy hit for all) -# 2) Two open trades but one without sell/buy hit -# 3) Two complete trades and one which should not be considered as it happend while -# there was an already open trade on pair -# 4) Three complete trades with buy=1 on the last frame +############################### SELL POINTS ##################################### +# 1) Three complete trades within dataframe (with sell hit for all) +# 2) Two complete trades but one without sell hit (remains open) +# 3) Two complete trades and one buy signal while one trade is open +# 4) Two complete trades with buy=1 on the last frame +################################# STOPLOSS ###################################### # 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss # 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss -# 7) Candle drops 4% recovers to 1% entry criteria are met candle drops +# 7) Candle drops 4% recovers to 1% entry criteria are met, candle drops # 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% - +############################ PRIORITY TO STOPLOSS ################################ +# 8) Stoploss and sell are hit. should sell on stoploss def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -27,3 +29,17 @@ def test_filter(mocker, default_conf): pairs = ['A/B', 'C/D', 'E/F', 'G/H'] assert(edge.filter(pairs) == ['E/F', 'C/D']) + + +def test_three_complete_trades(): + stoploss = -0.01 + three_sell_points_hit = [ + # Buy, O, H, L, C, Sell + [1, 15, 20, 12, 17, 0], # -> should enter the trade + [1, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) + [0, 14, 15, 11, 12, 0], # -> no action + [1, 12, 25, 11, 20, 0], # -> should enter the trade + [0, 20, 30, 21, 25, 1], # -> should sell (trade 2 completed) + [1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade + [0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) + ] From a46b3ec9e7191fbec37caa8e5d7402e1cdedff41 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 3 Oct 2018 10:37:36 +0200 Subject: [PATCH 095/174] first test completed --- freqtrade/tests/edge/test_edge.py | 79 +++++++++++++++++++++++++------ 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index c95cd07e5..5714b1530 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,20 +1,24 @@ -from freqtrade.tests.conftest import log_has, get_patched_exchange +from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge +from pandas import DataFrame # Cases to be tested: -############################### SELL POINTS ##################################### +# SELL POINTS: # 1) Three complete trades within dataframe (with sell hit for all) # 2) Two complete trades but one without sell hit (remains open) # 3) Two complete trades and one buy signal while one trade is open # 4) Two complete trades with buy=1 on the last frame -################################# STOPLOSS ###################################### +################################################################### +# STOPLOSS: # 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss # 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss # 7) Candle drops 4% recovers to 1% entry criteria are met, candle drops # 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% -############################ PRIORITY TO STOPLOSS ################################ +#################################################################### +# PRIORITY TO STOPLOSS: # 8) Stoploss and sell are hit. should sell on stoploss +#################################################################### def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -31,15 +35,62 @@ def test_filter(mocker, default_conf): assert(edge.filter(pairs) == ['E/F', 'C/D']) -def test_three_complete_trades(): - stoploss = -0.01 +def _validate_ohlc(buy_ohlc_sell_matrice): + for index, ohlc in enumerate(buy_ohlc_sell_matrice): + # if not high < open < low or not high < close < low + if not ohlc[3] > ohlc[2] > ohlc[4] or not ohlc[3] > ohlc[5] > ohlc[4]: + raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') + return True + + +def _build_dataframe(buy_ohlc_sell_matrice): + _validate_ohlc(buy_ohlc_sell_matrice) + + tickers = [] + for ohlc in buy_ohlc_sell_matrice: + ticker = { + 'date': ohlc[0], + 'buy': ohlc[1], + 'open': ohlc[2], + 'high': ohlc[3], + 'low': ohlc[4], + 'close': ohlc[5], + 'sell': ohlc[6] + } + tickers.append(ticker) + return DataFrame(tickers) + + +def test_three_complete_trades(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.90 # we don't want stoploss to be hit in this test three_sell_points_hit = [ - # Buy, O, H, L, C, Sell - [1, 15, 20, 12, 17, 0], # -> should enter the trade - [1, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) - [0, 14, 15, 11, 12, 0], # -> no action - [1, 12, 25, 11, 20, 0], # -> should enter the trade - [0, 20, 30, 21, 25, 1], # -> should sell (trade 2 completed) - [1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade - [0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) + # Date, Buy, O, H, L, C, Sell + [1, 1, 15, 20, 12, 17, 0], # -> should enter the trade + [2, 0, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) + [3, 0, 14, 15, 11, 12, 0], # -> no action + [4, 1, 12, 25, 11, 20, 0], # -> should enter the trade + [5, 0, 20, 30, 19, 25, 1], # -> should sell (trade 2 completed) + [6, 1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade + [7, 0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) ] + + ticker_df = _build_dataframe(three_sell_points_hit) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Three trades must have happened + assert len(trades) == 3 + + # First trade check + assert trades[0]['open_time'] == 1 + assert trades[0]['close_time'] == 2 + + # Second trade check + assert trades[1]['open_time'] == 4 + assert trades[1]['close_time'] == 5 + + # Third trade check + assert trades[2]['open_time'] == 6 + assert trades[2]['close_time'] == 7 From 6f79b55845d26169e504bbf71655526201515c0a Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 3 Oct 2018 14:22:27 +0200 Subject: [PATCH 096/174] - function renamed to be more readable - expectancy bug resolved --- freqtrade/edge/__init__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index edcd66337..bf855ef43 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -25,7 +25,7 @@ class Edge(): config: Dict = {} _last_updated: int # Timestamp of pairs last updated time _cached_pairs: list = [] # Keeps an array of - # [pair, winrate, risk reward ratio, required risk reward, expectancy] + # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] _total_capital: float _allowed_risk: float @@ -156,6 +156,14 @@ class Edge(): # Only return pairs which are included in "pairs" argument list final = [x for x in filtered_expectancy if x in pairs] + if final: + logger.info( + 'Edge validated only %s', + final + ) + else: + logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') + return final def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: @@ -234,7 +242,7 @@ class Edge(): # # Removing Pumps if self.edge_config.get('remove_pumps', True): - results = results[results.profit_abs < float(avg + 2 * std)] + results = results[results.profit_abs <= float(avg + 2 * std)] ########################################################################## # Removing trades having a duration more than X minutes (set in config) @@ -290,13 +298,13 @@ class Edge(): result: list = [] for stoploss in stoploss_range: - result += self._detect_stop_and_sell_points( + result += self._detect_next_stop_or_sell_point( buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair ) return result - def _detect_stop_and_sell_points( + def _detect_next_stop_or_sell_point( self, buy_column, sell_column, @@ -366,7 +374,7 @@ class Edge(): # Calling again the same function recursively but giving # it a view of exit_index till the end of array - return result + self._detect_stop_and_sell_points( + return result + self._detect_next_stop_or_sell_point( buy_column[exit_index:], sell_column[exit_index:], date_column[exit_index:], From b57d9edda8b28722e382652a0ed329593a8d22ae Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 3 Oct 2018 14:23:10 +0200 Subject: [PATCH 097/174] Edge test expectancy function (round 1) --- freqtrade/tests/edge/test_edge.py | 107 +++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 5714b1530..15fc38114 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,6 +1,8 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge -from pandas import DataFrame +from pandas import DataFrame, to_datetime +import arrow +import numpy as np # Cases to be tested: @@ -20,6 +22,10 @@ from pandas import DataFrame # 8) Stoploss and sell are hit. should sell on stoploss #################################################################### +ticker_start_time = arrow.get(2018, 10, 3) +ticker_interval_in_minute = 5 + + def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) @@ -45,11 +51,11 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] for ohlc in buy_ohlc_sell_matrice: ticker = { - 'date': ohlc[0], + # ticker every 5 min + 'date': ticker_start_time.shift(minutes=(ohlc[0] * 5)).timestamp * 1000, 'buy': ohlc[1], 'open': ohlc[2], 'high': ohlc[3], @@ -58,7 +64,71 @@ def _build_dataframe(buy_ohlc_sell_matrice): 'sell': ohlc[6] } tickers.append(ticker) - return DataFrame(tickers) + + frame = DataFrame(tickers) + frame['date'] = to_datetime(frame['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + + return frame + + +def test_process_expectancy(mocker, default_conf): + default_conf['edge']['min_trade_number'] = 2 + exchange = get_patched_exchange(mocker, default_conf) + + def get_fee(): + return 0.001 + + exchange.get_fee = get_fee + edge = Edge(default_conf, exchange) + + trades = [ + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:05:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:10:00.000000000'), + 'open_index': 1, + 'close_index': 1, + 'trade_duration': '', + 'open_rate': 17, + 'close_rate': 17, + 'exit_type': 'sell_signal'}, # sdfsdf + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 20, + 'exit_type': 'sell_signal'}, + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_time': np.datetime64('2018-10-03T00:30:00.000000000'), + 'close_time': np.datetime64('2018-10-03T00:40:00.000000000'), + 'open_index': 6, + 'close_index': 7, + 'trade_duration': '', + 'open_rate': 26, + 'close_rate': 34, + 'exit_type': 'sell_signal'} + ] + + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + assert len(final) == 1 def test_three_complete_trades(mocker, default_conf): @@ -80,17 +150,32 @@ def test_three_complete_trades(mocker, default_conf): ticker_df = _build_dataframe(three_sell_points_hit) trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - # Three trades must have happened + # Three trades must have occured assert len(trades) == 3 # First trade check - assert trades[0]['open_time'] == 1 - assert trades[0]['close_time'] == 2 + # open time should be on line 1 + assert trades[0]['open_time'] == np.datetime64(ticker_start_time.shift( + minutes=(1 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + # close time should be on line 2 + assert trades[0]['close_time'] == np.datetime64(ticker_start_time.shift( + minutes=(2 * ticker_interval_in_minute)).timestamp * 1000, 'ms') # Second trade check - assert trades[1]['open_time'] == 4 - assert trades[1]['close_time'] == 5 + # open time should be on line 4 + assert trades[1]['open_time'] == np.datetime64(ticker_start_time.shift( + minutes=(4 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + # close time should be on line 5 + assert trades[1]['close_time'] == np.datetime64(ticker_start_time.shift( + minutes=(5 * ticker_interval_in_minute)).timestamp * 1000, 'ms') # Third trade check - assert trades[2]['open_time'] == 6 - assert trades[2]['close_time'] == 7 + # open time should be on line 6 + assert trades[2]['open_time'] == np.datetime64(ticker_start_time.shift( + minutes=(6 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + + # close time should be on line 7 + assert trades[2]['close_time'] == np.datetime64(ticker_start_time.shift( + minutes=(7 * ticker_interval_in_minute)).timestamp * 1000, 'ms') From 06d75a8bad9581837678c06a77f24090ce851238 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 4 Oct 2018 18:05:46 +0200 Subject: [PATCH 098/174] test cases added: force_stoploss by Edge --- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/conftest.py | 20 ++++++ freqtrade/tests/edge/test_edge.py | 2 + freqtrade/tests/test_freqtradebot.py | 96 +++++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index be4b35a20..8b7f16334 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -649,7 +649,7 @@ class FreqtradeBot(object): if should_sell.sell_flag: self.execute_sell(trade, sell_rate, should_sell.sell_type) - logger.info('excuted sell') + logger.info('executed sell, reason: %s', should_sell.sell_type) return True return False diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index f6f067a41..8e691e3c8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -12,6 +12,7 @@ from telegram import Chat, Message, Update from freqtrade.exchange.exchange_helpers import parse_ticker_dataframe from freqtrade.exchange import Exchange +from freqtrade.edge import Edge from freqtrade.freqtradebot import FreqtradeBot logging.getLogger('').setLevel(logging.INFO) @@ -42,6 +43,25 @@ def get_patched_exchange(mocker, config, api_mock=None) -> Exchange: return exchange +def patch_edge(mocker) -> None: + # "ETH/BTC", + # "LTC/BTC", + # "XRP/BTC", + # "NEO/BTC" + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value=[ + ['NEO/BTC', -0.20, 0.66, 3.71, 0.50, 1.71], + ['LTC/BTC', -0.21, 0.66, 3.71, 0.50, 1.71], + ] + )) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + +def get_patched_edge(mocker, config) -> Edge: + patch_edge(mocker) + edge = Edge(config) + return edge + # Functions for recurrent object patching def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 15fc38114..b6dc60000 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -130,6 +130,8 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 + # TODO: check expectancy + win rate etc + def test_three_complete_trades(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index c302eeb43..9d5de004d 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -18,7 +18,7 @@ from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType from freqtrade.state import State from freqtrade.strategy.interface import SellType, SellCheckTuple -from freqtrade.tests.conftest import log_has, patch_exchange +from freqtrade.tests.conftest import log_has, patch_exchange, patch_edge # Functions for recurrent object patching @@ -251,6 +251,100 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None +def test_edge_overrides_stake_amount(mocker, default_conf) -> None: + default_conf['edge']['enabled'] = True + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + freqtrade = FreqtradeBot(default_conf) + + # strategy stoploss should be ignored + freqtrade.strategy.stoploss = -0.05 + + with pytest.raises(IndexError): + freqtrade._get_trade_stake_amount('ETH/BTC') + + assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 + + +def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, default_conf) -> None: + default_conf['edge']['enabled'] = True + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + + # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 + # Thus, if price falls 21%, stoploss should be triggered + # + # mocking the ticker: price is falling ... + buy_price = limit_buy_order['price'] + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': buy_price * 0.79, + 'ask': buy_price * 0.79, + 'last': buy_price * 0.79 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets, + ) + ############################################# + + # Create a trade with "limit_buy_order" price + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() + trade = Trade.query.first() + trade.update(limit_buy_order) + ############################################# + + # stoploss shoud be hit + assert freqtrade.handle_trade(trade) is True + + assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog.record_tuples) + assert trade.sell_reason == SellType.STOP_LOSS.value + + +def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, + mocker, default_conf) -> None: + default_conf['edge']['enabled'] = True + patch_RPCManager(mocker) + patch_exchange(mocker) + patch_edge(mocker) + + # Strategy stoploss is -0.1 but Edge imposes a stoploss at -0.2 + # Thus, if price falls 15%, stoploss should not be triggered + # + # mocking the ticker: price is falling ... + buy_price = limit_buy_order['price'] + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_ticker=MagicMock(return_value={ + 'bid': buy_price * 0.85, + 'ask': buy_price * 0.85, + 'last': buy_price * 0.85 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + get_markets=markets, + ) + ############################################# + + # Create a trade with "limit_buy_order" price + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False + freqtrade.create_trade() + trade = Trade.query.first() + trade.update(limit_buy_order) + ############################################# + + # stoploss shoud be hit + assert freqtrade.handle_trade(trade) is False + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 77cac9e562a9bc6b430fbc4cd68b77281630a538 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 4 Oct 2018 18:07:47 +0200 Subject: [PATCH 099/174] autopep8 applied --- freqtrade/tests/conftest.py | 5 ++++- freqtrade/tests/test_freqtradebot.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 8e691e3c8..86e7e73c8 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -53,16 +53,19 @@ def patch_edge(mocker) -> None: ['NEO/BTC', -0.20, 0.66, 3.71, 0.50, 1.71], ['LTC/BTC', -0.21, 0.66, 3.71, 0.50, 1.71], ] - )) + )) mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + def get_patched_edge(mocker, config) -> Edge: patch_edge(mocker) edge = Edge(config) return edge # Functions for recurrent object patching + + def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: """ This function patch _init_modules() to not call dependencies diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9d5de004d..f38ecd78a 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -268,7 +268,13 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 -def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, default_conf) -> None: +def test_edge_overrides_stoploss( + limit_buy_order, + fee, + markets, + caplog, + mocker, + default_conf) -> None: default_conf['edge']['enabled'] = True patch_RPCManager(mocker) patch_exchange(mocker) @@ -309,7 +315,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, - mocker, default_conf) -> None: + mocker, default_conf) -> None: default_conf['edge']['enabled'] = True patch_RPCManager(mocker) patch_exchange(mocker) @@ -345,6 +351,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, # stoploss shoud be hit assert freqtrade.handle_trade(trade) is False + def test_get_min_pair_stake_amount(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -1825,7 +1832,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, ca exchange='binance', open_rate=0.245441, open_order_id="123456" - ) + ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -2101,9 +2108,9 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2, markets) """ patch_exchange(mocker) mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_markets=markets, - get_order_book=order_book_l2 + 'freqtrade.exchange.Exchange', + get_markets=markets, + get_order_book=order_book_l2 ) default_conf['exchange']['name'] = 'binance' default_conf['bid_strategy']['use_order_book'] = True From d3078d756487e99ed22c2b69981a32fe68b48dc0 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 4 Oct 2018 18:51:59 +0200 Subject: [PATCH 100/174] test case added: edge stop loss for pair --- freqtrade/tests/edge/test_edge.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index b6dc60000..3cc2fd76c 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -41,6 +41,21 @@ def test_filter(mocker, default_conf): assert(edge.filter(pairs) == ['E/F', 'C/D']) +def test_stoploss(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( + return_value=[ + ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], + ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], + ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] + ] + )) + + pairs = ['A/B', 'C/D', 'E/F', 'G/H'] + assert edge.stoploss('E/F') == -0.01 + + def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low From 36d928d411ffaf0e75bf5b074e5de83493ed84e3 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:06:17 +0200 Subject: [PATCH 101/174] unnecessary if removed --- freqtrade/edge/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index bf855ef43..91594f4fb 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -107,15 +107,12 @@ class Edge(): trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) - # Switch List of Trade Dicts (trades) to Dataframe + # If no trade found then exit + if len(trades) == 0: + return False + # Fill missing, calculable columns, profit, duration , abs etc. - trades_df = DataFrame(trades) - - if len(trades_df) > 0: # Only post process a frame if it has a record - trades_df = self._fill_calculable_fields(trades_df) - else: - trades_df = DataFrame.from_records(trades_df, columns=BacktestResult._fields) - + trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) self._last_updated = arrow.utcnow().timestamp @@ -187,6 +184,7 @@ class Edge(): stake = self.config.get('stake_amount') fee = self.fee + open_fee = fee / 2 close_fee = fee / 2 From bd25212bd6ee83bd4b6bfe16db5cf79e7c0e0707 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:07:20 +0200 Subject: [PATCH 102/174] test case added: edge calculate function --- freqtrade/tests/edge/test_edge.py | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 3cc2fd76c..8ddadeb58 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,8 +1,12 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge +from freqtrade import optimize from pandas import DataFrame, to_datetime import arrow import numpy as np +import math + +from unittest.mock import MagicMock # Cases to be tested: @@ -89,6 +93,61 @@ def _build_dataframe(buy_ohlc_sell_matrice): return frame +def test_edge_heartbeat_calculate(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + heartbeat = default_conf['edge']['process_throttle_secs'] + + # should not recalculate if heartbeat not reached + edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + + assert edge.calculate() == False + + +def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, + timerange=None, exchange=None): + hz = 0.1 + base = 0.001 + + ETHBTC = [ + [ + ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base + 0.0001, + math.sin(x * hz) / 1000 + base - 0.0001, + math.sin(x * hz) / 1000 + base, + 123.45 + ] for x in range(0, 500)] + + hz = 0.2 + base = 0.002 + LTCBTC = [ + [ + ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base + 0.0001, + math.sin(x * hz) / 1000 + base - 0.0001, + math.sin(x * hz) / 1000 + base, + 123.45 + ] for x in range(0, 500)] + + pairdata = {'NEO/BTC': ETHBTC, 'LTC/BTC': LTCBTC} + return pairdata + + +def test_edge_process_downloaded_data(mocker, default_conf): + default_conf['datadir'] = None + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) + mocker.patch('freqtrade.optimize.load_data', mocked_load_data) + mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) + edge = Edge(default_conf, exchange) + + assert edge.calculate() + assert len(edge._cached_pairs) == 2 + assert edge._last_updated <= arrow.utcnow().timestamp + 2 + + def test_process_expectancy(mocker, default_conf): default_conf['edge']['min_trade_number'] = 2 exchange = get_patched_exchange(mocker, default_conf) From 9e44b260e29d2370ecf03d3a3dff556cda8cc0e4 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:19:20 +0200 Subject: [PATCH 103/174] BacktestResult removed as it is not used --- freqtrade/edge/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 91594f4fb..490bc065e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -9,7 +9,6 @@ import utils_find_1st as utf1st from pandas import DataFrame import freqtrade.optimize as optimize -from freqtrade.optimize.backtesting import BacktestResult from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType From 6d4f68fcdbeb8cbbde0cdb824762eb76b718c283 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 5 Oct 2018 17:25:56 +0200 Subject: [PATCH 104/174] unnecessary variables removed --- freqtrade/tests/edge/test_edge.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 8ddadeb58..7d750cc31 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,6 +1,5 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge -from freqtrade import optimize from pandas import DataFrame, to_datetime import arrow import numpy as np @@ -56,7 +55,6 @@ def test_stoploss(mocker, default_conf): ] )) - pairs = ['A/B', 'C/D', 'E/F', 'G/H'] assert edge.stoploss('E/F') == -0.01 @@ -101,7 +99,7 @@ def test_edge_heartbeat_calculate(mocker, default_conf): # should not recalculate if heartbeat not reached edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 - assert edge.calculate() == False + assert edge.calculate() is False def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, From 6aa9cd106088e89e451690a97fc2e63ab60dcc5e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 12 Oct 2018 19:37:23 +0200 Subject: [PATCH 105/174] removing outliers per pair and not across all pairs --- freqtrade/edge/__init__.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 490bc065e..e2af560c9 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -15,7 +15,6 @@ from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting - logger = logging.getLogger(__name__) @@ -221,25 +220,19 @@ class Edge(): and keep it in a storage. The calulation will be done per pair and per strategy. """ - # Removing pairs having less than min_trades_number min_trades_number = self.edge_config.get('min_trade_number', 15) - results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number) + results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) ################################### # Removing outliers (Only Pumps) from the dataset # The method to detect outliers is to calculate standard deviation # Then every value more than (standard deviation + 2*average) is out (pump) # - # Calculating standard deviation of profits - std = results[["profit_abs"]].std() - # - # Calculating average of profits - avg = results[["profit_abs"]].mean() - # # Removing Pumps if self.edge_config.get('remove_pumps', True): - results = results[results.profit_abs <= float(avg + 2 * std)] + results = results.groupby(['pair', 'stoploss']).apply( + lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) ########################################################################## # Removing trades having a duration more than X minutes (set in config) From 57bc4a866ac835dee52e6cccdba314d40b510b27 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 18 Oct 2018 11:09:10 +0200 Subject: [PATCH 106/174] average trade duration added --- freqtrade/edge/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e2af560c9..4e3c2c7c8 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -14,6 +14,7 @@ from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.optimize.backtesting import Backtesting +import sys logger = logging.getLogger(__name__) @@ -31,6 +32,7 @@ class Edge(): _timerange: TimeRange def __init__(self, config: Dict[str, Any], exchange=None) -> None: + sys.setrecursionlimit(10000) self.config = config self.exchange = exchange self.strategy: IStrategy = StrategyResolver(self.config).strategy @@ -68,7 +70,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=True, + refresh_pairs=False, exchange=self.exchange, timerange=self._timerange ) @@ -272,13 +274,19 @@ class Edge(): return x ############################## - final = results.groupby(['pair', 'stoploss'])['profit_abs'].\ - agg([winrate, risk_reward_ratio, required_risk_reward, expectancy]).\ - reset_index().sort_values(by=['expectancy', 'stoploss'], ascending=False)\ - .groupby('pair').first().sort_values(by=['expectancy'], ascending=False) + if results.empty: + return [] + + groupby_aggregator = {'profit_abs': [winrate, risk_reward_ratio, required_risk_reward, expectancy, 'count'], 'trade_duration': ['mean']} + final = results.groupby(['pair', 'stoploss'])['profit_abs','trade_duration'].agg(groupby_aggregator).reset_index(col_level=1) + final.columns = final.columns.droplevel(0) + final = final.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() + + final.rename(columns={'mean': 'avg_duration(min)'}, inplace=True) # Returning an array of pairs in order of "expectancy" - return final.reset_index().values + return final.values def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): buy_column = ticker_data['buy'].values From 67ace0a76cfad000faee7c7ef4bf4ef7e1131ffc Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 23 Oct 2018 19:32:20 +0200 Subject: [PATCH 107/174] trade open time bug resolved (was behind of the market) --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 4e3c2c7c8..a5be19a73 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -358,7 +358,7 @@ class Edge(): 'stoploss': stoploss, 'profit_percent': '', 'profit_abs': '', - 'open_time': date_column[open_trade_index], + 'open_time': date_column[open_trade_index + 1], 'close_time': date_column[exit_index], 'open_index': start_point + open_trade_index + 1, 'close_index': start_point + exit_index, From c5474794d194f4583d95f300444f6b37d8ce60ad Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 25 Oct 2018 16:57:49 +0200 Subject: [PATCH 108/174] 1) open_trade_index refactored 2) sell index is shifted by 1 --- freqtrade/edge/__init__.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a5be19a73..5c074ceb1 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -50,7 +50,7 @@ class Edge(): self._last_updated = 0 self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) + days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() @@ -277,8 +277,18 @@ class Edge(): if results.empty: return [] - groupby_aggregator = {'profit_abs': [winrate, risk_reward_ratio, required_risk_reward, expectancy, 'count'], 'trade_duration': ['mean']} - final = results.groupby(['pair', 'stoploss'])['profit_abs','trade_duration'].agg(groupby_aggregator).reset_index(col_level=1) + groupby_aggregator = { + 'profit_abs': [ + winrate, + risk_reward_ratio, + required_risk_reward, + expectancy, + 'count'], + 'trade_duration': ['mean']} + + final = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + groupby_aggregator).reset_index(col_level=1) + final.columns = final.columns.droplevel(0) final = final.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() @@ -319,21 +329,24 @@ class Edge(): # we find a buy but at the of array if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: return [] + else: + open_trade_index += 1 # when a buy signal is seen, + # trade opens in reality on the next candle stop_price_percentage = stoploss + 1 - open_price = ohlc_columns[open_trade_index + 1, 0] + open_price = ohlc_columns[open_trade_index, 0] stop_price = (open_price * stop_price_percentage) # Searching for the index where stoploss is hit stop_index = utf1st.find_1st( - ohlc_columns[open_trade_index + 1:, 2], stop_price, utf1st.cmp_smaller) + ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) # If we don't find it then we assume stop_index will be far in future (infinite number) if stop_index == -1: stop_index = float('inf') # Searching for the index where sell is hit - sell_index = utf1st.find_1st(sell_column[open_trade_index + 1:], 1, utf1st.cmp_equal) + sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) # If we don't find it then we assume sell_index will be far in future (infinite number) if sell_index == -1: @@ -346,11 +359,17 @@ class Edge(): return [] if stop_index <= sell_index: - exit_index = open_trade_index + stop_index + 1 + exit_index = open_trade_index + stop_index exit_type = SellType.STOP_LOSS exit_price = stop_price elif stop_index > sell_index: + # if exit is SELL then we exit at the next candle exit_index = open_trade_index + sell_index + 1 + + # check if we have the next candle + if len(ohlc_columns) - 1 < exit_index: + return [] + exit_type = SellType.SELL_SIGNAL exit_price = ohlc_columns[exit_index, 0] @@ -358,9 +377,9 @@ class Edge(): 'stoploss': stoploss, 'profit_percent': '', 'profit_abs': '', - 'open_time': date_column[open_trade_index + 1], + 'open_time': date_column[open_trade_index], 'close_time': date_column[exit_index], - 'open_index': start_point + open_trade_index + 1, + 'open_index': start_point + open_trade_index, 'close_index': start_point + exit_index, 'trade_duration': '', 'open_rate': round(open_price, 15), From dfeabcf7e56ca1c2eef69f9b07fc5ed3f378985a Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 25 Oct 2018 16:59:05 +0200 Subject: [PATCH 109/174] Edge tests template refactored to be more readable --- freqtrade/tests/edge/test_edge.py | 91 +++++++++++++++++-------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 7d750cc31..d2f2b4b13 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -10,7 +10,7 @@ from unittest.mock import MagicMock # Cases to be tested: # SELL POINTS: -# 1) Three complete trades within dataframe (with sell hit for all) +# 1) Two complete trades within dataframe (with sell hit for all) # 2) Two complete trades but one without sell hit (remains open) # 3) Two complete trades and one buy signal while one trade is open # 4) Two complete trades with buy=1 on the last frame @@ -26,7 +26,8 @@ from unittest.mock import MagicMock #################################################################### ticker_start_time = arrow.get(2018, 10, 3) -ticker_interval_in_minute = 5 +ticker_interval_in_minute = 60 +_ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} def test_filter(mocker, default_conf): @@ -72,7 +73,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): for ohlc in buy_ohlc_sell_matrice: ticker = { # ticker every 5 min - 'date': ticker_start_time.shift(minutes=(ohlc[0] * 5)).timestamp * 1000, + 'date': ticker_start_time.shift(minutes=(ohlc[0] * ticker_interval_in_minute)).timestamp * 1000, 'buy': ohlc[1], 'open': ohlc[2], 'high': ohlc[3], @@ -90,6 +91,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): return frame +def _time_on_candle(number): + return np.datetime64(ticker_start_time.shift( + minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') def test_edge_heartbeat_calculate(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) @@ -110,7 +114,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals ETHBTC = [ [ ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, - math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, math.sin(x * hz) / 1000 + base, @@ -122,7 +126,7 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals LTCBTC = [ [ ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, - math.sin(x * hz) / 1000 + base, # But replace O,H,L,C + math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, math.sin(x * hz) / 1000 + base, @@ -205,51 +209,58 @@ def test_process_expectancy(mocker, default_conf): # TODO: check expectancy + win rate etc -def test_three_complete_trades(mocker, default_conf): +def test_remove_open_trade_at_the_end(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) - stoploss = -0.90 # we don't want stoploss to be hit in this test - three_sell_points_hit = [ - # Date, Buy, O, H, L, C, Sell - [1, 1, 15, 20, 12, 17, 0], # -> should enter the trade - [2, 0, 17, 18, 13, 14, 1], # -> should sell (trade 1 completed) - [3, 0, 14, 15, 11, 12, 0], # -> no action - [4, 1, 12, 25, 11, 20, 0], # -> should enter the trade - [5, 0, 20, 30, 19, 25, 1], # -> should sell (trade 2 completed) - [6, 1, 25, 27, 22, 26, 1], # -> buy and sell, should enter the trade - [7, 0, 26, 36, 25, 35, 1], # -> should sell (trade 3 completed) + stoploss = -0.99 # we don't want stoploss to be hit in this test + ticker = [ + #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [3, 1, 12, 25, 11, 20, 0], # -> + [4, 0, 20, 30, 19, 25, 1], # -> should enter the trade ] - ticker_df = _build_dataframe(three_sell_points_hit) + ticker_df = _build_dataframe(ticker) trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - # Three trades must have occured - assert len(trades) == 3 + # No trade should be found + assert len(trades) == 0 - # First trade check - # open time should be on line 1 - assert trades[0]['open_time'] == np.datetime64(ticker_start_time.shift( - minutes=(1 * ticker_interval_in_minute)).timestamp * 1000, 'ms') +def test_two_complete_trades(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) - # close time should be on line 2 - assert trades[0]['close_time'] == np.datetime64(ticker_start_time.shift( - minutes=(2 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + stoploss = -0.99 # we don't want stoploss to be hit in this test + ticker = [ + #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 17, 18, 13, 14, 1], # -> should enter the trade as B signal recieved on last candle + [2, 0, 14, 15, 11, 12, 0], # -> exit the trade as the sell signal recieved on last candle + [3, 1, 12, 25, 11, 20, 0], # -> no action + [4, 0, 20, 30, 19, 25, 0], # -> should enter the trade + [5, 0, 25, 27, 22, 26, 1], # -> no action + [6, 0, 26, 36, 25, 35, 0], # -> should sell + ] - # Second trade check - # open time should be on line 4 - assert trades[1]['open_time'] == np.datetime64(ticker_start_time.shift( - minutes=(4 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + ticker_df = _build_dataframe(ticker) + ticker_df.to_json('/Users/misaghshakeri/Projects/freq/misagh/bslap_test_df.json') + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - # close time should be on line 5 - assert trades[1]['close_time'] == np.datetime64(ticker_start_time.shift( - minutes=(5 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + # Two trades must have occured + assert len(trades) == 2 - # Third trade check - # open time should be on line 6 - assert trades[2]['open_time'] == np.datetime64(ticker_start_time.shift( - minutes=(6 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + ################### First trade check ######################## + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(2) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == ticker[2][_ohlc['open']] + ############################################################## - # close time should be on line 7 - assert trades[2]['close_time'] == np.datetime64(ticker_start_time.shift( - minutes=(7 * ticker_interval_in_minute)).timestamp * 1000, 'ms') + ################### Second trade check ######################## + assert trades[1]['open_time'] == _time_on_candle(4) + assert trades[1]['close_time'] == _time_on_candle(6) + assert trades[1]['open_rate'] == ticker[4][_ohlc['open']] + assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] + ############################################################## From 426db721260e209ffabcc49599da035da74931cd Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 25 Oct 2018 17:24:33 +0200 Subject: [PATCH 110/174] removing test line --- freqtrade/tests/edge/test_edge.py | 49 ++++++++++++++++++------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index d2f2b4b13..ec8af1200 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,6 +1,7 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge from pandas import DataFrame, to_datetime +from freqtrade.strategy.interface import SellType import arrow import numpy as np import math @@ -9,20 +10,22 @@ from unittest.mock import MagicMock # Cases to be tested: -# SELL POINTS: -# 1) Two complete trades within dataframe (with sell hit for all) -# 2) Two complete trades but one without sell hit (remains open) -# 3) Two complete trades and one buy signal while one trade is open -# 4) Two complete trades with buy=1 on the last frame +# 1) Open trade should be removed from the end +# 2) Two complete trades within dataframe (with sell hit for all) +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +# 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss +# 5) Entered, sl 2%, candle drops 4%, recovers to 1%, entry met, candle drops 20% => +# Trade 1 closed: loss 2%, Trade 2 opened, Trade 2 closed: loss 2% +# 6) ################################################################### # STOPLOSS: -# 5) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss -# 6) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss -# 7) Candle drops 4% recovers to 1% entry criteria are met, candle drops +# 6) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss +# 7) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss +# 8) Candle drops 4% recovers to 1% entry criteria are met, candle drops # 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% #################################################################### # PRIORITY TO STOPLOSS: -# 8) Stoploss and sell are hit. should sell on stoploss +# 9) Stoploss and sell are hit. should sell on stoploss #################################################################### ticker_start_time = arrow.get(2018, 10, 3) @@ -72,15 +75,17 @@ def _build_dataframe(buy_ohlc_sell_matrice): tickers = [] for ohlc in buy_ohlc_sell_matrice: ticker = { - # ticker every 5 min - 'date': ticker_start_time.shift(minutes=(ohlc[0] * ticker_interval_in_minute)).timestamp * 1000, + 'date': ticker_start_time.shift( + minutes=( + ohlc[0] * + ticker_interval_in_minute)).timestamp * + 1000, 'buy': ohlc[1], 'open': ohlc[2], 'high': ohlc[3], 'low': ohlc[4], 'close': ohlc[5], - 'sell': ohlc[6] - } + 'sell': ohlc[6]} tickers.append(ticker) frame = DataFrame(tickers) @@ -91,10 +96,12 @@ def _build_dataframe(buy_ohlc_sell_matrice): return frame + def _time_on_candle(number): return np.datetime64(ticker_start_time.shift( minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') + def test_edge_heartbeat_calculate(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) @@ -215,8 +222,8 @@ def test_remove_open_trade_at_the_end(mocker, default_conf): stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ - #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + # D, B, O, H, L, C, S [3, 1, 12, 25, 11, 20, 0], # -> [4, 0, 20, 30, 19, 25, 1], # -> should enter the trade ] @@ -227,14 +234,15 @@ def test_remove_open_trade_at_the_end(mocker, default_conf): # No trade should be found assert len(trades) == 0 + def test_two_complete_trades(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ - #D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 17, 18, 13, 14, 1], # -> should enter the trade as B signal recieved on last candle [2, 0, 14, 15, 11, 12, 0], # -> exit the trade as the sell signal recieved on last candle @@ -245,22 +253,23 @@ def test_two_complete_trades(mocker, default_conf): ] ticker_df = _build_dataframe(ticker) - ticker_df.to_json('/Users/misaghshakeri/Projects/freq/misagh/bslap_test_df.json') trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) # Two trades must have occured assert len(trades) == 2 - ################### First trade check ######################## + # First trade check assert trades[0]['open_time'] == _time_on_candle(1) assert trades[0]['close_time'] == _time_on_candle(2) assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] assert trades[0]['close_rate'] == ticker[2][_ohlc['open']] + assert trades[0]['exit_type'] == SellType.SELL_SIGNAL ############################################################## - ################### Second trade check ######################## + # Second trade check assert trades[1]['open_time'] == _time_on_candle(4) assert trades[1]['close_time'] == _time_on_candle(6) assert trades[1]['open_rate'] == ticker[4][_ohlc['open']] assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] + assert trades[1]['exit_type'] == SellType.SELL_SIGNAL ############################################################## From 2f6aafe66c9d6ee9c1e74327d994f4369169d6b4 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 18:07:38 +0100 Subject: [PATCH 111/174] Edge calculation refactored: removing redundant calculations --- freqtrade/edge/__init__.py | 86 ++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 49 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5c074ceb1..ab36b8aa8 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -215,15 +215,11 @@ class Edge(): def _process_expectancy(self, results: DataFrame) -> list: """ - This is a temporary version of edge positioning calculation. - The function will be eventually moved to a plugin called Edge in order - to calculate necessary WR, RRR and - other indictaors related to money management periodically (each X minutes) - and keep it in a storage. + This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs The calulation will be done per pair and per strategy. """ # Removing pairs having less than min_trades_number - min_trades_number = self.edge_config.get('min_trade_number', 15) + min_trades_number = self.edge_config.get('min_trade_number', 10) results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) ################################### @@ -242,61 +238,53 @@ class Edge(): results = results[results.trade_duration < max_trade_duration] ####################################################################### - # Win Rate is the number of profitable trades - # Divided by number of trades - def winrate(x): - x = x[x > 0].count() / x.count() - return x - ############################# - - # Risk Reward Ratio - # 1 / ((loss money / losing trades) / (gained money / winning trades)) - def risk_reward_ratio(x): - x = abs(1 / ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count()))) - return x - ############################## - - # Required Risk Reward - # (1/(winrate - 1) - def required_risk_reward(x): - x = (1 / (x[x > 0].count() / x.count()) - 1) - return x - ############################## - - # Expectancy - # Tells you the interest percentage you should hope - # E.x. if expectancy is 0.35, on $1 trade you should expect a target of $1.35 - def expectancy(x): - average_win = float(x[x > 0].sum() / x[x > 0].count()) - average_loss = float(abs(x[x < 0].sum() / x[x < 0].count())) - winrate = float(x[x > 0].count() / x.count()) - x = ((1 + average_win / average_loss) * winrate) - 1 - return x - ############################## - if results.empty: return [] groupby_aggregator = { 'profit_abs': [ - winrate, - risk_reward_ratio, - required_risk_reward, - expectancy, - 'count'], - 'trade_duration': ['mean']} + ('nb_trades', 'count'), # number of all trades + ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades + ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades + ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades + ], + 'trade_duration': [('avg_trade_duration', 'mean')] + } - final = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( + # Group by (pair and stoploss) the applying above aggregator + df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( groupby_aggregator).reset_index(col_level=1) - final.columns = final.columns.droplevel(0) - final = final.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + # Dropping level 0 as we don't need it + df.columns = df.columns.droplevel(0) + + # Calculating number of losing trades, average win and average loss + df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] + df['average_win'] = df['profit_sum'] / df['nb_win_trades'] + df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + + # Win rate = number of profitable trades / number of trades + df['winrate'] = df['nb_win_trades'] / df['nb_trades'] + + # risk_reward_ratio = 1 / (average loss / average win) + df['risk_reward_ratio'] = 1 / (df['average_loss'] / df['average_win']) + + # required_risk_reward = (1 / winrate) - 1 + df['required_risk_reward'] = (1 / df['winrate']) - 1 + + # expectancy = ((1 + average_win/average_loss) * winrate) - 1 + df['expectancy'] = ((1 + df['average_win'] / df['average_loss']) * df['winrate']) - 1 + + # sort by expectancy and stoploss + df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - final.rename(columns={'mean': 'avg_duration(min)'}, inplace=True) + # dropping unecessary columns + df.drop(columns=['nb_loss_trades', 'nb_win_trades', 'average_win', 'average_loss', + 'profit_sum', 'loss_sum', 'avg_trade_duration', 'nb_trades'], inplace=True) # Returning an array of pairs in order of "expectancy" - return final.values + return df.values def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): buy_column = ticker_data['buy'].values From 05b80104605791c199baaa8df6171f204994a4b7 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 18:10:03 +0100 Subject: [PATCH 112/174] removing unnecessary test cases --- freqtrade/tests/edge/test_edge.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index ec8af1200..4a2e06567 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -16,16 +16,7 @@ from unittest.mock import MagicMock # 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss # 5) Entered, sl 2%, candle drops 4%, recovers to 1%, entry met, candle drops 20% => # Trade 1 closed: loss 2%, Trade 2 opened, Trade 2 closed: loss 2% -# 6) -################################################################### -# STOPLOSS: -# 6) Candle drops 8%, stoploss at 1%: Trade closed, 1% loss -# 7) Candle drops 4% but recovers to 1% loss, stoploss at 3%: Trade closed, 3% loss -# 8) Candle drops 4% recovers to 1% entry criteria are met, candle drops -# 20%, stoploss at 2%: Trade 1 closed, Loss 2%, Trade 2 opened, Trade 2 closed, Loss 2% -#################################################################### -# PRIORITY TO STOPLOSS: -# 9) Stoploss and sell are hit. should sell on stoploss +# 6) Stoploss and sell are hit. should sell on stoploss #################################################################### ticker_start_time = arrow.get(2018, 10, 3) @@ -179,7 +170,7 @@ def test_process_expectancy(mocker, default_conf): 'trade_duration': '', 'open_rate': 17, 'close_rate': 17, - 'exit_type': 'sell_signal'}, # sdfsdf + 'exit_type': 'sell_signal'}, {'pair': 'TEST/BTC', 'stoploss': -0.9, From 333d505b661b113853f674735ca301e67d28fdc5 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:01:37 +0100 Subject: [PATCH 113/174] OHLC validation corrected --- freqtrade/tests/edge/test_edge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4a2e06567..22e4f2352 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -56,7 +56,7 @@ def test_stoploss(mocker, default_conf): def _validate_ohlc(buy_ohlc_sell_matrice): for index, ohlc in enumerate(buy_ohlc_sell_matrice): # if not high < open < low or not high < close < low - if not ohlc[3] > ohlc[2] > ohlc[4] or not ohlc[3] > ohlc[5] > ohlc[4]: + if not ohlc[3] >= ohlc[2] >= ohlc[4] or not ohlc[3] >= ohlc[5] >= ohlc[4]: raise Exception('Line ' + str(index + 1) + ' of ohlc has invalid values!') return True From bb791eac7ea4fcb22c77b5be418012489f660d90 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:19:28 +0100 Subject: [PATCH 114/174] backtesting remove from import + whitespace removed --- freqtrade/edge/__init__.py | 1 - freqtrade/freqtradebot.py | 1 - 2 files changed, 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 67e0787e7..1b0229c61 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -13,7 +13,6 @@ from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from freqtrade.optimize.backtesting import Backtesting import sys logger = logging.getLogger(__name__) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index cade551d5..890f9e563 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -196,7 +196,6 @@ class FreqtradeBot(object): # Refreshing candles self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) - # Query trades from persistence layer trades = Trade.query.filter(Trade.is_open.is_(True)).all() From 3eeaa50fe5c8bcb2f6c3360a23d5119d4ec88b71 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:54:32 +0100 Subject: [PATCH 115/174] stoploss and sell signal tests done --- freqtrade/tests/edge/test_edge.py | 96 ++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 22e4f2352..b1946bdb1 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -14,9 +14,7 @@ from unittest.mock import MagicMock # 2) Two complete trades within dataframe (with sell hit for all) # 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss # 4) Entered, sl 3%, candle drops 4%, recovers to 1% => Trade closed, 3% loss -# 5) Entered, sl 2%, candle drops 4%, recovers to 1%, entry met, candle drops 20% => -# Trade 1 closed: loss 2%, Trade 2 opened, Trade 2 closed: loss 2% -# 6) Stoploss and sell are hit. should sell on stoploss +# 5) Stoploss and sell are hit. should sell on stoploss #################################################################### ticker_start_time = arrow.get(2018, 10, 3) @@ -264,3 +262,95 @@ def test_two_complete_trades(mocker, default_conf): assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] assert trades[1]['exit_type'] == SellType.SELL_SIGNAL ############################################################## + + +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +def test_case_3(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.01 # we don't want stoploss to be hit in this test + ticker = [ + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 14, 15, 11, 12, 0], # -> enter to trade, stoploss hit + [2, 1, 12, 25, 11, 20, 0], # -> no action + ] + + ticker_df = _build_dataframe(ticker) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Two trades must have occured + assert len(trades) == 1 + + # First trade check + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(1) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] + assert trades[0]['exit_type'] == SellType.STOP_LOSS + ############################################################## + + +# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss +def test_case_4(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.03 # we don't want stoploss to be hit in this test + ticker = [ + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade + [2, 0, 16, 17, 14.4, 15.5, 0], # -> stoploss hit + [3, 0, 17, 25, 16.9, 22, 0], # -> no action + ] + + ticker_df = _build_dataframe(ticker) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Two trades must have occured + assert len(trades) == 1 + + # First trade check + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(2) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] + assert trades[0]['exit_type'] == SellType.STOP_LOSS + ############################################################## + + +# 5) Stoploss and sell are hit. should sell on stoploss +def test_case_5(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + + stoploss = -0.03 # we don't want stoploss to be hit in this test + ticker = [ + # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell + #D, B, O, H, L, C, S + [0, 1, 15, 20, 12, 17, 0], # -> no action + [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade + [2, 0, 16, 17, 14.4, 15.5, 1], # -> stoploss hit and also sell signal + [3, 0, 17, 25, 16.9, 22, 0], # -> no action + ] + + ticker_df = _build_dataframe(ticker) + trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) + + # Two trades must have occured + assert len(trades) == 1 + + # First trade check + assert trades[0]['open_time'] == _time_on_candle(1) + assert trades[0]['close_time'] == _time_on_candle(2) + assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] + assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] + assert trades[0]['exit_type'] == SellType.STOP_LOSS + ############################################################## From 2ef2754ffd731cd141ca776b36dda5a2376cd384 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:55:41 +0100 Subject: [PATCH 116/174] flake8 happiness satisfied --- freqtrade/tests/edge/test_edge.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index b1946bdb1..bb2af9bf8 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -272,7 +272,7 @@ def test_case_3(mocker, default_conf): stoploss = -0.01 # we don't want stoploss to be hit in this test ticker = [ # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 14, 15, 11, 12, 0], # -> enter to trade, stoploss hit [2, 1, 12, 25, 11, 20, 0], # -> no action @@ -301,7 +301,7 @@ def test_case_4(mocker, default_conf): stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade [2, 0, 16, 17, 14.4, 15.5, 0], # -> stoploss hit @@ -334,7 +334,7 @@ def test_case_5(mocker, default_conf): stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - #D, B, O, H, L, C, S + # D, B, O, H, L, C, S [0, 1, 15, 20, 12, 17, 0], # -> no action [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade [2, 0, 16, 17, 14.4, 15.5, 1], # -> stoploss hit and also sell signal From 237233c3000e122319f238fa18616f40d150680a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 19:59:06 +0100 Subject: [PATCH 117/174] renaming tests --- freqtrade/tests/edge/test_edge.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index bb2af9bf8..adb3ea00f 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -204,8 +204,8 @@ def test_process_expectancy(mocker, default_conf): # TODO: check expectancy + win rate etc - -def test_remove_open_trade_at_the_end(mocker, default_conf): +# 1) Open trade should be removed from the end +def test_case_1(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) @@ -224,7 +224,8 @@ def test_remove_open_trade_at_the_end(mocker, default_conf): assert len(trades) == 0 -def test_two_complete_trades(mocker, default_conf): +# 2) Two complete trades within dataframe (with sell hit for all) +def test_case_2(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) From d1ba994e5497603ad381317838216eea63fa0bbc Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:07:45 +0100 Subject: [PATCH 118/174] expectancy test completed --- freqtrade/tests/edge/test_edge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index adb3ea00f..0d3bf44cf 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -202,7 +202,12 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 - # TODO: check expectancy + win rate etc + assert final[0][0] == 'TEST/BTC' + assert final[0][1] == -0.9 + assert round(final[0][2], 10) == 0.3333333333 + assert round(final[0][3], 10) == 306.5384615384 + assert round(final[0][4], 10) == 2.0 + assert round(final[0][5], 10) == 101.5128205128 # 1) Open trade should be removed from the end def test_case_1(mocker, default_conf): From ece1c8a70210cb6041fc5039a7cd8415ac05578c Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:12:48 +0100 Subject: [PATCH 119/174] flake8 again and again and again and again https://www.youtube.com/watch?v=MuSK3pDDYD4 --- freqtrade/tests/edge/test_edge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 0d3bf44cf..510a79af1 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -202,13 +202,14 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 - assert final[0][0] == 'TEST/BTC' + assert final[0][0] == 'TEST/BTC' assert final[0][1] == -0.9 assert round(final[0][2], 10) == 0.3333333333 assert round(final[0][3], 10) == 306.5384615384 assert round(final[0][4], 10) == 2.0 assert round(final[0][5], 10) == 101.5128205128 + # 1) Open trade should be removed from the end def test_case_1(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) From 85768fcc51ce41e34812913c2323736b159dedb1 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:35:46 +0100 Subject: [PATCH 120/174] beginning of doc --- docs/index.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 730f1095e..b333e2db1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # freqtrade documentation Welcome to freqtrade documentation. Please feel free to contribute to -this documentation if you see it became outdated by sending us a -Pull-request. Do not hesitate to reach us on +this documentation if you see it became outdated by sending us a +Pull-request. Do not hesitate to reach us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtMjQ5NTM0OTYzMzY3LWMxYzE3M2MxNDdjMGM3ZTYwNzFjMGIwZGRjNTc3ZGU3MGE3NzdmZGMwNmU3NDM5ZTNmM2Y3NjRiNzk4NmM4OGE) if you do not find the answer to your questions. @@ -24,7 +24,10 @@ Pull-request. Do not hesitate to reach us on - [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Test your strategy with Backtesting] + (https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Edge positionning] + (https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 83b3323c569b607ab9411157330c01d98400f8e3 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:48:35 +0100 Subject: [PATCH 121/174] formating md --- docs/index.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index b333e2db1..a4b172817 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,10 +24,8 @@ Pull-request. Do not hesitate to reach us on - [Bot Optimization](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md) - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - - [Test your strategy with Backtesting] - (https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning] - (https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) + - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) + - [Edge positionning](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From b57ae20af495eb34cd058288bf0e6501f9171270 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:49:31 +0100 Subject: [PATCH 122/174] edge doc file added --- docs/edge.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/edge.md diff --git a/docs/edge.md b/docs/edge.md new file mode 100644 index 000000000..cbf11fc47 --- /dev/null +++ b/docs/edge.md @@ -0,0 +1 @@ +Something \ No newline at end of file From f77fa6b5925e77d4fab9ee63f5a05a2cb71d737b Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:51:55 +0100 Subject: [PATCH 123/174] misharizing temporarily for doc --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index a4b172817..bd670b102 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning](https://github.com/freqtrade/freqtrade/blob/develop/docs/edge.md) + - [Edge positionning](https://github.com/mishaker/freqtrade/blob/develop/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 9cb660776cc9e4c398c84d89d5342ab8f6de583a Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 20:52:46 +0100 Subject: [PATCH 124/174] money_mgt added --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index bd670b102..879ee4f80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning](https://github.com/mishaker/freqtrade/blob/develop/docs/edge.md) + - [Edge positionning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From f6498bf5f7c4c441a45c81d5ec3c26df54c7bd0e Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 2 Nov 2018 22:13:38 +0100 Subject: [PATCH 125/174] beginning --- docs/edge.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index cbf11fc47..d72422d53 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1 +1,14 @@ -Something \ No newline at end of file +# Edge positionning + +## Introduction +Trading is all about probability. no one can claim having the strategy working all the time. you have to assume that sometimes you lose.

+But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

+But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... +That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

+lets complicate it more: you win 80% of time but only 2$, I win 20% of time but if I win I win 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: how do you calculate that? how do you know if you wanna play with me? +The answer comes to two factors: +- Win Rate +- Risk Reward Ratio + +Win rate is quite self explanatory. means over X trades what is the perctange winning number of trades compared to total number of trades (note that we don't consider how much you gained but only If you won or not). \ No newline at end of file From b6d4e11e88327a6883dfaf65d1faf48fab7a67ed Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 3 Nov 2018 14:31:34 +0100 Subject: [PATCH 126/174] added minimum win rate to config --- config.json.example | 13 ++++++------- config_full.json.example | 10 +++++----- freqtrade/constants.py | 2 +- freqtrade/edge/__init__.py | 14 +++++++++++--- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/config.json.example b/config.json.example index 4ab4f2c72..3554260bb 100644 --- a/config.json.example +++ b/config.json.example @@ -55,19 +55,18 @@ }, "edge": { "enabled": false, - "process_throttle_secs": 1800, - "calculate_since_number_of_days": 14, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 2, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, - "stoploss_range_step": -0.001, - "maximum_winrate": 0.80, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, "minimum_expectancy": 0.20, - "min_trade_number": 15, + "min_trade_number": 10, "max_trade_duration_minute": 1440, - "remove_pumps": true, - "minimum_delta": 1 + "remove_pumps": false }, "telegram": { "enabled": true, diff --git a/config_full.json.example b/config_full.json.example index e8a3417df..7182c1f85 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -61,18 +61,18 @@ }, "edge": { "enabled": false, - "process_throttle_secs": 1800, - "calculate_since_number_of_days": 14, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 2, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, "stoploss_range_step": -0.01, - "maximum_winrate": 0.80, + "minimum_winrate": 0.60, "minimum_expectancy": 0.20, - "min_trade_number": 15, + "min_trade_number": 10, "max_trade_duration_minute": 1440, - "remove_pumps": true + "remove_pumps": false }, "experimental": { "use_sell_signal": false, diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 217855ecf..df98c30b8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -182,7 +182,7 @@ CONF_SCHEMA = { "stoploss_range_min": {'type': 'number'}, "stoploss_range_max": {'type': 'number'}, "stoploss_range_step": {'type': 'number'}, - "maximum_winrate": {'type': 'number'}, + "minimum_winrate": {'type': 'number'}, "minimum_expectancy": {'type': 'number'}, "min_trade_number": {'type': 'number'}, "max_trade_duration_minute": {'type': 'integer'}, diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 1b0229c61..f0ca6af06 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -145,10 +145,18 @@ class Edge(): def filter(self, pairs) -> list: # Filtering pairs acccording to the expectancy filtered_expectancy: list = [] + + # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] filtered_expectancy = [ - x[0] for x in self._cached_pairs if x[5] > float( - self.edge_config.get( - 'minimum_expectancy', 0.2))] + x[0] for x in self._cached_pairs if ( + (x[5] > float( + self.edge_config.get( + 'minimum_expectancy', + 0.2))) & ( + x[2] > float( + self.edge_config.get( + 'minimum_winrate', + 0.60))))] # Only return pairs which are included in "pairs" argument list final = [x for x in filtered_expectancy if x in pairs] From d7821acbf09548fec29afbd79a375135992043e2 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 3 Nov 2018 14:33:17 +0100 Subject: [PATCH 127/174] refreshing pairs on each iteration --- freqtrade/edge/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index f0ca6af06..2afb913bd 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -69,7 +69,7 @@ class Edge(): self.config['datadir'], pairs=pairs, ticker_interval=self.ticker_interval, - refresh_pairs=False, + refresh_pairs=True, exchange=self.exchange, timerange=self._timerange ) From 14bfd4b7ee3e4b9a42453ea3698891329e28876c Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:11:58 +0100 Subject: [PATCH 128/174] using named tuples for keeping pairs data --- freqtrade/edge/__init__.py | 76 +++++++++++++++---------------- freqtrade/tests/edge/test_edge.py | 54 +++++++++++----------- 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2afb913bd..e5334fb46 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -13,6 +13,7 @@ from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver +from collections import namedtuple import sys logger = logging.getLogger(__name__) @@ -21,16 +22,11 @@ logger = logging.getLogger(__name__) class Edge(): config: Dict = {} - _last_updated: int # Timestamp of pairs last updated time - _cached_pairs: list = [] # Keeps an array of - # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] - - _total_capital: float - _allowed_risk: float - _since_number_of_days: int - _timerange: TimeRange + _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs def __init__(self, config: Dict[str, Any], exchange=None) -> None: + + # Increasing recursive limit as with need it for large datasets sys.setrecursionlimit(10000) self.config = config self.exchange = exchange @@ -42,13 +38,18 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - self._cached_pairs: list = [] - self._total_capital = self.edge_config.get('total_capital_in_stake_currency') - self._allowed_risk = self.edge_config.get('allowed_risk') - self._since_number_of_days = self.edge_config.get('calculate_since_number_of_days', 14) - self._last_updated = 0 - self._timerange = Arguments.parse_timerange("%s-" % arrow.now().shift( + # pair info data type + self._pair_info = namedtuple( + 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + self._total_capital: float = self.edge_config.get('total_capital_in_stake_currency') + self._allowed_risk: float = self.edge_config.get('allowed_risk') + self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) + self._last_updated: int = 0 # Timestamp of pairs last updated time + + self._timerange: TimeRange = Arguments.parse_timerange("%s-" % arrow.now().shift( days=-1 * self._since_number_of_days).format('YYYYMMDD')) self.fee = self.exchange.get_fee() @@ -132,34 +133,24 @@ class Edge(): return True def stake_amount(self, pair: str) -> float: - info = [x for x in self._cached_pairs if x[0] == pair][0] - stoploss = info[1] + stoploss = self._cached_pairs[pair].stoploss allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5) position_size = abs(round((allowed_capital_at_risk / stoploss), 5)) return position_size def stoploss(self, pair: str) -> float: - info = [x for x in self._cached_pairs if x[0] == pair][0] - return info[1] + return self._cached_pairs[pair].stoploss def filter(self, pairs) -> list: - # Filtering pairs acccording to the expectancy - filtered_expectancy: list = [] - # [pair, stoploss, winrate, risk reward ratio, required risk reward, expectancy] - filtered_expectancy = [ - x[0] for x in self._cached_pairs if ( - (x[5] > float( - self.edge_config.get( - 'minimum_expectancy', - 0.2))) & ( - x[2] > float( - self.edge_config.get( - 'minimum_winrate', - 0.60))))] + final = [] + + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ + pair in pairs: + final.append(pair) - # Only return pairs which are included in "pairs" argument list - final = [x for x in filtered_expectancy if x in pairs] if final: logger.info( 'Edge validated only %s', @@ -220,7 +211,7 @@ class Edge(): return result - def _process_expectancy(self, results: DataFrame) -> list: + def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: """ This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs The calulation will be done per pair and per strategy. @@ -246,7 +237,7 @@ class Edge(): ####################################################################### if results.empty: - return [] + return {} groupby_aggregator = { 'profit_abs': [ @@ -286,12 +277,17 @@ class Edge(): df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - # dropping unecessary columns - df.drop(columns=['nb_loss_trades', 'nb_win_trades', 'average_win', 'average_loss', - 'profit_sum', 'loss_sum', 'avg_trade_duration', 'nb_trades'], inplace=True) + final = {} + for x in df.itertuples(): + final[x.pair] = self._pair_info( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy) - # Returning an array of pairs in order of "expectancy" - return df.values + # Returning a list of pairs in order of "expectancy" + return final def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): buy_column = ticker_data['buy'].values diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 510a79af1..710920e91 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -2,6 +2,7 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType +from collections import namedtuple import arrow import numpy as np import math @@ -20,17 +21,19 @@ from unittest.mock import MagicMock ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} +_pair_info = namedtuple( + 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') def test_filter(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( - return_value=[ - ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], - ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], - ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] - ] + return_value={ + 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + } )) pairs = ['A/B', 'C/D', 'E/F', 'G/H'] @@ -41,11 +44,11 @@ def test_stoploss(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( - return_value=[ - ['E/F', -0.01, 0.66, 3.71, 0.50, 1.71], - ['C/D', -0.01, 0.66, 3.71, 0.50, 1.71], - ['N/O', -0.01, 0.66, 3.71, 0.50, 1.71] - ] + return_value={ + 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + } )) assert edge.stoploss('E/F') == -0.01 @@ -61,7 +64,7 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] + tickers= [] for ohlc in buy_ohlc_sell_matrice: ticker = { 'date': ticker_start_time.shift( @@ -79,9 +82,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): frame = DataFrame(tickers) frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) + unit = 'ms', + utc = True, + infer_datetime_format = True) return frame @@ -92,17 +95,17 @@ def _time_on_candle(number): def test_edge_heartbeat_calculate(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) - heartbeat = default_conf['edge']['process_throttle_secs'] + exchange=get_patched_exchange(mocker, default_conf) + edge=Edge(default_conf, exchange) + heartbeat=default_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + edge._last_updated=arrow.utcnow().timestamp - heartbeat + 1 assert edge.calculate() is False -def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, +def mocked_load_data(datadir, pairs = [], ticker_interval = '0m', refresh_pairs = False, timerange=None, exchange=None): hz = 0.1 base = 0.001 @@ -202,13 +205,12 @@ def test_process_expectancy(mocker, default_conf): final = edge._process_expectancy(trades_df) assert len(final) == 1 - assert final[0][0] == 'TEST/BTC' - assert final[0][1] == -0.9 - assert round(final[0][2], 10) == 0.3333333333 - assert round(final[0][3], 10) == 306.5384615384 - assert round(final[0][4], 10) == 2.0 - assert round(final[0][5], 10) == 101.5128205128 - + assert 'TEST/BTC' in final + assert final['TEST/BTC'].stoploss == -0.9 + assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333 + assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 + assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 + assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 # 1) Open trade should be removed from the end def test_case_1(mocker, default_conf): From 120655d262b754de46fb4fdd30246266e7ab0ee1 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:43:57 +0100 Subject: [PATCH 129/174] fixing tests for namedtuple --- freqtrade/edge/__init__.py | 16 +++++++++------- freqtrade/tests/conftest.py | 24 +++++++++++++----------- freqtrade/tests/test_freqtradebot.py | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e5334fb46..2cf2d8b96 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -149,7 +149,7 @@ class Edge(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ pair in pairs: - final.append(pair) + final.append(pair) if final: logger.info( @@ -279,12 +279,14 @@ class Edge(): final = {} for x in df.itertuples(): - final[x.pair] = self._pair_info( - x.stoploss, - x.winrate, - x.risk_reward_ratio, - x.required_risk_reward, - x.expectancy) + info = { + 'stoploss': x.stoploss, + 'winrate': x.winrate, + 'risk_reward_ratio': x.risk_reward_ratio, + 'required_risk_reward': x.required_risk_reward, + 'expectancy': x.expectancy + } + final[x.pair] = self._pair_info(**info) # Returning a list of pairs in order of "expectancy" return final diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 86e7e73c8..3dd7b1f2c 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -4,6 +4,7 @@ import logging from datetime import datetime from functools import reduce from typing import Dict, Optional +from collections import namedtuple from unittest.mock import MagicMock, PropertyMock import arrow @@ -48,19 +49,20 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" + pair_info = namedtuple('pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( - return_value=[ - ['NEO/BTC', -0.20, 0.66, 3.71, 0.50, 1.71], - ['LTC/BTC', -0.21, 0.66, 3.71, 0.50, 1.71], - ] + return_value={ + 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), + 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), + } )) - mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) - mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value = -0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value = True)) def get_patched_edge(mocker, config) -> Edge: patch_edge(mocker) - edge = Edge(config) + edge=Edge(config) return edge # Functions for recurrent object patching @@ -84,15 +86,15 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config) -def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: +def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]]=None) -> None: """ Mocker to coinmarketcap to speed up tests :param mocker: mocker to patch coinmarketcap class :return: None """ - tickermock = MagicMock(return_value={'price_usd': 12345.0}) - listmock = MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', + tickermock=MagicMock(return_value={'price_usd': 12345.0}) + listmock=MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', 'website_slug': 'bitcoin'}, {'id': 1027, 'name': 'Ethereum', 'symbol': 'ETH', 'website_slug': 'ethereum'} @@ -108,7 +110,7 @@ def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> Non @pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ - configuration = { + configuration={ "max_open_trades": 1, "stake_currency": "BTC", "stake_amount": 0.001, diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 266ad82ee..217e438df 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -256,7 +256,7 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: # strategy stoploss should be ignored freqtrade.strategy.stoploss = -0.05 - with pytest.raises(IndexError): + with pytest.raises(KeyError): freqtrade._get_trade_stake_amount('ETH/BTC') assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 From 8ea9b3746bfbedc20983b32fcbb5ebdaa5661dc4 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:51:54 +0100 Subject: [PATCH 130/174] passing pair to get_trade_stake_amount --- freqtrade/rpc/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c3cbce2e7..7e7b60ebc 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -410,7 +410,7 @@ class RPC(object): raise RPCException(f'position for {pair} already open - id: {trade.id}') # gen stake amount - stakeamount = self._freqtrade._get_trade_stake_amount() + stakeamount = self._freqtrade._get_trade_stake_amount(pair) # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): From ed24d96a7944dcdf0c6547cf53f134053476c578 Mon Sep 17 00:00:00 2001 From: misagh Date: Sun, 4 Nov 2018 18:57:57 +0100 Subject: [PATCH 131/174] some formatting for flake8 --- freqtrade/tests/conftest.py | 18 ++++++++++-------- freqtrade/tests/edge/test_edge.py | 20 +++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 3dd7b1f2c..37109767b 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -49,20 +49,22 @@ def patch_edge(mocker) -> None: # "LTC/BTC", # "XRP/BTC", # "NEO/BTC" - pair_info = namedtuple('pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') + pair_info = namedtuple( + 'pair_info', + 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'NEO/BTC': pair_info(-0.20, 0.66, 3.71, 0.50, 1.71), 'LTC/BTC': pair_info(-0.21, 0.66, 3.71, 0.50, 1.71), } )) - mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value = -0.20)) - mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value = True)) + mocker.patch('freqtrade.edge.Edge.stoploss', MagicMock(return_value=-0.20)) + mocker.patch('freqtrade.edge.Edge.calculate', MagicMock(return_value=True)) def get_patched_edge(mocker, config) -> Edge: patch_edge(mocker) - edge=Edge(config) + edge = Edge(config) return edge # Functions for recurrent object patching @@ -86,15 +88,15 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: return FreqtradeBot(config) -def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]]=None) -> None: +def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]] = None) -> None: """ Mocker to coinmarketcap to speed up tests :param mocker: mocker to patch coinmarketcap class :return: None """ - tickermock=MagicMock(return_value={'price_usd': 12345.0}) - listmock=MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', + tickermock = MagicMock(return_value={'price_usd': 12345.0}) + listmock = MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', 'website_slug': 'bitcoin'}, {'id': 1027, 'name': 'Ethereum', 'symbol': 'ETH', 'website_slug': 'ethereum'} @@ -110,7 +112,7 @@ def patch_coinmarketcap(mocker, value: Optional[Dict[str, float]]=None) -> None: @pytest.fixture(scope="function") def default_conf(): """ Returns validated configuration suitable for most tests """ - configuration={ + configuration = { "max_open_trades": 1, "stake_currency": "BTC", "stake_amount": 0.001, diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 710920e91..18e934352 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -64,7 +64,7 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers= [] + tickers = [] for ohlc in buy_ohlc_sell_matrice: ticker = { 'date': ticker_start_time.shift( @@ -82,9 +82,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): frame = DataFrame(tickers) frame['date'] = to_datetime(frame['date'], - unit = 'ms', - utc = True, - infer_datetime_format = True) + unit='ms', + utc=True, + infer_datetime_format=True) return frame @@ -95,17 +95,17 @@ def _time_on_candle(number): def test_edge_heartbeat_calculate(mocker, default_conf): - exchange=get_patched_exchange(mocker, default_conf) - edge=Edge(default_conf, exchange) - heartbeat=default_conf['edge']['process_throttle_secs'] + exchange = get_patched_exchange(mocker, default_conf) + edge = Edge(default_conf, exchange) + heartbeat = default_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated=arrow.utcnow().timestamp - heartbeat + 1 + edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 assert edge.calculate() is False -def mocked_load_data(datadir, pairs = [], ticker_interval = '0m', refresh_pairs = False, +def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=False, timerange=None, exchange=None): hz = 0.1 base = 0.001 @@ -213,6 +213,8 @@ def test_process_expectancy(mocker, default_conf): assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 # 1) Open trade should be removed from the end + + def test_case_1(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) From 2c0fc3c73564ec7eec9d309758a83eb0f6ea3ef9 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 14:36:10 +0100 Subject: [PATCH 132/174] Test latex images --- docs/edge.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index d72422d53..40ecedc55 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,14 +1,22 @@ -# Edge positionning +# Edge positioning + +This page explains how to use Edge Positioning module in your bot in order to enter into a trade only of the trade has a reasonable win rate and risk reward ration, and consequently adjust your position size and stoploss. + +## Table of Contents + +- [Introduction](#introduction) ## Introduction -Trading is all about probability. no one can claim having the strategy working all the time. you have to assume that sometimes you lose.

+Trading is all about probability. no one can claim that he has the strategy working all the time. you have to assume that sometimes you lose.

But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

-lets complicate it more: you win 80% of time but only 2$, I win 20% of time but if I win I win 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

-The question is: how do you calculate that? how do you know if you wanna play with me? +lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: how do you calculate that? how do you know if you wanna play? The answer comes to two factors: - Win Rate - Risk Reward Ratio -Win rate is quite self explanatory. means over X trades what is the perctange winning number of trades compared to total number of trades (note that we don't consider how much you gained but only If you won or not). \ No newline at end of file +Win rate means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). + + \ No newline at end of file From cc41317670f7aa695ebdbda4893143caea84a82c Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 15:49:10 +0100 Subject: [PATCH 133/174] documentation WIP 1 --- docs/edge.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 40ecedc55..cab2de748 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -17,6 +17,30 @@ The answer comes to two factors: - Win Rate - Risk Reward Ratio -Win rate means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). - \ No newline at end of file +### Win Rate +Means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). + + + + +### Risk Reward Ratio +Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. it is basically what you potentially win divided by what you potentially lose: + + + +Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: + + + +### Expectancy + +At this point we can combine W and R to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades, and subtracting the percentage of losing trades, which is calculated as follows: + +Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate + +Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. + +It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. + +You can also use this number to evaluate the effectiveness of modifications to this system. From 49d30ad0e250aa935a51b49e250c90ae3e297967 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 15:55:35 +0100 Subject: [PATCH 134/174] doc WIP 2 --- docs/edge.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index cab2de748..d5648d9df 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -22,16 +22,20 @@ The answer comes to two factors: Means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). - +W = (Number of winning trades) / (Number of losing trades) ### Risk Reward Ratio Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. it is basically what you potentially win divided by what you potentially lose: - +R = Profit / Loss Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: - +average profit = (Sum of profits) / (Number of winning trades) + +average loss = (Sum of losses) / (Number of losing trades) + +R = (average profit) / (average loss) ### Expectancy @@ -39,6 +43,10 @@ At this point we can combine W and R to create an expectancy ratio. This is a si Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate +So lets say your Win rate is 28% and your Risk Reward Ratio is 5: + +Expectancy = (5 * 0.28) - 0.72 = 0.68 + Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. From 4fbabd3b99f1cd8ed3ea9054dbc5c8bbb6ec711f Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 17:24:11 +0100 Subject: [PATCH 135/174] Doc for Edge WIP 3 --- docs/edge.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index d5648d9df..07ccafd83 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -5,6 +5,8 @@ This page explains how to use Edge Positioning module in your bot in order to en ## Table of Contents - [Introduction](#introduction) +- [How does it work?](#how-does-it-work?) +- [Configurations](#configurations) ## Introduction Trading is all about probability. no one can claim that he has the strategy working all the time. you have to assume that sometimes you lose.

@@ -52,3 +54,86 @@ Superficially, this means that on average you expect this strategy’s trades to It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. You can also use this number to evaluate the effectiveness of modifications to this system. + +**NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data , there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. + +## How does it work? +If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. it then calculates win rate and expectancy over X trades for each stoploss. here is an example: + +| Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | +|----------|:-------------:|-------------:|------------------:|-----------:| +| XZC/ETH | -0.03 | 0.52 |1.359670 | 0.228 | +| XZC/ETH | -0.01 | 0.50 |1.176384 | 0.088 | +| XZC/ETH | -0.02 | 0.51 |1.115941 | 0.079 | + +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. in the above example stoploss at 3% leads to the maximum expectancy according to historical data. + +Edge then forces stoploss to your strategy dynamically. + +### Position size +Edge dictates the stake amount for each trade to the bot according to the following factors: + +- Allowed capital at risk +- Stoploss + +Alowed capital at risk is calculated as follows: + +**allowed capital at risk** = **total capital** X **allowed risk per trade** + +**Stoploss** is calculated as described above against historical data. + +Your position size then will be: + +**position size** = **allowed capital at risk** / **stoploss** + +## Configurations +Edge has following configurations: + +#### enabled +If true, then Edge will run periodically + +#### process_throttle_secs +How often should Edge run ? (in seconds) + +#### calculate_since_number_of_days +Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Expectancy +Note that it downloads historical data so increasing this number would lead to slowing down the bot + +#### total_capital_in_stake_currency +This your total capital at risk. if edge is enabled then stake_amount is ignored in favor of this parameter + +#### allowed_risk +Percentage of allowed risk per trade + +#### stoploss_range_min +Minimum stoploss (default to -0.01) + +#### stoploss_range_max +Maximum stoploss (default to -0.10) + +#### stoploss_range_step +As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. +Note than having a smaller step means having a bigger range which could lead to slow calculation.
+if you set this parameter to -0.001, you then slow down the Edge calculatiob by a factor of 10 + +#### minimum_winrate +It filters pairs which don't have at least minimum_winrate (default to 0.60) +This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio. + +#### minimum_expectancy +It filters paris which have an expectancy lower than this number (default to 0.20) +Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. + +#### min_trade_number +When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. the more this number is the more Edge is reliable. having a win rate of 100% on a single trade doesn't mean anything at all. but having a win rate of 70% over past 100 trades means clearly something.
+ +Default to 10 (it is highly recommanded not to decrease this number) + +#### max_trade_duration_minute +Edge will filter out trades with long duration. if a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. but if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+Default to 1 day (1440 = 60 * 24) + + +#### remove_pumps +Edge will remove sudden pumps in a given market while going through historical data. however, given that pumps happen very often in crypto markets, we recommand you keep this off.
+Default to false \ No newline at end of file From 8a25490146091c0f1de35a397a22153b0f89d531 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 17:28:07 +0100 Subject: [PATCH 136/174] Typo corrected --- docs/edge.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 07ccafd83..f2756e831 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -100,10 +100,11 @@ Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Ex Note that it downloads historical data so increasing this number would lead to slowing down the bot #### total_capital_in_stake_currency -This your total capital at risk. if edge is enabled then stake_amount is ignored in favor of this parameter +This your total capital at risk in your stake currency. if edge is enabled then stake_amount is ignored in favor of this parameter #### allowed_risk -Percentage of allowed risk per trade +Percentage of allowed risk per trade
+default to 1% #### stoploss_range_min Minimum stoploss (default to -0.01) @@ -114,7 +115,7 @@ Maximum stoploss (default to -0.10) #### stoploss_range_step As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. Note than having a smaller step means having a bigger range which could lead to slow calculation.
-if you set this parameter to -0.001, you then slow down the Edge calculatiob by a factor of 10 +if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10 #### minimum_winrate It filters pairs which don't have at least minimum_winrate (default to 0.60) @@ -133,7 +134,6 @@ Default to 10 (it is highly recommanded not to decrease this number) Edge will filter out trades with long duration. if a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. but if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
Default to 1 day (1440 = 60 * 24) - #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. however, given that pumps happen very often in crypto markets, we recommand you keep this off.
Default to false \ No newline at end of file From 5754e5196019335d8a6032c20fe1900bcb8ccaa6 Mon Sep 17 00:00:00 2001 From: misagh Date: Mon, 5 Nov 2018 17:32:12 +0100 Subject: [PATCH 137/174] more typos --- docs/edge.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index f2756e831..808c4f008 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,6 +1,6 @@ # Edge positioning -This page explains how to use Edge Positioning module in your bot in order to enter into a trade only of the trade has a reasonable win rate and risk reward ration, and consequently adjust your position size and stoploss. +This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. ## Table of Contents @@ -9,7 +9,7 @@ This page explains how to use Edge Positioning module in your bot in order to en - [Configurations](#configurations) ## Introduction -Trading is all about probability. no one can claim that he has the strategy working all the time. you have to assume that sometimes you lose.

+Trading is all about probability. no one can claim that he has a strategy working all the time. you have to assume that sometimes you lose.

But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

From 5c38b92a75f6ac3e375d7a93043c4e7d59ad2b1b Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:05:42 +0100 Subject: [PATCH 138/174] simplifying calculations to be more readable --- freqtrade/edge/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 2cf2d8b96..81d3d7c99 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -226,7 +226,7 @@ class Edge(): # Then every value more than (standard deviation + 2*average) is out (pump) # # Removing Pumps - if self.edge_config.get('remove_pumps', True): + if self.edge_config.get('remove_pumps', False): results = results.groupby(['pair', 'stoploss']).apply( lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) ########################################################################## @@ -249,7 +249,7 @@ class Edge(): 'trade_duration': [('avg_trade_duration', 'mean')] } - # Group by (pair and stoploss) the applying above aggregator + # Group by (pair and stoploss) by applying above aggregator df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( groupby_aggregator).reset_index(col_level=1) @@ -264,14 +264,14 @@ class Edge(): # Win rate = number of profitable trades / number of trades df['winrate'] = df['nb_win_trades'] / df['nb_trades'] - # risk_reward_ratio = 1 / (average loss / average win) - df['risk_reward_ratio'] = 1 / (df['average_loss'] / df['average_win']) + # risk_reward_ratio = average win / average loss + df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] # required_risk_reward = (1 / winrate) - 1 df['required_risk_reward'] = (1 / df['winrate']) - 1 - # expectancy = ((1 + average_win/average_loss) * winrate) - 1 - df['expectancy'] = ((1 + df['average_win'] / df['average_loss']) * df['winrate']) - 1 + # expectancy = (risk_reward_ratio * winrate) - (lossrate) + df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) # sort by expectancy and stoploss df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( From 133ba5d6a1856d2bfdc1623c63f8b3ab4972cf1f Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:16:20 +0100 Subject: [PATCH 139/174] =?UTF-8?q?moving=20stop=20loss=20range=20to=20ini?= =?UTF-8?q?t=20as=20it=20doesn=E2=80=99t=20need=20to=20be=20called=20on=20?= =?UTF-8?q?each=20iteration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 81d3d7c99..748f95a6e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -20,6 +20,13 @@ logger = logging.getLogger(__name__) class Edge(): + """ + Calculates Win Rate, Risk Reward Ratio, Expectancy + against historical data for a give set of markets and a strategy + it then adjusts stoploss and position size accordingly + and force it into the strategy + Author: https://github.com/mishaker + """ config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs @@ -49,6 +56,17 @@ class Edge(): self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time + self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + # calculating stoploss range + self._stoploss_range = np.arange( + self._stoploss_range_min, + self._stoploss_range_max, + self._stoploss_range_step + ) + self._timerange: TimeRange = Arguments.parse_timerange("%s-" % arrow.now().shift( days=-1 * self._since_number_of_days).format('YYYYMMDD')) @@ -91,11 +109,6 @@ class Edge(): ) headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) - stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) - stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - stoploss_range = np.arange(stoploss_range_min, stoploss_range_max, stoploss_range_step) - trades: list = [] for pair, pair_data in preprocessed.items(): # Sorting dataframe by date and reset index @@ -105,7 +118,7 @@ class Edge(): ticker_data = self.advise_sell( self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - trades += self._find_trades_for_stoploss_range(ticker_data, pair, stoploss_range) + trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) # If no trade found then exit if len(trades) == 0: From bcecaa69d4cedc651de5887b59e158a5b8377752 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:41:46 +0100 Subject: [PATCH 140/174] removing global variable modification --- freqtrade/edge/__init__.py | 2 -- freqtrade/strategy/interface.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 748f95a6e..648130e70 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -33,8 +33,6 @@ class Edge(): def __init__(self, config: Dict[str, Any], exchange=None) -> None: - # Increasing recursive limit as with need it for large datasets - sys.setrecursionlimit(10000) self.config = config self.exchange = exchange self.strategy: IStrategy = StrategyResolver(self.config).strategy diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 4d613265a..de5fc1a60 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,7 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool, force_stoploss=0) -> SellCheckTuple: + sell: bool, force_stoploss: float=0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. From 1b457e902c1e168068f8bad396419108764bc2f4 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 19:45:41 +0100 Subject: [PATCH 141/174] config initializer refactored --- freqtrade/freqtradebot.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 330306307..00dfa6cbc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,8 +58,13 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) # Initializing Edge only if enabled - if self.config.get('edge', {}).get('enabled', False): - self.edge = Edge(self.config, self.exchange) + self.edge = Edge( + self.config, + self.exchange) if self.config.get( + 'edge', + {}).get( + 'enabled', + False) else None self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() @@ -190,7 +195,7 @@ class FreqtradeBot(object): # Should be called before refresh_tickers # Otherwise it will override cached klines in exchange # with delta value (klines only from last refresh_pairs) - if self.config.get('edge', {}).get('enabled', False): + if self.edge: self.edge.calculate() # Refreshing candles @@ -332,7 +337,7 @@ class FreqtradeBot(object): for the stake currency :return: float: Stake Amount """ - if self.config['edge']['enabled']: + if self.edge: stake_amount = self.edge.stake_amount(pair) else: stake_amount = self.config['stake_amount'] @@ -354,7 +359,7 @@ class FreqtradeBot(object): stake_amount, self.config['stake_currency']) ) - return float(stake_amount) + return stake_amount def _get_min_pair_stake_amount(self, pair: str, price: float) -> Optional[float]: markets = self.exchange.get_markets() @@ -407,7 +412,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals - if self.config['edge']['enabled']: + if self.edge: whitelist = self.edge.filter(whitelist) for _pair in whitelist: From 23d3a7f31e70c2afa5d0e94132608c4d0cd0e2e9 Mon Sep 17 00:00:00 2001 From: misagh Date: Tue, 6 Nov 2018 20:11:15 +0100 Subject: [PATCH 142/174] capital after dots and default values corrected --- config.json.example | 2 +- docs/edge.md | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/config.json.example b/config.json.example index 57b97c61c..bbd9648da 100644 --- a/config.json.example +++ b/config.json.example @@ -56,7 +56,7 @@ "edge": { "enabled": false, "process_throttle_secs": 3600, - "calculate_since_number_of_days": 2, + "calculate_since_number_of_days": 7, "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, diff --git a/docs/edge.md b/docs/edge.md index 808c4f008..b083d1575 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -9,12 +9,12 @@ This page explains how to use Edge Positioning module in your bot in order to en - [Configurations](#configurations) ## Introduction -Trading is all about probability. no one can claim that he has a strategy working all the time. you have to assume that sometimes you lose.

-But it doesn't mean there is no rule, it only means rules should work "most of the time". let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. is it an interetsing game ? no, it is quite boring, isn't it?

-But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. now it is becoming interesting ... -That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. that means over time you will win 8$ risking only 2$ on each toss of coin.

-lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. the calculation is: 80% * 2$ versus 20% * 8$. it is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

-The question is: how do you calculate that? how do you know if you wanna play? +Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.

+But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interetsing game ? no, it is quite boring, isn't it?

+But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. Now it is becoming interesting ... +That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin.

+Lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+The question is: How do you calculate that? how do you know if you wanna play? The answer comes to two factors: - Win Rate - Risk Reward Ratio @@ -27,7 +27,7 @@ Means over X trades what is the perctange of winning trades to total number of t W = (Number of winning trades) / (Number of losing trades) ### Risk Reward Ratio -Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. it is basically what you potentially win divided by what you potentially lose: +Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: R = Profit / Loss @@ -58,7 +58,7 @@ You can also use this number to evaluate the effectiveness of modifications to t **NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data , there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. ## How does it work? -If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. it then calculates win rate and expectancy over X trades for each stoploss. here is an example: +If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over X trades for each stoploss. Here is an example: | Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | |----------|:-------------:|-------------:|------------------:|-----------:| @@ -66,7 +66,7 @@ If enabled in config, Edge will go through historical data with a range of stopl | XZC/ETH | -0.01 | 0.50 |1.176384 | 0.088 | | XZC/ETH | -0.02 | 0.51 |1.115941 | 0.079 | -The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. in the above example stoploss at 3% leads to the maximum expectancy according to historical data. +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at 3% leads to the maximum expectancy according to historical data. Edge then forces stoploss to your strategy dynamically. @@ -93,18 +93,19 @@ Edge has following configurations: If true, then Edge will run periodically #### process_throttle_secs -How often should Edge run ? (in seconds) +How often should Edge run in seconds? (default to 3600 so one hour) #### calculate_since_number_of_days Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Expectancy -Note that it downloads historical data so increasing this number would lead to slowing down the bot +Note that it downloads historical data so increasing this number would lead to slowing down the bot
+(default to 7) #### total_capital_in_stake_currency -This your total capital at risk in your stake currency. if edge is enabled then stake_amount is ignored in favor of this parameter +This your total capital at risk in your stake currency. If edge is enabled then stake_amount is ignored in favor of this parameter #### allowed_risk Percentage of allowed risk per trade
-default to 1% +(default to 1%) #### stoploss_range_min Minimum stoploss (default to -0.01) @@ -126,14 +127,14 @@ It filters paris which have an expectancy lower than this number (default to 0.2 Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. #### min_trade_number -When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. the more this number is the more Edge is reliable. having a win rate of 100% on a single trade doesn't mean anything at all. but having a win rate of 70% over past 100 trades means clearly something.
+When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
Default to 10 (it is highly recommanded not to decrease this number) #### max_trade_duration_minute -Edge will filter out trades with long duration. if a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. but if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
Default to 1 day (1440 = 60 * 24) #### remove_pumps -Edge will remove sudden pumps in a given market while going through historical data. however, given that pumps happen very often in crypto markets, we recommand you keep this off.
+Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommand you keep this off.
Default to false \ No newline at end of file From 553e5656ac143619347f86ae86c071fb993bcaf0 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:12:46 +0100 Subject: [PATCH 143/174] forcestoploss refactored --- freqtrade/freqtradebot.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 00dfa6cbc..bff1b35ba 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,13 +58,8 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) # Initializing Edge only if enabled - self.edge = Edge( - self.config, - self.exchange) if self.config.get( - 'edge', - {}).get( - 'enabled', - False) else None + self.edge = Edge(self.config, self.exchange) if \ + self.config.get('edge', {}).get('enabled', False) else None self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] self._init_modules() @@ -653,10 +648,10 @@ class FreqtradeBot(object): return False def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool: - if (self.config['edge']['enabled']): + if self.edge: stoploss = self.edge.stoploss(trade.pair) should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.utcnow(), buy, sell, stoploss) + trade, sell_rate, datetime.utcnow(), buy, sell, force_stoploss=stoploss) else: should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell) From 6d63de1932f3c0d1d4c6a7ef99ef750ac224b4a6 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:15:04 +0100 Subject: [PATCH 144/174] removing unnecessary lib --- freqtrade/edge/__init__.py | 1 - freqtrade/strategy/interface.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 648130e70..0a56e8bfa 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -14,7 +14,6 @@ from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType from freqtrade.strategy.resolver import IStrategy, StrategyResolver from collections import namedtuple -import sys logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 27016b18c..212559c8c 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -203,7 +203,8 @@ class IStrategy(ABC): return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, - sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: + sell: bool, low: float = None, high: float = None, + force_stoploss: float = 0) -> SellCheckTuple: """ This function evaluate if on the condition required to trigger a sell has been reached if the threshold is reached and updates the trade record. From bd1b05828ef0969d854c2608a94416df0081fda5 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:19:58 +0100 Subject: [PATCH 145/174] typos in documentation corrected --- docs/edge.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index b083d1575..48461ec5b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -10,10 +10,10 @@ This page explains how to use Edge Positioning module in your bot in order to en ## Introduction Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose.

-But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interetsing game ? no, it is quite boring, isn't it?

-But lets say the probabiliy that we have heads is 80%, and the probablilty that we have tails is 20%. Now it is becoming interesting ... +But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: You give me 10$. Is it an interesting game ? no, it is quite boring, isn't it?

+But let's say the probability that we have heads is 80%, and the probability that we have tails is 20%. Now it is becoming interesting ... That means 10$ x 80% versus 10$ x 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin.

-Lets complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

+Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% * 2$ versus 20% * 8$. It is becoming boring again because overtime you win $1.6$ (80% x 2$) and me $1.6 (20% * 8$) too.

The question is: How do you calculate that? how do you know if you wanna play? The answer comes to two factors: - Win Rate @@ -21,7 +21,7 @@ The answer comes to two factors: ### Win Rate -Means over X trades what is the perctange of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). +Means over X trades what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). W = (Number of winning trades) / (Number of losing trades) @@ -76,7 +76,7 @@ Edge dictates the stake amount for each trade to the bot according to the follow - Allowed capital at risk - Stoploss -Alowed capital at risk is calculated as follows: +Allowed capital at risk is calculated as follows: **allowed capital at risk** = **total capital** X **allowed risk per trade** @@ -96,7 +96,7 @@ If true, then Edge will run periodically How often should Edge run in seconds? (default to 3600 so one hour) #### calculate_since_number_of_days -Number of days of data agaist which Edge calculates Win Rate, Risk Reward and Expectancy +Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy Note that it downloads historical data so increasing this number would lead to slowing down the bot
(default to 7) @@ -127,14 +127,14 @@ It filters paris which have an expectancy lower than this number (default to 0.2 Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. #### min_trade_number -When calulating W and R and E (expectancy) against histoical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
+When calculating W and R and E (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
-Default to 10 (it is highly recommanded not to decrease this number) +Default to 10 (it is highly recommended not to decrease this number) #### max_trade_duration_minute -Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the stratgy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
Default to 1 day (1440 = 60 * 24) #### remove_pumps -Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommand you keep this off.
+Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
Default to false \ No newline at end of file From 5bd3bae5aff4aa9ac67af2af8e62fbc60c8fe566 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:24:13 +0100 Subject: [PATCH 146/174] unifying default value explanations --- docs/edge.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 48461ec5b..427e37103 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -90,10 +90,12 @@ Your position size then will be: Edge has following configurations: #### enabled -If true, then Edge will run periodically +If true, then Edge will run periodically
+(default to false) #### process_throttle_secs -How often should Edge run in seconds? (default to 3600 so one hour) +How often should Edge run in seconds?
+(default to 3600 so one hour) #### calculate_since_number_of_days Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy @@ -108,33 +110,37 @@ Percentage of allowed risk per trade
(default to 1%) #### stoploss_range_min -Minimum stoploss (default to -0.01) +Minimum stoploss
+(default to -0.01) #### stoploss_range_max -Maximum stoploss (default to -0.10) +Maximum stoploss
+(default to -0.10) #### stoploss_range_step As an example if this is set to -0.01 then Edge will test the strategy for [-0.01, -0,02, -0,03 ..., -0.09, -0.10] ranges. Note than having a smaller step means having a bigger range which could lead to slow calculation.
-if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10 +if you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
+(default to -0.01) #### minimum_winrate -It filters pairs which don't have at least minimum_winrate (default to 0.60) -This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio. +It filters pairs which don't have at least minimum_winrate. +This comes handy if you want to be conservative and don't comprise win rate in favor of risk reward ratio.
+(default to 0.60) #### minimum_expectancy -It filters paris which have an expectancy lower than this number (default to 0.20) -Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. +It filters paris which have an expectancy lower than this number . +Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
+(default to 0.20) #### min_trade_number When calculating W and R and E (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
- -Default to 10 (it is highly recommended not to decrease this number) +(default to 10, it is highly recommended not to decrease this number) #### max_trade_duration_minute Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
-Default to 1 day (1440 = 60 * 24) +(default to 1 day, 1440 = 60 * 24) #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
-Default to false \ No newline at end of file +(default to false) \ No newline at end of file From f75606d2955d9ac197e55ab2e1391aa53ec46829 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:27:10 +0100 Subject: [PATCH 147/174] formulas markdown style --- docs/edge.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index 427e37103..21a5566ab 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -24,20 +24,20 @@ The answer comes to two factors: Means over X trades what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only If you won or not). -W = (Number of winning trades) / (Number of losing trades) +`W = (Number of winning trades) / (Number of losing trades)` ### Risk Reward Ratio Risk Reward Ratio is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: -R = Profit / Loss +`R = Profit / Loss` Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: -average profit = (Sum of profits) / (Number of winning trades) +`Average profit = (Sum of profits) / (Number of winning trades)` -average loss = (Sum of losses) / (Number of losing trades) +`Average loss = (Sum of losses) / (Number of losing trades)` -R = (average profit) / (average loss) +`R = (Average profit) / (Average loss)` ### Expectancy @@ -47,7 +47,7 @@ Expectancy Ratio = (Risk Reward Ratio x Win Rate) – Loss Rate So lets say your Win rate is 28% and your Risk Reward Ratio is 5: -Expectancy = (5 * 0.28) - 0.72 = 0.68 +`Expectancy = (5 * 0.28) - 0.72 = 0.68` Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your losers. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. From b425cc3e3ba161a31ffbeaf2506f8d1d51eb466b Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:33:35 +0100 Subject: [PATCH 148/174] adding explanation regarding max trade duration and interval --- docs/edge.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/edge.md b/docs/edge.md index 21a5566ab..8fd5607ec 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -139,6 +139,7 @@ When calculating W and R and E (expectancy) against historical data, you always #### max_trade_duration_minute Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
+**NOTICE:** While configuring this value, you should take into consideration your ticker interval. as an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. default value is set assuming your strategy interval is relatively small (1m or 5m, etc).
(default to 1 day, 1440 = 60 * 24) #### remove_pumps From 96a43327ca5c3093bffc1aa2a7d7d87cfc231800 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:52:15 +0100 Subject: [PATCH 149/174] _pair_info moved to class header for reusibility --- freqtrade/edge/__init__.py | 9 +++++---- freqtrade/tests/edge/test_edge.py | 15 ++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 0a56e8bfa..5d62c1415 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -30,6 +30,11 @@ class Edge(): config: Dict = {} _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + # pair info data type + _pair_info = namedtuple( + 'pair_info', + ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) + def __init__(self, config: Dict[str, Any], exchange=None) -> None: self.config = config @@ -42,10 +47,6 @@ class Edge(): self.advise_buy = self.strategy.advise_buy self.edge_config = self.config.get('edge', {}) - - # pair info data type - self._pair_info = namedtuple( - 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs self._total_capital: float = self.edge_config.get('total_capital_in_stake_currency') diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 18e934352..4dee004cb 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -2,7 +2,6 @@ from freqtrade.tests.conftest import get_patched_exchange from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType -from collections import namedtuple import arrow import numpy as np import math @@ -21,8 +20,6 @@ from unittest.mock import MagicMock ticker_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} -_pair_info = namedtuple( - 'pair_info', 'stoploss, winrate, risk_reward_ratio, required_risk_reward, expectancy') def test_filter(mocker, default_conf): @@ -30,9 +27,9 @@ def test_filter(mocker, default_conf): edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) } )) @@ -45,9 +42,9 @@ def test_stoploss(mocker, default_conf): edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ - 'E/F': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'C/D': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), - 'N/O': _pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) + 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'C/D': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), + 'N/O': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71) } )) From 934dd97eb2d160e83b7cc0e86501d605710cac24 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 18:54:21 +0100 Subject: [PATCH 150/174] adding init for edge test folder --- freqtrade/tests/edge/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 freqtrade/tests/edge/__init__.py diff --git a/freqtrade/tests/edge/__init__.py b/freqtrade/tests/edge/__init__.py new file mode 100644 index 000000000..e69de29bb From 7b8098553391e9297ecab09127996bfa40a25872 Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 19:00:18 +0100 Subject: [PATCH 151/174] comments on recursive function + indentation of function declaration --- freqtrade/edge/__init__.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5d62c1415..61cf6940e 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -316,15 +316,16 @@ class Edge(): return result - def _detect_next_stop_or_sell_point( - self, - buy_column, - sell_column, - date_column, - ohlc_columns, - stoploss, - pair, - start_point=0): + def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, + ohlc_columns, stoploss, pair, start_point=0): + """ + Iterate through ohlc_columns recursively in order to find the next trade + Next trade opens from the first buy signal noticed to + The sell or stoploss signal after it. + It then calls itself cutting OHLC, buy_column, sell_colum and date_column + Cut from (the exit trade index) + 1 + Author: https://github.com/mishaker + """ result: list = [] open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) From 3330d327ed97f7a099f2f02abdb771d8706572ce Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 19:03:08 +0100 Subject: [PATCH 152/174] =?UTF-8?q?removing=20reserve=20keyword=20?= =?UTF-8?q?=E2=80=9Cfilter=E2=80=9D:=20replaced=20by=20=E2=80=9Cadjust?= =?UTF-8?q?=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- freqtrade/edge/__init__.py | 11 +++++------ freqtrade/freqtradebot.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 61cf6940e..c0df17909 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -152,10 +152,12 @@ class Edge(): def stoploss(self, pair: str) -> float: return self._cached_pairs[pair].stoploss - def filter(self, pairs) -> list: + def adjust(self, pairs) -> list: + """ + Filters out and sorts "pairs" according to Edge calculated pairs + """ final = [] - for pair, info in self._cached_pairs.items(): if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ @@ -163,10 +165,7 @@ class Edge(): final.append(pair) if final: - logger.info( - 'Edge validated only %s', - final - ) + logger.info('Edge validated only %s', final) else: logger.info('Edge removed all pairs as no pair with minimum expectancy was found !') diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bff1b35ba..f82f6da59 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -408,7 +408,7 @@ class FreqtradeBot(object): # running get_signal on historical data fetched # to find buy signals if self.edge: - whitelist = self.edge.filter(whitelist) + whitelist = self.edge.adjust(whitelist) for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) From 866da8aaa1d49f13ffa1b6fc169b234aa1c40cba Mon Sep 17 00:00:00 2001 From: misagh Date: Wed, 7 Nov 2018 19:24:53 +0100 Subject: [PATCH 153/174] reinitializing Edge calculated data in case of inability to download backtesting data --- freqtrade/edge/__init__.py | 2 ++ freqtrade/tests/edge/test_edge.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index c0df17909..5d1ce5f11 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -92,6 +92,8 @@ class Edge(): ) if not data: + # Reinitializing cached pairs + self._cached_pairs = {} logger.critical("No data found. Edge is stopped ...") return False diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 4dee004cb..ed47a60eb 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -22,7 +22,7 @@ ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} -def test_filter(mocker, default_conf): +def test_adjust(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf) edge = Edge(default_conf, exchange) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( @@ -34,7 +34,7 @@ def test_filter(mocker, default_conf): )) pairs = ['A/B', 'C/D', 'E/F', 'G/H'] - assert(edge.filter(pairs) == ['E/F', 'C/D']) + assert(edge.adjust(pairs) == ['E/F', 'C/D']) def test_stoploss(mocker, default_conf): From e5c6499706ba9c7523b57b3c91c72ca61147ad06 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 00:22:46 +0100 Subject: [PATCH 154/174] assigning strategy to edge from FreqtradeBot --- freqtrade/edge/__init__.py | 8 ++--- freqtrade/freqtradebot.py | 2 +- freqtrade/tests/edge/test_edge.py | 52 ++++++++++++++++--------------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 5d1ce5f11..a468a76a3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -2,6 +2,7 @@ """ Edge positioning package """ import logging from typing import Any, Dict +from collections import namedtuple import arrow import numpy as np @@ -12,8 +13,7 @@ import freqtrade.optimize as optimize from freqtrade.arguments import Arguments from freqtrade.arguments import TimeRange from freqtrade.strategy.interface import SellType -from freqtrade.strategy.resolver import IStrategy, StrategyResolver -from collections import namedtuple + logger = logging.getLogger(__name__) @@ -35,11 +35,11 @@ class Edge(): 'pair_info', ['stoploss', 'winrate', 'risk_reward_ratio', 'required_risk_reward', 'expectancy']) - def __init__(self, config: Dict[str, Any], exchange=None) -> None: + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: self.config = config self.exchange = exchange - self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.strategy = strategy self.ticker_interval = self.strategy.ticker_interval self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe self.get_timeframe = optimize.get_timeframe diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f82f6da59..babe0a1da 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -58,7 +58,7 @@ class FreqtradeBot(object): self.exchange = Exchange(self.config) # Initializing Edge only if enabled - self.edge = Edge(self.config, self.exchange) if \ + self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None self.active_pair_whitelist: List[str] = self.config['exchange']['pair_whitelist'] diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index ed47a60eb..1ac5e62c4 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,4 +1,7 @@ -from freqtrade.tests.conftest import get_patched_exchange +# pragma pylint: disable=missing-docstring, C0103 +# pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments + +from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType @@ -23,8 +26,8 @@ _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell' def test_adjust(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), @@ -38,8 +41,8 @@ def test_adjust(mocker, default_conf): def test_stoploss(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': Edge._pair_info(-0.01, 0.66, 3.71, 0.50, 1.71), @@ -92,8 +95,8 @@ def _time_on_candle(number): def test_edge_heartbeat_calculate(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) heartbeat = default_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached @@ -135,11 +138,10 @@ def mocked_load_data(datadir, pairs=[], ticker_interval='0m', refresh_pairs=Fals def test_edge_process_downloaded_data(mocker, default_conf): default_conf['datadir'] = None - exchange = get_patched_exchange(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) mocker.patch('freqtrade.optimize.load_data', mocked_load_data) - mocker.patch('freqtrade.exchange.Exchange.refresh_tickers', MagicMock()) - edge = Edge(default_conf, exchange) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) assert edge.calculate() assert len(edge._cached_pairs) == 2 @@ -148,13 +150,13 @@ def test_edge_process_downloaded_data(mocker, default_conf): def test_process_expectancy(mocker, default_conf): default_conf['edge']['min_trade_number'] = 2 - exchange = get_patched_exchange(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) def get_fee(): return 0.001 - exchange.get_fee = get_fee - edge = Edge(default_conf, exchange) + freqtrade.exchange.get_fee = get_fee + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) trades = [ {'pair': 'TEST/BTC', @@ -213,8 +215,8 @@ def test_process_expectancy(mocker, default_conf): def test_case_1(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ @@ -233,8 +235,8 @@ def test_case_1(mocker, default_conf): # 2) Two complete trades within dataframe (with sell hit for all) def test_case_2(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.99 # we don't want stoploss to be hit in this test ticker = [ @@ -274,8 +276,8 @@ def test_case_2(mocker, default_conf): # 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss def test_case_3(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.01 # we don't want stoploss to be hit in this test ticker = [ @@ -303,8 +305,8 @@ def test_case_3(mocker, default_conf): # 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss def test_case_4(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ @@ -333,11 +335,11 @@ def test_case_4(mocker, default_conf): # 5) Stoploss and sell are hit. should sell on stoploss def test_case_5(mocker, default_conf): - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - exchange = get_patched_exchange(mocker, default_conf) - edge = Edge(default_conf, exchange) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) stoploss = -0.03 # we don't want stoploss to be hit in this test ticker = [ From 6d80c038773252d9ae065adc4c729983844f5dfc Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 00:28:20 +0100 Subject: [PATCH 155/174] removing raise KeyError in test --- freqtrade/tests/test_freqtradebot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 1afb86024..9c72f9177 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -256,9 +256,7 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: # strategy stoploss should be ignored freqtrade.strategy.stoploss = -0.05 - with pytest.raises(KeyError): - freqtrade._get_trade_stake_amount('ETH/BTC') - + assert 'ETH/BTC' not in freqtrade.edge._cached_pairs assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 From 5d850825f52bd7e5ee30da90745ce27d22ee3a87 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 14:10:52 +0100 Subject: [PATCH 156/174] adding a notice about the incompatibility of Edge with Dynamic whitelist --- docs/edge.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index 8fd5607ec..5cace4ea3 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -2,6 +2,8 @@ This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. +**NOTICE:** Edge positioning is not compatible with dynamic whitelist. it overrides dynamic whitelist. + ## Table of Contents - [Introduction](#introduction) From a7dc8f5f4f1782db3e7518ffcffd64654ec3b482 Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 14:16:46 +0100 Subject: [PATCH 157/174] adding edge configuration to configuration.md and removed whitespaces --- docs/configuration.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 15ba4b48d..372cd5bb3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,7 +13,7 @@ This page explains how to configure your `config.json` file. We recommend to copy and use the `config.json.example` as a template for your bot configuration. -The table below will list all configuration parameters. +The table below will list all configuration parameters. | Command | Default | Mandatory | Description | |----------|---------|----------|-------------| @@ -21,11 +21,11 @@ The table below will list all configuration parameters. | `stake_currency` | BTC | Yes | Crypto-currency used for trading. | `stake_amount` | 0.05 | Yes | Amount of crypto-currency your bot will use for each trade. Per default, the bot will use (0.05 BTC x 3) = 0.15 BTC in total will be always engaged. Set it to 'unlimited' to allow the bot to use all avaliable balance. | `ticker_interval` | [1m, 5m, 30m, 1h, 1d] | No | The ticker interval to use (1min, 5 min, 30 min, 1 hour or 1 day). Default is 5 minutes -| `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. +| `fiat_display_currency` | USD | Yes | Fiat currency used to show your profits. More information below. | `dry_run` | true | Yes | Define if the bot must be in Dry-run or production mode. -| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. -| `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. -| `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. +| `process_only_new_candles` | false | No | If set to true indicators are processed only once a new candle arrives. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. Can be set either in Configuration or in the strategy. +| `minimal_roi` | See below | No | Set the threshold in percent the bot will use to sell a trade. More information below. If set, this parameter will override `minimal_roi` from your strategy file. +| `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `trailing_stop` | false | No | Enables trailing stop-loss (based on `stoploss` in either configuration or strategy file). | `trailing_stop_positve` | 0 | No | Changes stop-loss once profit has been reached. | `trailing_stop_positve_offset` | 0 | No | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. @@ -47,6 +47,7 @@ The table below will list all configuration parameters. | `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) +| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` @@ -70,7 +71,7 @@ The definition of each config parameters is in [misc.py](https://github.com/freq ### Understand stake_amount `stake_amount` is an amount of crypto-currency your bot will use for each trade. -The minimal value is 0.0005. If there is not enough crypto-currency in +The minimal value is 0.0005. If there is not enough crypto-currency in the account an exception is generated. To allow the bot to trade all the avaliable `stake_currency` in your account set `stake_amount` = `unlimited`. In this case a trade amount is calclulated as `currency_balanse / (max_open_trades - current_open_trades)`. @@ -186,13 +187,13 @@ creating trades. } ``` -Once you will be happy with your bot performance, you can switch it to +Once you will be happy with your bot performance, you can switch it to production mode. ## Switch to production mode -In production mode, the bot will engage your money. Be careful a wrong -strategy can lose all your money. Be aware of what you are doing when +In production mode, the bot will engage your money. Be careful a wrong +strategy can lose all your money. Be aware of what you are doing when you run it in production mode. ### To switch your bot in production mode: @@ -242,7 +243,7 @@ freqtrade ### Embedding Strategies -FreqTrade provides you with with an easy way to embed the strategy into your configuration file. +FreqTrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. From aefc20738af5714cdab91602ce1d3aa1ccd2f8be Mon Sep 17 00:00:00 2001 From: misagh Date: Thu, 8 Nov 2018 14:18:07 +0100 Subject: [PATCH 158/174] adding dot to the end of the phrase. --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 372cd5bb3..d70a47b38 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,7 +47,7 @@ The table below will list all configuration parameters. | `exchange.ccxt_rate_limit` | True | No | DEPRECATED!! Have CCXT handle Exchange rate limits. Depending on the exchange, having this to false can lead to temporary bans from the exchange. | `exchange.ccxt_config` | None | No | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) | `exchange.ccxt_async_config` | None | No | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation) -| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation +| `edge` | false | No | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.use_sell_signal` | false | No | Use your sell strategy in addition of the `minimal_roi`. | `experimental.sell_profit_only` | false | No | waits until you have made a positive profit before taking a sell decision. | `experimental.ignore_roi_if_buy_signal` | false | No | Does not sell if the buy-signal is still active. Takes preference over `minimal_roi` and `use_sell_signal` From 12e735e8316fc681ee0970f45b72f61fa17360fc Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 9 Nov 2018 20:51:15 +0100 Subject: [PATCH 159/174] 1) extracting edge_conf to a fixture 2) test cased adjusted to Backtesting 3) Formatted backtesting_details a bit --- freqtrade/tests/conftest.py | 35 ++- freqtrade/tests/edge/test_edge.py | 272 +++++++----------- .../tests/optimize/test_backtest_detail.py | 18 +- 3 files changed, 137 insertions(+), 188 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 37109767b..cbaca6c04 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -156,21 +156,6 @@ def default_conf(): "NEO/BTC" ] }, - "edge": { - "enabled": False, - "process_throttle_secs": 1800, - "calculate_since_number_of_days": 14, - "total_capital_in_stake_currency": 0.5, - "allowed_risk": 0.01, - "stoploss_range_min": -0.01, - "stoploss_range_max": -0.1, - "stoploss_range_step": -0.01, - "maximum_winrate": 0.80, - "minimum_expectancy": 0.20, - "min_trade_number": 15, - "max_trade_duration_minute": 1440, - "remove_pumps": True - }, "telegram": { "enabled": True, "token": "token", @@ -794,3 +779,23 @@ def buy_order_fee(): 'status': 'closed', 'fee': None } + +@pytest.fixture(scope="function") +def edge_conf(default_conf): + default_conf['edge'] = { + "enabled": False, + "process_throttle_secs": 1800, + "calculate_since_number_of_days": 14, + "total_capital_in_stake_currency": 0.5, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "maximum_winrate": 0.80, + "minimum_expectancy": 0.20, + "min_trade_number": 15, + "max_trade_duration_minute": 1440, + "remove_pumps": False + } + + return default_conf diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index 1ac5e62c4..aa45ddfff 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -1,17 +1,20 @@ -# pragma pylint: disable=missing-docstring, C0103 +# pragma pylint: disable=missing-docstring, C0103, C0330 # pragma pylint: disable=protected-access, too-many-lines, invalid-name, too-many-arguments +import pytest +import logging from freqtrade.tests.conftest import get_patched_freqtradebot from freqtrade.edge import Edge from pandas import DataFrame, to_datetime from freqtrade.strategy.interface import SellType +from freqtrade.tests.optimize import (BTrade, BTContainer, _build_backtest_dataframe, + _get_frame_time_from_offset) import arrow import numpy as np import math from unittest.mock import MagicMock - # Cases to be tested: # 1) Open trade should be removed from the end # 2) Two complete trades within dataframe (with sell hit for all) @@ -25,6 +28,101 @@ ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} +# Open trade should be removed from the end +tc0 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 1]], # enter trade (signal on last candle) + stop_loss=-0.99, roi=float('inf'), profit_perc=0.00, + trades=[] +) + +# Two complete trades within dataframe(with sell hit for all) +tc1 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 1], # enter trade (signal on last candle) + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], # exit at open + [3, 5000, 5025, 4975, 4987, 6172, 1, 0], # no action + [4, 5000, 5025, 4975, 4987, 6172, 0, 0], # should enter the trade + [5, 5000, 5025, 4975, 4987, 6172, 0, 1], # no action + [6, 5000, 5025, 4975, 4987, 6172, 0, 0], # should sell +], + stop_loss=-0.99, roi=float('inf'), profit_perc=0.00, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=2), + BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=4, close_tick=6)] +) + +# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss +tc2 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4600, 4987, 6172, 0, 0], # enter trade, stoploss hit + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.01, roi=float('inf'), profit_perc=-0.01, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss +tc3 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4800, 4987, 6172, 0, 0], # enter trade, stoploss hit + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +#5) Stoploss and sell are hit. should sell on stoploss +tc4=BTContainer(data = [ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4800, 4987, 6172, 0, 1], # enter trade, stoploss hit, sell signal + [2, 5000, 5025, 4975, 4987, 6172, 0, 0], +], + stop_loss = -0.03, roi = float('inf'), profit_perc = -0.03, + trades = [BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] +) + +TESTS = [ + tc0, + tc1, + tc2, + tc3, + tc4 +] + + +@pytest.mark.parametrize("data", TESTS) +def test_edge_results(edge_conf, mocker, caplog, data) -> None: + """ + run functional tests + """ + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + frame = _build_backtest_dataframe(data.data) + caplog.set_level(logging.DEBUG) + edge.fee = 0 + + trades = edge._find_trades_for_stoploss_range(frame, 'TEST/BTC', [data.stop_loss]) + results = edge._fill_calculable_fields(DataFrame(trades)) if trades else DataFrame() + + print(results) + + assert len(trades) == len(data.trades) + + if not results.empty: + assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) + + for c, trade in enumerate(data.trades): + res = results.iloc[c] + assert res.exit_type == trade.sell_reason + assert res.open_time == _get_frame_time_from_offset(trade.open_tick) + assert res.close_time == _get_frame_time_from_offset(trade.close_tick) + + def test_adjust(mocker, default_conf): freqtrade = get_patched_freqtradebot(mocker, default_conf) edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) @@ -94,10 +192,10 @@ def _time_on_candle(number): minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') -def test_edge_heartbeat_calculate(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - heartbeat = default_conf['edge']['process_throttle_secs'] +def test_edge_heartbeat_calculate(mocker, edge_conf): + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + heartbeat = edge_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 @@ -148,15 +246,15 @@ def test_edge_process_downloaded_data(mocker, default_conf): assert edge._last_updated <= arrow.utcnow().timestamp + 2 -def test_process_expectancy(mocker, default_conf): - default_conf['edge']['min_trade_number'] = 2 - freqtrade = get_patched_freqtradebot(mocker, default_conf) +def test_process_expectancy(mocker, edge_conf): + edge_conf['edge']['min_trade_number'] = 2 + freqtrade = get_patched_freqtradebot(mocker, edge_conf) def get_fee(): return 0.001 freqtrade.exchange.get_fee = get_fee - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) trades = [ {'pair': 'TEST/BTC', @@ -210,157 +308,3 @@ def test_process_expectancy(mocker, default_conf): assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 - -# 1) Open trade should be removed from the end - - -def test_case_1(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.99 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [3, 1, 12, 25, 11, 20, 0], # -> - [4, 0, 20, 30, 19, 25, 1], # -> should enter the trade - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # No trade should be found - assert len(trades) == 0 - - -# 2) Two complete trades within dataframe (with sell hit for all) -def test_case_2(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.99 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 17, 18, 13, 14, 1], # -> should enter the trade as B signal recieved on last candle - [2, 0, 14, 15, 11, 12, 0], # -> exit the trade as the sell signal recieved on last candle - [3, 1, 12, 25, 11, 20, 0], # -> no action - [4, 0, 20, 30, 19, 25, 0], # -> should enter the trade - [5, 0, 25, 27, 22, 26, 1], # -> no action - [6, 0, 26, 36, 25, 35, 0], # -> should sell - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 2 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(2) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == ticker[2][_ohlc['open']] - assert trades[0]['exit_type'] == SellType.SELL_SIGNAL - ############################################################## - - # Second trade check - assert trades[1]['open_time'] == _time_on_candle(4) - assert trades[1]['close_time'] == _time_on_candle(6) - assert trades[1]['open_rate'] == ticker[4][_ohlc['open']] - assert trades[1]['close_rate'] == ticker[6][_ohlc['open']] - assert trades[1]['exit_type'] == SellType.SELL_SIGNAL - ############################################################## - - -# 3) Entered, sl 1%, candle drops 8% => Trade closed, 1% loss -def test_case_3(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.01 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 14, 15, 11, 12, 0], # -> enter to trade, stoploss hit - [2, 1, 12, 25, 11, 20, 0], # -> no action - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 1 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(1) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] - assert trades[0]['exit_type'] == SellType.STOP_LOSS - ############################################################## - - -# 4) Entered, sl 3 %, candle drops 4%, recovers to 1 % = > Trade closed, 3 % loss -def test_case_4(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.03 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade - [2, 0, 16, 17, 14.4, 15.5, 0], # -> stoploss hit - [3, 0, 17, 25, 16.9, 22, 0], # -> no action - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 1 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(2) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] - assert trades[0]['exit_type'] == SellType.STOP_LOSS - ############################################################## - - -# 5) Stoploss and sell are hit. should sell on stoploss -def test_case_5(mocker, default_conf): - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - freqtrade = get_patched_freqtradebot(mocker, default_conf) - edge = Edge(default_conf, freqtrade.exchange, freqtrade.strategy) - - stoploss = -0.03 # we don't want stoploss to be hit in this test - ticker = [ - # D=Date, B=Buy, O=Open, H=High, L=Low, C=Close, S=Sell - # D, B, O, H, L, C, S - [0, 1, 15, 20, 12, 17, 0], # -> no action - [1, 0, 17, 22, 16.90, 17, 0], # -> enter to trade - [2, 0, 16, 17, 14.4, 15.5, 1], # -> stoploss hit and also sell signal - [3, 0, 17, 25, 16.9, 22, 0], # -> no action - ] - - ticker_df = _build_dataframe(ticker) - trades = edge._find_trades_for_stoploss_range(ticker_df, 'TEST/BTC', [stoploss]) - - # Two trades must have occured - assert len(trades) == 1 - - # First trade check - assert trades[0]['open_time'] == _time_on_candle(1) - assert trades[0]['close_time'] == _time_on_candle(2) - assert trades[0]['open_rate'] == ticker[1][_ohlc['open']] - assert trades[0]['close_rate'] == (stoploss + 1) * trades[0]['open_rate'] - assert trades[0]['exit_type'] == SellType.STOP_LOSS - ############################################################## diff --git a/freqtrade/tests/optimize/test_backtest_detail.py b/freqtrade/tests/optimize/test_backtest_detail.py index 806c136bc..eaec3bf49 100644 --- a/freqtrade/tests/optimize/test_backtest_detail.py +++ b/freqtrade/tests/optimize/test_backtest_detail.py @@ -1,4 +1,4 @@ -# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument +# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, C0330, unused-argument import logging from unittest.mock import MagicMock @@ -34,15 +34,15 @@ tc0 = BTContainer(data=[ # TC2: Stop-Loss Triggered 3% Loss tc1 = BTContainer(data=[ # D O H L C V B S - [0, 5000, 5025, 4975, 4987, 6172, 1, 0], - [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) - [2, 4987, 5012, 4962, 4975, 6172, 0, 0], - [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit - [4, 4962, 4987, 4937, 4950, 6172, 0, 0], - [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4962, 4975, 6172, 0, 0], + [3, 4975, 5000, 4800, 4962, 6172, 0, 0], # exit with stoploss hit + [4, 4962, 4987, 4937, 4950, 6172, 0, 0], + [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.03, roi=1, profit_perc=-0.03, trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] - ) +) # Test 3 Candle drops 4%, Recovers 1%. @@ -127,7 +127,7 @@ tc6 = BTContainer(data=[ [5, 4950, 4975, 4925, 4950, 6172, 0, 0]], stop_loss=-0.02, roi=0.03, profit_perc=0.03, trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] - ) +) TESTS = [ tc0, From 617a58402f82b4cb19623a6d7fe7260a901eadfb Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 9 Nov 2018 20:52:03 +0100 Subject: [PATCH 160/174] putting edge adjust function in _process not in create_trade --- freqtrade/freqtradebot.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index babe0a1da..4a17e889e 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -192,6 +192,7 @@ class FreqtradeBot(object): # with delta value (klines only from last refresh_pairs) if self.edge: self.edge.calculate() + self.active_pair_whitelist = self.edge.adjust(self.active_pair_whitelist) # Refreshing candles self.exchange.refresh_tickers(self.active_pair_whitelist, self.strategy.ticker_interval) @@ -406,10 +407,6 @@ class FreqtradeBot(object): raise DependencyException('No currency pairs in whitelist') # running get_signal on historical data fetched - # to find buy signals - if self.edge: - whitelist = self.edge.adjust(whitelist) - for _pair in whitelist: (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair)) if buy and not sell: From 4dcd15da1d7e393f593b78548f525511df6986c8 Mon Sep 17 00:00:00 2001 From: misagh Date: Fri, 9 Nov 2018 20:59:28 +0100 Subject: [PATCH 161/174] improving documentation for positioning --- docs/edge.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/edge.md b/docs/edge.md index 5cace4ea3..dc4fda72a 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -88,6 +88,10 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** +Example: +Let's say your total capital is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
+**Notice:** if Edge is enabled, the stake_amount config is overriden by total_capital_in_stake_currency config explained below (see configuration part) + ## Configurations Edge has following configurations: From 0f2ddbbef2f521ec0c791e55884d1d761f5f30c5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 10 Nov 2018 13:34:06 +0100 Subject: [PATCH 162/174] Update ccxt from 1.17.485 to 1.17.488 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4e38f66de..4007cc0ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.485 +ccxt==1.17.488 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 523a9a603cedb0df6e5672937cc2d135b4dbf379 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 17:20:11 +0100 Subject: [PATCH 163/174] fix tests --- freqtrade/tests/conftest.py | 3 ++- freqtrade/tests/edge/test_edge.py | 8 ++++---- freqtrade/tests/optimize/__init__.py | 2 +- freqtrade/tests/test_freqtradebot.py | 27 ++++++++++----------------- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index cbaca6c04..b93140860 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -780,10 +780,11 @@ def buy_order_fee(): 'fee': None } + @pytest.fixture(scope="function") def edge_conf(default_conf): default_conf['edge'] = { - "enabled": False, + "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, "total_capital_in_stake_currency": 0.5, diff --git a/freqtrade/tests/edge/test_edge.py b/freqtrade/tests/edge/test_edge.py index aa45ddfff..a7b1882a5 100644 --- a/freqtrade/tests/edge/test_edge.py +++ b/freqtrade/tests/edge/test_edge.py @@ -75,15 +75,15 @@ tc3 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] ) -#5) Stoploss and sell are hit. should sell on stoploss -tc4=BTContainer(data = [ +# 5) Stoploss and sell are hit. should sell on stoploss +tc4 = BTContainer(data=[ # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], [1, 5000, 5025, 4800, 4987, 6172, 0, 1], # enter trade, stoploss hit, sell signal [2, 5000, 5025, 4975, 4987, 6172, 0, 0], ], - stop_loss = -0.03, roi = float('inf'), profit_perc = -0.03, - trades = [BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] + stop_loss=-0.03, roi=float('inf'), profit_perc=-0.03, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=1)] ) TESTS = [ diff --git a/freqtrade/tests/optimize/__init__.py b/freqtrade/tests/optimize/__init__.py index 2b7222e88..3d3066950 100644 --- a/freqtrade/tests/optimize/__init__.py +++ b/freqtrade/tests/optimize/__init__.py @@ -31,7 +31,7 @@ class BTContainer(NamedTuple): def _get_frame_time_from_offset(offset): return ticker_start_time.shift( - minutes=(offset * ticker_interval_in_minute)).datetime + minutes=(offset * ticker_interval_in_minute)).datetime.replace(tzinfo=None) def _build_backtest_dataframe(ticker_with_signals): diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 9c72f9177..907d228d6 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -246,12 +246,11 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None -def test_edge_overrides_stake_amount(mocker, default_conf) -> None: - default_conf['edge']['enabled'] = True +def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(edge_conf) # strategy stoploss should be ignored freqtrade.strategy.stoploss = -0.05 @@ -261,14 +260,8 @@ def test_edge_overrides_stake_amount(mocker, default_conf) -> None: assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 -def test_edge_overrides_stoploss( - limit_buy_order, - fee, - markets, - caplog, - mocker, - default_conf) -> None: - default_conf['edge']['enabled'] = True +def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: + patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -292,7 +285,8 @@ def test_edge_overrides_stoploss( ############################################# # Create a trade with "limit_buy_order" price - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(edge_conf) + freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -302,14 +296,12 @@ def test_edge_overrides_stoploss( # stoploss shoud be hit assert freqtrade.handle_trade(trade) is True - assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog.record_tuples) assert trade.sell_reason == SellType.STOP_LOSS.value def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, - mocker, default_conf) -> None: - default_conf['edge']['enabled'] = True + mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) @@ -333,7 +325,8 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, ############################################# # Create a trade with "limit_buy_order" price - freqtrade = FreqtradeBot(default_conf) + freqtrade = FreqtradeBot(edge_conf) + freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = lambda trade, current_profit, current_time: False freqtrade.create_trade() @@ -341,7 +334,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, markets, trade.update(limit_buy_order) ############################################# - # stoploss shoud be hit + # stoploss shoud not be hit assert freqtrade.handle_trade(trade) is False From 97fd33d75267941102faa7a2ebb75defedb99cae Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:03:46 +0100 Subject: [PATCH 164/174] adding test for process --- freqtrade/tests/test_freqtradebot.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 907d228d6..e83163a30 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -246,6 +246,20 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, assert result is None +def test_edge_called_in_process(mocker, edge_conf) -> None: + patch_RPCManager(mocker) + patch_edge(mocker) + def _refresh_whitelist(list): + return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] + + patch_exchange(mocker) + freqtrade = FreqtradeBot(edge_conf) + freqtrade._refresh_whitelist = _refresh_whitelist + patch_get_signal(freqtrade) + freqtrade._process() + assert freqtrade.active_pair_whitelist == ['NEO/BTC', 'LTC/BTC'] + + def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) From 7dd74c374a5bc4ca6897768edff6c7bee5dd8e1f Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:09:32 +0100 Subject: [PATCH 165/174] flake happiness provided --- freqtrade/tests/test_freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index e83163a30..66c5d6a1c 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -249,6 +249,7 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, def test_edge_called_in_process(mocker, edge_conf) -> None: patch_RPCManager(mocker) patch_edge(mocker) + def _refresh_whitelist(list): return ['ETH/BTC', 'LTC/BTC', 'XRP/BTC', 'NEO/BTC'] From d61355330679a8e45949c141b739261ba426440b Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:22:34 +0100 Subject: [PATCH 166/174] base position on stake amount instead of total capital --- freqtrade/edge/__init__.py | 2 +- freqtrade/tests/test_freqtradebot.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index a468a76a3..31d70add1 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -49,7 +49,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._total_capital: float = self.edge_config.get('total_capital_in_stake_currency') + self._total_capital: float = self.config.get('stake_amount') self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time diff --git a/freqtrade/tests/test_freqtradebot.py b/freqtrade/tests/test_freqtradebot.py index 66c5d6a1c..4cba9e308 100644 --- a/freqtrade/tests/test_freqtradebot.py +++ b/freqtrade/tests/test_freqtradebot.py @@ -267,12 +267,8 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: patch_edge(mocker) freqtrade = FreqtradeBot(edge_conf) - # strategy stoploss should be ignored - freqtrade.strategy.stoploss = -0.05 - - assert 'ETH/BTC' not in freqtrade.edge._cached_pairs - assert freqtrade._get_trade_stake_amount('NEO/BTC') == 0.025 - assert freqtrade._get_trade_stake_amount('LTC/BTC') == 0.02381 + assert freqtrade._get_trade_stake_amount('NEO/BTC') == (0.001 * 0.01) / 0.20 + assert freqtrade._get_trade_stake_amount('LTC/BTC') == (0.001 * 0.01) / 0.20 def test_edge_overrides_stoploss(limit_buy_order, fee, markets, caplog, mocker, edge_conf) -> None: From aacc1d5004004e47627de66739167a963418cecc Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:28:05 +0100 Subject: [PATCH 167/174] removing total capital in favour of stake amount --- config_full.json.example | 1 - docs/edge.md | 8 +++----- freqtrade/constants.py | 1 - freqtrade/tests/conftest.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index a7fdd5bab..9dba8f539 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -63,7 +63,6 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 2, - "total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/docs/edge.md b/docs/edge.md index dc4fda72a..cfb95b936 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -82,6 +82,8 @@ Allowed capital at risk is calculated as follows: **allowed capital at risk** = **total capital** X **allowed risk per trade** +**total capital** is your stake amount. + **Stoploss** is calculated as described above against historical data. Your position size then will be: @@ -89,8 +91,7 @@ Your position size then will be: **position size** = **allowed capital at risk** / **stoploss** Example: -Let's say your total capital is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
-**Notice:** if Edge is enabled, the stake_amount config is overriden by total_capital_in_stake_currency config explained below (see configuration part) +Let's say your stake amount is 3 ETH, you would allow 1% of risk for each trade. thus your allowed capital at risk would be **3 x 0.01 = 0.03 ETH**. Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.03 / 0.02= 1.5ETH**.
## Configurations Edge has following configurations: @@ -108,9 +109,6 @@ Number of days of data against which Edge calculates Win Rate, Risk Reward and E Note that it downloads historical data so increasing this number would lead to slowing down the bot
(default to 7) -#### total_capital_in_stake_currency -This your total capital at risk in your stake currency. If edge is enabled then stake_amount is ignored in favor of this parameter - #### allowed_risk Percentage of allowed risk per trade
(default to 1%) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index d37e78687..b7c069c45 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -178,7 +178,6 @@ CONF_SCHEMA = { "enabled": {'type': 'boolean'}, "process_throttle_secs": {'type': 'integer', 'minimum': 600}, "calculate_since_number_of_days": {'type': 'integer'}, - "total_capital_in_stake_currency": {'type': 'number'}, "allowed_risk": {'type': 'number'}, "stoploss_range_min": {'type': 'number'}, "stoploss_range_max": {'type': 'number'}, diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index b93140860..487d0d150 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -787,7 +787,7 @@ def edge_conf(default_conf): "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, - "total_capital_in_stake_currency": 0.5, + #"total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From 9bbaeb4e6f59aa89f9c314c75114ef1722c32a71 Mon Sep 17 00:00:00 2001 From: misagh Date: Sat, 10 Nov 2018 18:39:49 +0100 Subject: [PATCH 168/174] mypy expression --- freqtrade/edge/__init__.py | 2 +- freqtrade/tests/conftest.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index 31d70add1..7b25a8306 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -49,7 +49,7 @@ class Edge(): self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._total_capital: float = self.config.get('stake_amount') + self._total_capital: float = self.config['stake_amount'] self._allowed_risk: float = self.edge_config.get('allowed_risk') self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) self._last_updated: int = 0 # Timestamp of pairs last updated time diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 487d0d150..8a497725f 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -787,7 +787,6 @@ def edge_conf(default_conf): "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, - #"total_capital_in_stake_currency": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, From ef2c31b5434d0822af6a7f7219fb046c96fa5905 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Nov 2018 13:34:06 +0100 Subject: [PATCH 169/174] Update ccxt from 1.17.488 to 1.17.489 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4007cc0ac..ecc6cb682 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.488 +ccxt==1.17.489 SQLAlchemy==1.2.13 python-telegram-bot==11.1.0 arrow==0.12.1 From 261cd7746bb21b9c4071669ce24ad00c5786e39a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sun, 11 Nov 2018 13:34:07 +0100 Subject: [PATCH 170/174] Update sqlalchemy from 1.2.13 to 1.2.14 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ecc6cb682..f0fe0e05d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ ccxt==1.17.489 -SQLAlchemy==1.2.13 +SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 cachetools==3.0.0 From c29543dd6c74bee082c3da42d82d67e93244ec85 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 12 Nov 2018 13:34:07 +0100 Subject: [PATCH 171/174] Update ccxt from 1.17.489 to 1.17.491 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f0fe0e05d..b48679838 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.489 +ccxt==1.17.491 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1 From 028139fa3aea004a5af5b97410280ef382b90383 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 12 Nov 2018 13:34:08 +0100 Subject: [PATCH 172/174] Update pytest from 3.10.0 to 3.10.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b48679838..f79e425b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ scipy==1.1.0 jsonschema==2.6.0 numpy==1.15.4 TA-Lib==0.4.17 -pytest==3.10.0 +pytest==3.10.1 pytest-mock==1.10.0 pytest-asyncio==0.9.0 pytest-cov==2.6.0 From bdba6186d8e9d4408fbd7155922cf969e3e4f678 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 12 Nov 2018 19:43:20 +0100 Subject: [PATCH 173/174] Fix doc-typos --- docs/edge.md | 4 ++-- docs/index.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/edge.md b/docs/edge.md index cfb95b936..f74f8fdcc 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -111,7 +111,7 @@ Note that it downloads historical data so increasing this number would lead to s #### allowed_risk Percentage of allowed risk per trade
-(default to 1%) +(default to 0.01 [1%]) #### stoploss_range_min Minimum stoploss
@@ -148,4 +148,4 @@ Edge will filter out trades with long duration. If a trade is profitable after 1 #### remove_pumps Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
-(default to false) \ No newline at end of file +(default to false) diff --git a/docs/index.md b/docs/index.md index 879ee4f80..e6e643ba7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ Pull-request. Do not hesitate to reach us on - [Change your strategy](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#change-your-strategy) - [Add more Indicator](https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md#add-more-indicator) - [Test your strategy with Backtesting](https://github.com/freqtrade/freqtrade/blob/develop/docs/backtesting.md) - - [Edge positionning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) + - [Edge positioning](https://github.com/mishaker/freqtrade/blob/money_mgt/docs/edge.md) - [Find optimal parameters with Hyperopt](https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md) - [Control the bot with telegram](https://github.com/freqtrade/freqtrade/blob/develop/docs/telegram-usage.md) - [Receive notifications via webhook](https://github.com/freqtrade/freqtrade/blob/develop/docs/webhook-config.md) From 4e64bc3d29e6925c68c10e9b76a7ddd48c430937 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 13 Nov 2018 13:34:07 +0100 Subject: [PATCH 174/174] Update ccxt from 1.17.491 to 1.17.492 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f79e425b6..2c0011aea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ccxt==1.17.491 +ccxt==1.17.492 SQLAlchemy==1.2.14 python-telegram-bot==11.1.0 arrow==0.12.1