Merge pull request #1113 from berlinguyinca/backslap_develop

Backslap develop

freqtrade/arguments.py

@@ -161,6 +161,14 @@ class Arguments(object):
             dest='exportfilename',
             metavar='PATH',
         )
+        parser.add_argument(
+            '--backslap',
+            help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more "
+                 "accurate results, unless you are utilizing Min/Max function in your strategy.",
+            required=False,
+            dest='backslap',
+            action='store_true'
+        )
 
     @staticmethod
     def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:

freqtrade/optimize/__init__.py

@@ -10,9 +10,16 @@ import arrow
 from freqtrade import misc, constants, OperationalException
 from freqtrade.exchange import Exchange
 from freqtrade.arguments import TimeRange
+import importlib
+ujson_found = importlib.util.find_spec("ujson")
+if ujson_found is not None:
+    import ujson
 
 logger = logging.getLogger(__name__)
 
+if ujson_found is not None:
+    logger.debug('Loaded UltraJson ujson in optimize.py')
+
 
 def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
     if not tickerlist:
@@ -63,10 +70,16 @@ def load_tickerdata_file(
     if os.path.isfile(gzipfile):
         logger.debug('Loading ticker data from file %s', gzipfile)
         with gzip.open(gzipfile) as tickerdata:
+            if ujson_found is not None:
+                pairdata = ujson.load(tickerdata, precise_float=True)
+            else:
                 pairdata = json.load(tickerdata)
     elif os.path.isfile(file):
         logger.debug('Loading ticker data from file %s', file)
         with open(file) as tickerdata:
+            if ujson_found is not None:
+                pairdata = ujson.load(tickerdata, precise_float=True)
+            else:
                 pairdata = json.load(tickerdata)
     else:
         return None
@@ -163,6 +176,9 @@ def load_cached_data_for_updating(filename: str,
     # read the cached file
     if os.path.isfile(filename):
         with open(filename, "rt") as file:
+            if ujson_found is not None:
+                data = ujson.load(file, precise_float=True)
+            else:
                 data = json.load(file)
             # remove the last item, because we are not sure if it is correct
             # it could be fetched when the candle was incompleted

freqtrade/optimize/backslapping.py

@@ -0,0 +1,789 @@
+import timeit
+from typing import Dict, Any
+
+from pandas import DataFrame
+
+from freqtrade.exchange import Exchange
+from freqtrade.strategy import IStrategy
+from freqtrade.strategy.interface import SellType
+from freqtrade.strategy.resolver import StrategyResolver
+
+
+class Backslapping:
+    """
+    provides a quick way to evaluate strategies over a longer term of time
+    """
+
+    def __init__(self, config: Dict[str, Any], exchange = None) -> None:
+        """
+        constructor
+        """
+
+        self.config = config
+        self.strategy: IStrategy = StrategyResolver(self.config).strategy
+        self.ticker_interval = self.strategy.ticker_interval
+        self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe
+        self.populate_buy_trend = self.strategy.populate_buy_trend
+        self.populate_sell_trend = self.strategy.populate_sell_trend
+
+        ###
+        #
+        ###
+        if exchange is None:
+            self.config['exchange']['secret'] = ''
+            self.config['exchange']['password'] = ''
+            self.config['exchange']['uid'] = ''
+            self.config['dry_run'] = True
+            self.exchange = Exchange(self.config)
+        else:
+            self.exchange = exchange
+
+        self.fee = self.exchange.get_fee()
+
+        self.stop_loss_value = self.strategy.stoploss
+
+        #### backslap config
+        '''
+        Numpy arrays are used for 100x speed up
+        We requires setting Int values for
+        buy stop triggers and stop calculated on
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
+        '''
+        self.np_buy: int = 0
+        self.np_open: int = 1
+        self.np_close: int = 2
+        self.np_sell: int = 3
+        self.np_high: int = 4
+        self.np_low: int = 5
+        self.np_stop: int = 6
+        self.np_bto: int = self.np_close  # buys_triggered_on - should be close
+        self.np_bco: int = self.np_open  # buys calculated on - open of the next candle.
+        self.np_sto: int = self.np_low  # stops_triggered_on - Should be low, FT uses close
+        self.np_sco: int = self.np_stop  # stops_calculated_on - Should be stop, FT uses close
+        # self.np_sto: int = self.np_close  # stops_triggered_on - Should be low, FT uses close
+        # self.np_sco: int = self.np_close  # stops_calculated_on - Should be stop, FT uses close
+
+        self.debug = False  # Main debug enable, very print heavy, enable 2 loops recommended
+        self.debug_timing = False  # Stages within Backslap
+        self.debug_2loops = False  # Limit each pair to two loops, useful when debugging
+        self.debug_vector = False  # Debug vector calcs
+        self.debug_timing_main_loop = False  # print overall timing per pair - works in Backtest and Backslap
+
+        self.backslap_show_trades = False  # prints trades in addition to summary report
+        self.backslap_save_trades = True  # saves trades as a pretty table to backslap.txt
+
+        self.stop_stops: int = 9999  # stop back testing any pair with this many stops, set to 999999 to not hit
+
+    def s(self):
+        st = timeit.default_timer()
+        return st
+
+    def f(self, st):
+        return (timeit.default_timer() - st)
+
+    def run(self,args):
+
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
+        processed = args['processed']
+        max_open_trades = args.get('max_open_trades', 0)
+        realistic = args.get('realistic', False)
+        trades = []
+        trade_count_lock: Dict = {}
+
+        ########################### Call out BSlap Loop instead of Original BT code
+        bslap_results: list = []
+        for pair, pair_data in processed.items():
+            if self.debug_timing:  # Start timer
+                fl = self.s()
+
+            ticker_data = self.populate_sell_trend(
+                self.populate_buy_trend(pair_data))[headers].copy()
+
+            if self.debug_timing:  # print time taken
+                flt = self.f(fl)
+                # print("populate_buy_trend:", pair, round(flt, 10))
+                st = self.s()
+
+            # #dump same DFs to disk for offline testing in scratch
+            # f_pair:str = pair
+            # csv = f_pair.replace("/", "_")
+            # csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv
+            # ticker_data.to_csv(csv, sep='\t', encoding='utf-8')
+
+            # call bslap - results are a list of dicts
+            bslap_pair_results = self.backslap_pair(ticker_data, pair)
+            last_bslap_results = bslap_results
+            bslap_results = last_bslap_results + bslap_pair_results
+
+            if self.debug_timing:  # print time taken
+                tt = self.f(st)
+                print("Time to  BackSlap :", pair, round(tt, 10))
+                print("-----------------------")
+
+        # Switch List of Trade Dicts (bslap_results) to Dataframe
+        # Fill missing, calculable columns, profit, duration , abs etc.
+        bslap_results_df = DataFrame(bslap_results)
+
+        if len(bslap_results_df) > 0:  # Only post process a frame if it has a record
+            # bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time'])
+            # bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time'])
+            # if debug:
+            #     print("open_time and close_time converted to datetime columns")
+
+            bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair)
+        else:
+            from freqtrade.optimize.backtesting import BacktestResult
+
+            bslap_results_df = []
+            bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields)
+
+        return bslap_results_df
+
+    def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str):
+        """
+        The Results frame contains a number of columns that are calculable
+        from othe columns. These are left blank till all rows are added,
+        to be populated in single vector calls.
+
+        Columns to be populated are:
+        - Profit
+        - trade duration
+        - profit abs
+        :param bslap_results Dataframe
+        :return: bslap_results Dataframe
+        """
+        import pandas as pd
+        import numpy as np
+        debug = self.debug_vector
+
+        # stake and fees
+        # stake = 0.015
+        # 0.05% is 0.0005
+        # fee = 0.001
+
+        stake = self.config.get('stake_amount')
+        fee = self.fee
+        open_fee = fee / 2
+        close_fee = fee / 2
+
+        if debug:
+            print("Stake is,", stake, "the sum of currency to spend per trade")
+            print("The open fee is", open_fee, "The close fee is", close_fee)
+        if debug:
+            from pandas import set_option
+            set_option('display.max_rows', 5000)
+            set_option('display.max_columns', 20)
+            pd.set_option('display.width', 1000)
+            pd.set_option('max_colwidth', 40)
+            pd.set_option('precision', 12)
+
+        # # Get before
+        # csv = "cryptosher_before_debug"
+        # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8')
+
+        # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8')
+
+        bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time']
+        bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60))
+
+        ## Spends, Takes, Profit, Absolute Profit
+        # print(bslap_results_df)
+        # Buy Price
+        bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate']  # How many target are we buying
+        bslap_results_df['buy_fee'] = stake * open_fee
+        bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee']  # How much we're spending
+
+        # Sell price
+        bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate']
+        bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee
+        bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee']
+        # profit_percent
+        bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \
+                                             / bslap_results_df['buy_spend']
+        # Absolute profit
+        bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend']
+
+        # # Get After
+        # csv="cryptosher_after_debug"
+        # bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8')
+
+        if debug:
+            print("\n")
+            print(bslap_results_df[
+                      ['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent',
+                       'profit_abs', 'exit_type']])
+
+        return bslap_results_df
+
+    def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int,
+                          stop_stops_count: int):
+        import utils_find_1st as utf1st
+        """
+         The purpose of this def is to return the next "buy" = 1
+         after t_exit_ind.
+         
+         This function will also check is the stop limit for the pair has been reached. 
+         if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit.
+
+         t_exit_ind is the index the last trade exited on
+         or 0 if first time around this loop.
+         
+         stop_stops i
+         """
+        debug = self.debug
+
+        # Timers, to be called if in debug
+        def s():
+            st = timeit.default_timer()
+            return st
+
+        def f(st):
+            return (timeit.default_timer() - st)
+
+        st = s()
+        t_open_ind: int
+
+        """
+        Create a view on our buy index starting after last trade exit
+        Search for next buy
+        """
+        np_buy_arr_v = np_buy_arr[t_exit_ind:]
+        t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal)
+
+        '''
+        If -1 is returned no buy has been found, preserve the value
+        '''
+        if t_open_ind != -1:  # send back the -1 if no buys found. otherwise update index
+            t_open_ind = t_open_ind + t_exit_ind  # Align numpy index
+
+        if t_open_ind == np_buy_arr_len - 1:  # If buy found on last candle ignore, there is no OPEN in next to use
+            t_open_ind = -1  # -1 ends the loop
+
+        if stop_stops_count >= stop_stops:  # if maximum number of stops allowed in a pair is hit, exit loop
+            t_open_ind = -1  # -1 ends the loop
+            if debug:
+                print("Max stop limit ", stop_stops, "reached. Moving to next pair")
+
+        return t_open_ind
+
+    def backslap_pair(self, ticker_data, pair):
+        import pandas as pd
+        import numpy as np
+        import timeit
+        import utils_find_1st as utf1st
+        from datetime import datetime
+
+        ### backslap debug wrap
+        # debug_2loops = False  # only loop twice, for faster debug
+        # debug_timing = False  # print timing for each step
+        # debug = False  # print values, to check accuracy
+        debug_2loops = self.debug_2loops  # only loop twice, for faster debug
+        debug_timing = self.debug_timing  # print timing for each step
+        debug = self.debug  # print values, to check accuracy
+
+        # Read Stop Loss Values and Stake
+        stop = self.stop_loss_value
+        p_stop = (stop + 1)  # What stop really means, e.g 0.01 is 0.99 of price
+
+        if debug:
+            print("Stop is ", stop, "value from stragey file")
+            print("p_stop is", p_stop, "value used to multiply to entry price")
+
+        if debug:
+            from pandas import set_option
+            set_option('display.max_rows', 5000)
+            set_option('display.max_columns', 8)
+            pd.set_option('display.width', 1000)
+            pd.set_option('max_colwidth', 40)
+            pd.set_option('precision', 12)
+
+        def s():
+            st = timeit.default_timer()
+            return st
+
+        def f(st):
+            return (timeit.default_timer() - st)
+
+        #### backslap config
+        '''
+        Numpy arrays are used for 100x speed up
+        We requires setting Int values for
+        buy stop triggers and stop calculated on
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
+        '''
+
+        #######
+        #  Use vars set at top of backtest
+        np_buy: int = self.np_buy
+        np_open: int = self.np_open
+        np_close: int = self.np_close
+        np_sell: int = self.np_sell
+        np_high: int = self.np_high
+        np_low: int = self.np_low
+        np_stop: int = self.np_stop
+        np_bto: int = self.np_bto  # buys_triggered_on - should be close
+        np_bco: int = self.np_bco  # buys calculated on - open of the next candle.
+        np_sto: int = self.np_sto  # stops_triggered_on - Should be low, FT uses close
+        np_sco: int = self.np_sco  # stops_calculated_on - Should be stop, FT uses close
+
+        ### End Config
+
+        pair: str = pair
+
+        # ticker_data: DataFrame = ticker_dfs[t_file]
+        bslap: DataFrame = ticker_data
+
+        # Build a single dimension numpy array from "buy" index for faster search
+        # (500x faster than pandas)
+        np_buy_arr = bslap['buy'].values
+        np_buy_arr_len: int = len(np_buy_arr)
+
+        # use numpy array for faster searches in loop, 20x faster than pandas
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+        np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']])
+
+        # Build a numpy list of date-times.
+        # We use these when building the trade
+        # The rationale is to address a value from a pandas cell is thousands of
+        # times more expensive. Processing time went X25 when trying to use any data from pandas
+        np_bslap_dates = bslap['date'].values
+
+        loop: int = 0  # how many time around the loop
+        t_exit_ind = 0  # Start loop from first index
+        t_exit_last = 0  # To test for exit
+
+        stop_stops = self.stop_stops  # Int of stops within a pair to stop trading a pair at
+        stop_stops_count = 0  # stop counter per pair
+
+        st = s()  # Start timer for processing dataframe
+        if debug:
+            print('Processing:', pair)
+
+        # Results will be stored in a list of dicts
+        bslap_pair_results: list = []
+        bslap_result: dict = {}
+
+        while t_exit_ind < np_buy_arr_len:
+            loop = loop + 1
+            if debug or debug_timing:
+                print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair)
+            if debug_2loops:
+                if loop == 3:
+                    print(
+                        "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking")
+                    break
+            '''
+             Dev phases
+             Phase 1
+              1) Manage buy, sell, stop enter/exit
+               a) Find first buy index
+               b) Discover first stop and sell hit after buy index
+               c) Chose first instance as trade exit
+    
+             Phase 2
+              2) Manage dynamic Stop and ROI Exit
+               a) Create trade slice from 1
+               b) search within trade slice for dynamice stop hit
+               c) search within trade slice for ROI hit
+             '''
+
+            if debug_timing:
+                st = s()
+            '''
+            0 - Find next buy entry
+            Finds index for first (buy = 1) flag
+    
+            Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1"
+            Required: t_exit_ind - Either 0, first loop. Or The index we last exited on
+            Requires: np_buy_arr_len - length of pair array. 
+            Requires: stops_stops - number of stops allowed before stop trading a pair 
+            Requires: stop_stop_counts - count of stops hit in the pair
+            Provides: The next "buy" index after t_exit_ind
+    
+            If -1 is returned no buy has been found in remainder of array, skip to exit loop
+            '''
+            t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count)
+
+            if debug:
+                print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind)
+                print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop")
+            if debug_timing:
+                t_t = f(st)
+                print("0-numpy", str.format('{0:.17f}', t_t))
+                st = s()
+
+            if t_open_ind != -1:
+
+                """
+                1 - Create views to search within for our open trade
+    
+                The views are our search space for the next Stop or Sell
+                Numpy view is employed as:
+                1,000 faster than pandas searches
+                Pandas cannot assure it will always return a view, it may make a slow copy.
+    
+                The view contains columns:
+                buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+    
+                Requires: np_bslap is our numpy array of the ticker DataFrame
+                Requires: t_open_ind is the index row with the  buy.
+                Provides: np_t_open_v View of array after buy.
+                Provides: np_t_open_v_stop View of array after buy +1  
+                          (Stop will search in here to prevent stopping in the past)
+                """
+                np_t_open_v = np_bslap[t_open_ind:]
+                np_t_open_v_stop = np_bslap[t_open_ind + 1:]
+
+                if debug:
+                    print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind)
+                    print("Numpy View: Buy - Open - Close - Sell - High - Low")
+                    print("Row 0", np_t_open_v[0])
+                    print("Row 1", np_t_open_v[1], )
+                if debug_timing:
+                    t_t = f(st)
+                    print("2-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                2 - Calculate our stop-loss price
+    
+                As stop is based on buy price of our trade
+                - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle
+                - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle.
+                This is as we only see the CLOSE after it has happened.
+                The  back test assumption is we have bought at first available price, the OPEN
+    
+                Requires: np_bslap   - is our numpy array of the ticker DataFrame
+                Requires: t_open_ind - is the index row with the first buy.
+                Requires: p_stop     - is the stop rate, ie. 0.99 is -1%
+                Provides: np_t_stop_pri - The value stop-loss will be triggered on
+                '''
+                np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop)
+
+                if debug:
+                    print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri)
+                if debug_timing:
+                    t_t = f(st)
+                    print("2-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                3 -  Find candle STO is under Stop-Loss After Trade opened.
+    
+                where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri
+    
+                Requires: np_t_open_v_stop   Numpy view of ticker_data after buy row +1 (when trade was opened)
+                Requires: np_sto        User Var(STO)StopTriggeredOn. Typically set to "low" or "close"
+                Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop
+                Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss
+                '''
+                np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto],
+                                                np_t_stop_pri,
+                                                utf1st.cmp_smaller)
+
+                # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out.
+                np_t_stop_ind = np_t_stop_ind + 1
+
+                if debug:
+                    print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is",
+                          np_t_stop_ind - 1,
+                          ". STO is using field", np_sto,
+                          "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n")
+
+                    print(
+                        "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage")
+                    print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind])
+                    print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1])
+                if debug_timing:
+                    t_t = f(st)
+                    print("3-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                4 - Find first sell index after trade open
+    
+                First index in the view np_t_open_v where ['sell'] = 1
+    
+                Requires: np_t_open_v - view of ticker_data from buy onwards
+                Requires: no_sell - integer '3', the buy column in the array
+                Provides: np_t_sell_ind index of view where first sell=1 after buy
+                '''
+                # Use numpy array for faster search for sell
+                # Sell uses column 3.
+                # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+                # Numpy searches 25-35x quicker than pandas on this data
+
+                np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell],
+                                                1, utf1st.cmp_equal)
+                if debug:
+                    print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind)
+                    print("If 0 or less is returned there is no sell found to end of view, then next lines garbage")
+                    print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind])
+                    print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1])
+                if debug_timing:
+                    t_t = f(st)
+                    print("4-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                5 - Determine which was hit first a stop or sell
+                To then use as exit index price-field (sell on buy, stop on stop)
+    
+                STOP takes priority over SELL as would be 'in candle' from tick data
+                Sell would use Open from Next candle.
+                So in a draw Stop would be hit first on ticker data in live
+    
+                Validity of when types of trades may be executed can be summarised as:
+    
+                            Tick	View
+                            index	index	Buy Sell open	low	     close	 high   Stop price
+                open 2am	94	        -1	0	 0	-----	------	 ------  -----   -----
+                open 3am 	95	        0	1	 0	-----	------	 trg buy -----   -----
+                open 4am 	96	        1	0	 1	Enter	trgstop	 trg sel ROI out Stop out
+                open 5am 	97	        2	0	 0	Exit	------	 ------- -----   -----
+                open 6am	98	        3	0	 0	----- 	------	 ------- -----   -----
+    
+                -1 means not found till end of view i.e no valid Stop found. Exclude from match.
+                Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid.
+    
+                Buys and sells are triggered at candle close
+                Both will open their postions at the open of the next candle. i/e  + 1 index
+    
+                Stop and buy Indexes are on the view. To map to the ticker dataframe
+                the t_open_ind index should be summed.
+    
+                np_t_stop_ind: Stop Found index in view
+                t_exit_ind   : Sell found in view
+                t_open_ind   : Where view was started on ticker_data
+    
+                TODO: fix this frig for logic test,, case/switch/dictionary would be better...
+                      more so when later testing many options, dynamic stop / roi etc
+                cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found)
+                cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found)
+    
+                '''
+                if debug:
+                    print("\n(5) numpy debug\nStop or Sell Logic Processing")
+
+                # cludge for logic test (-1) means it was not found, set crazy high to lose < test
+                np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind
+                np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind
+
+                # Stoploss trigger found before a sell =1
+                if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind:
+                    t_exit_ind = t_open_ind + np_t_stop_ind  # Set Exit row index
+                    t_exit_type = SellType.STOP_LOSS  # Set Exit type (stop)
+                    np_t_exit_pri = np_sco  # The price field our STOP exit will use
+                    if debug:
+                        print("Type STOP is first exit condition. "
+                              "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind)
+
+                # Buy = 1 found before a stoploss triggered
+                elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind:
+                    # move sell onto next candle, we only look back on sell
+                    # will use the open price later.
+                    t_exit_ind = t_open_ind + np_t_sell_ind + 1  # Set Exit row index
+                    t_exit_type = SellType.SELL_SIGNAL  # Set Exit type (sell)
+                    np_t_exit_pri = np_open  # The price field our SELL exit will use
+                    if debug:
+                        print("Type SELL is first exit condition. "
+                              "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind)
+
+                # No stop or buy left in view - set t_exit_last -1 to handle gracefully
+                else:
+                    t_exit_last: int = -1  # Signal loop to exit, no buys or sells found.
+                    t_exit_type = SellType.NONE
+                    np_t_exit_pri = 999  # field price should be calculated on. 999 a non-existent column
+                    if debug:
+                        print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit")
+
+                # TODO: fix having to cludge/uncludge this ..
+                # Undo cludge
+                np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind
+                np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind
+
+                if debug_timing:
+                    t_t = f(st)
+                    print("5-logic", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                if debug:
+                    '''
+                    Print out the buys, stops, sells
+                    Include Line before and after to for easy
+                    Human verification
+                    '''
+                    # Combine the np_t_stop_pri value to bslap dataframe to make debug
+                    # life easy. This is the current stop price based on buy price_
+                    # This is slow but don't care about performance in debug
+                    #
+                    # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1
+                    # as there is no data column in the numpy array.
+                    bslap['np_stop_pri'] = np_t_stop_pri
+
+                    # Buy
+                    print("\n\nDATAFRAME DEBUG =================== BUY ", pair)
+                    print("Numpy Array BUY Index is:", 0)
+                    print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n")
+                    print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1)
+                    op_is = t_open_ind - 1  # Print open index start, line before
+                    op_if = t_open_ind + 3  # Print open index finish, line after
+                    print(bslap.iloc[op_is:op_if], "\n")
+
+                    # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On)
+                    if np_t_stop_ind < 0:
+                        print("DATAFRAME DEBUG =================== STOP  ", pair)
+                        print("No STOPS were found until the end of ticker data file\n")
+                    else:
+                        print("DATAFRAME DEBUG =================== STOP  ", pair)
+                        print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind)
+                        df_stop_index = (t_open_ind + np_t_stop_ind)
+
+                        print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n")
+                        print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ",
+                              df_stop_index, ": \n",
+                              str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]),
+                              "is less than", str.format('{0:.17f}', np_t_stop_pri))
+
+                        print("A stoploss exit will be calculated at rate:",
+                              str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1]))
+
+                        print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index,
+                              ": As live STOPs are not linked to O-C times")
+
+                        st_is = df_stop_index - 1  # Print stop index start, line before
+                        st_if = df_stop_index + 2  # Print stop index finish, line after
+                        print(bslap.iloc[st_is:st_if], "\n")
+
+                    # Sell
+                    if np_t_sell_ind < 0:
+                        print("DATAFRAME DEBUG =================== SELL ", pair)
+                        print("No SELLS were found till the end of ticker data file\n")
+                    else:
+                        print("DATAFRAME DEBUG =================== SELL ", pair)
+                        print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind)
+                        df_sell_index = (t_open_ind + np_t_sell_ind)
+
+                        print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n")
+                        print("First Sell Index after Trade open is in candle", df_sell_index)
+                        print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle",
+                              df_sell_index + 1)
+                        sl_is = df_sell_index - 1  # Print sell index start, line before
+                        sl_if = df_sell_index + 3  # Print sell index finish, line after
+                        print(bslap.iloc[sl_is:sl_if], "\n")
+
+                    # Chosen Exit (stop or sell)
+
+                    print("DATAFRAME DEBUG =================== EXIT ", pair)
+                    print("Exit type is :", t_exit_type)
+                    print("trade exit price field is", np_t_exit_pri, "\n")
+
+                if debug_timing:
+                    t_t = f(st)
+                    print("6-depra", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                ## use numpy view "np_t_open_v" for speed. Columns are
+                # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+                # exception is 6 which is use the stop value.
+
+                # TODO no! this is hard coded bleh fix this open
+                np_trade_enter_price = np_bslap[t_open_ind + 1, np_open]
+                if t_exit_type == SellType.STOP_LOSS:
+                    if np_t_exit_pri == 6:
+                        np_trade_exit_price = np_t_stop_pri
+                    else:
+                        np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri]
+                if t_exit_type == SellType.SELL_SIGNAL:
+                    np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri]
+
+                # Catch no exit found
+                if t_exit_type == SellType.NONE:
+                    np_trade_exit_price = 0
+
+                if debug_timing:
+                    t_t = f(st)
+                    print("7-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                if debug:
+                    print("//////////////////////////////////////////////")
+                    print("+++++++++++++++++++++++++++++++++ Trade Enter ")
+                    print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price))
+                    print("--------------------------------- Trade Exit ")
+                    print("Trade Exit Type is ", t_exit_type)
+                    print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price))
+                    print("//////////////////////////////////////////////")
+
+            else:  # no buys were found, step 0 returned -1
+                # Gracefully exit the loop
+                t_exit_last == -1
+                if debug:
+                    print("\n(E) No buys were found in remaining ticker file. Exiting", pair)
+
+            # Loop control - catch no closed trades.
+            if debug:
+                print("---------------------------------------- end of loop", loop,
+                      " Dataframe Exit Index is: ", t_exit_ind)
+                print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind)
+
+            if t_exit_last >= t_exit_ind or t_exit_last == -1:
+                """
+                Break loop and go on to next pair.
+    
+                When last trade exit equals index of last exit, there is no
+                opportunity to close any more trades.
+                """
+                # TODO :add handing here to record none closed open trades
+
+                if debug:
+                    print(bslap_pair_results)
+                break
+            else:
+                """
+                Add trade to backtest looking results list of dicts
+                Loop back to look for more trades.
+                """
+                # Build trade dictionary
+                ## In general if a field can be calculated later from other fields leave blank here
+                ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time
+
+                # create a new dict
+                close_index: int = t_exit_ind
+                bslap_result = {}  # Must have at start or we end up with a list of multiple same last result
+                bslap_result["pair"] = pair
+                bslap_result["profit_percent"] = ""  # To be 1 vector calc across trades when loop complete
+                bslap_result["profit_abs"] = ""  # To be 1 vector calc across trades when loop complete
+                bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1]  # use numpy array, pandas 20x slower
+                bslap_result["close_time"] = np_bslap_dates[close_index]  # use numpy array, pandas 20x slower
+                bslap_result["open_index"] = t_open_ind + 1  # +1 as we buy on next.
+                bslap_result["close_index"] = close_index
+                bslap_result["trade_duration"] = ""  # To be 1 vector calc across trades when loop complete
+                bslap_result["open_at_end"] = False
+                bslap_result["open_rate"] = round(np_trade_enter_price, 15)
+                bslap_result["close_rate"] = round(np_trade_exit_price, 15)
+                bslap_result["exit_type"] = t_exit_type
+                bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care
+                # append the dict to the list and print list
+                bslap_pair_results.append(bslap_result)
+
+                if t_exit_type is SellType.STOP_LOSS:
+                    stop_stops_count = stop_stops_count + 1
+
+                if debug:
+                    print("The trade dict is: \n", bslap_result)
+                    print("Trades dicts in list after append are: \n ", bslap_pair_results)
+
+                """
+                Loop back to start. t_exit_last becomes where loop
+                will seek to open new trades from.
+                Push index on 1 to not open on close
+                """
+                t_exit_last = t_exit_ind + 1
+
+            if debug_timing:
+                t_t = f(st)
+                print("8+trade", str.format('{0:.17f}', t_t))
+
+        # Send back List of trade dicts
+        return bslap_pair_results

freqtrade/optimize/backtesting.py

@@ -10,7 +10,7 @@ from datetime import datetime, timedelta
 from typing import Any, Dict, List, NamedTuple, Optional, Tuple
 
 import arrow
-from pandas import DataFrame
+from pandas import DataFrame, to_datetime
 from tabulate import tabulate
 
 import freqtrade.optimize as optimize
@@ -19,9 +19,14 @@ from freqtrade.arguments import Arguments
 from freqtrade.configuration import Configuration
 from freqtrade.exchange import Exchange
 from freqtrade.misc import file_dump_json
+from freqtrade.optimize.backslapping import Backslapping
 from freqtrade.persistence import Trade
 from freqtrade.strategy.interface import SellType
 from freqtrade.strategy.resolver import IStrategy, StrategyResolver
+from profilehooks import profile
+from collections import OrderedDict
+import timeit
+from time import sleep
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +57,7 @@ class Backtesting(object):
     backtesting = Backtesting(config)
     backtesting.start()
     """
+
     def __init__(self, config: Dict[str, Any]) -> None:
         self.config = config
         self.strategy: IStrategy = StrategyResolver(self.config).strategy
@@ -69,6 +75,49 @@ class Backtesting(object):
         self.exchange = Exchange(self.config)
         self.fee = self.exchange.get_fee()
 
+        self.stop_loss_value = self.strategy.stoploss
+
+        #### backslap config
+        '''
+        Numpy arrays are used for 100x speed up
+        We requires setting Int values for
+        buy stop triggers and stop calculated on
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
+        '''
+        self.np_buy: int = 0
+        self.np_open: int = 1
+        self.np_close: int = 2
+        self.np_sell: int = 3
+        self.np_high: int = 4
+        self.np_low: int = 5
+        self.np_stop: int = 6
+        self.np_bto: int = self.np_close  # buys_triggered_on - should be close
+        self.np_bco: int = self.np_open  # buys calculated on - open of the next candle.
+        self.np_sto: int = self.np_low  # stops_triggered_on - Should be low, FT uses close
+        self.np_sco: int = self.np_stop  # stops_calculated_on - Should be stop, FT uses close
+        # self.np_sto: int = self.np_close  # stops_triggered_on - Should be low, FT uses close
+        # self.np_sco: int = self.np_close  # stops_calculated_on - Should be stop, FT uses close
+
+        if 'backslap' in config:
+            self.use_backslap = config['backslap']  # Enable backslap - if false Orginal code is executed.
+        else:
+            self.use_backslap = False
+
+        logger.info("using backslap: {}".format(self.use_backslap))
+
+        self.debug = False  # Main debug enable, very print heavy, enable 2 loops recommended
+        self.debug_timing = False  # Stages within Backslap
+        self.debug_2loops = False  # Limit each pair to two loops, useful when debugging
+        self.debug_vector = False  # Debug vector calcs
+        self.debug_timing_main_loop = False  # print overall timing per pair - works in Backtest and Backslap
+
+        self.backslap_show_trades = False  # prints trades in addition to summary report
+        self.backslap_save_trades = True  # saves trades as a pretty table to backslap.txt
+
+        self.stop_stops: int = 9999  # stop back testing any pair with this many stops, set to 999999 to not hit
+
+        self.backslap = Backslapping(config)
+
     @staticmethod
     def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
         """
@@ -126,6 +175,7 @@ class Backtesting(object):
         """
         Generate small table outlining Backtest results
         """
+
         tabular_data = []
         headers = ['Sell Reason', 'Count']
         for reason, count in results['sell_reason'].value_counts().iteritems():
@@ -168,7 +218,6 @@ class Backtesting(object):
             sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
                                              sell_row.sell)
             if sell.sell_flag:
-
                 return BacktestResult(pair=pair,
                                       profit_percent=trade.calc_profit_percent(rate=sell_row.open),
                                       profit_abs=trade.calc_profit(rate=sell_row.open),
@@ -205,6 +254,13 @@ class Backtesting(object):
             return btr
         return None
 
+    def s(self):
+        st = timeit.default_timer()
+        return st
+
+    def f(self, st):
+        return (timeit.default_timer() - st)
+
     def backtest(self, args: Dict) -> DataFrame:
         """
         Implements backtesting functionality
@@ -220,13 +276,26 @@ class Backtesting(object):
             position_stacking: do we allow position stacking? (default: False)
         :return: DataFrame
         """
+
+        use_backslap = self.use_backslap
+        debug_timing = self.debug_timing_main_loop
+
+        if use_backslap:  # Use Back Slap code
+            return self.backslap.run(args)
+        else:  # use Original Back test code
+            ########################## Original BT loop
+
             headers = ['date', 'buy', 'open', 'close', 'sell']
             processed = args['processed']
             max_open_trades = args.get('max_open_trades', 0)
             position_stacking = args.get('position_stacking', False)
             trades = []
             trade_count_lock: Dict = {}
+
             for pair, pair_data in processed.items():
+                if debug_timing:  # Start timer
+                    fl = self.s()
+
                 pair_data['buy'], pair_data['sell'] = 0, 0  # cleanup from previous run
 
                 ticker_data = self.advise_sell(
@@ -238,6 +307,11 @@ class Backtesting(object):
 
                 ticker_data.drop(ticker_data.head(1).index, inplace=True)
 
+                if debug_timing:  # print time taken
+                    flt = self.f(fl)
+                    # print("populate_buy_trend:", pair, round(flt, 10))
+                    st = self.s()
+
                 # Convert from Pandas to list for performance reasons
                 # (Looping Pandas is slow.)
                 ticker = [x for x in ticker_data.itertuples()]
@@ -268,7 +342,13 @@ class Backtesting(object):
                         # This happens only if the buy-signal was with the last candle
                         lock_pair_until = ticker_data.iloc[-1].date
 
+                if debug_timing:  # print time taken
+                    tt = self.f(st)
+                    print("Time to BackTest :", pair, round(tt, 10))
+                    print("-----------------------")
+
             return DataFrame.from_records(trades, columns=BacktestResult._fields)
+            ####################### Original BT loop end
 
     def start(self) -> None:
         """
@@ -289,6 +369,7 @@ class Backtesting(object):
 
             timerange = Arguments.parse_timerange(None if self.config.get(
                 'timerange') is None else str(self.config.get('timerange')))
+
             data = optimize.load_data(
                 self.config['datadir'],
                 pairs=pairs,
@@ -298,6 +379,7 @@ class Backtesting(object):
                 timerange=timerange
             )
 
+        ld_files = self.s()
         if not data:
             logger.critical("No data found. Terminating.")
             return
@@ -309,6 +391,8 @@ class Backtesting(object):
             max_open_trades = 0
 
         preprocessed = self.tickerdata_to_dataframe(data)
+        t_t = self.f(ld_files)
+        print("Load from json to file to df in mem took", t_t)
 
         # Print timeframe
         min_date, max_date = self.get_timeframe(preprocessed)
@@ -332,20 +416,52 @@ class Backtesting(object):
         if self.config.get('export', False):
             self._store_backtest_result(self.config.get('exportfilename'), results)
 
+        if self.use_backslap:
             logger.info(
-            '\n' + '=' * 49 +
+                '\n====================================================== '
-            ' BACKTESTING REPORT ' +
+                'BackSLAP REPORT'
-            '=' * 50 + '\n'
+                ' =======================================================\n'
                 '%s',
                 self._generate_text_table(
                     data,
                     results
                 )
             )
-        # logger.info(
+            # optional print trades
-        #     results[['sell_reason']].groupby('sell_reason').count()
+            if self.backslap_show_trades:
-        # )
+                TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
+                                              'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
 
+                def to_fwf(df, fname):
+                    content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
+                    print(content)
+
+                DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
+
+            # optional save trades
+            if self.backslap_save_trades:
+                TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
+                                              'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
+
+                def to_fwf(df, fname):
+                    content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
+                    open(fname, "w").write(content)
+
+                DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
+
+        else:
+            logger.info(
+                '\n================================================= '
+                'BACKTEST REPORT'
+                ' ==================================================\n'
+                '%s',
+                self._generate_text_table(
+                    data,
+                    results
+                )
+            )
+
+        if 'sell_reason' in results.columns:
             logger.info(
                 '\n' +
                 ' SELL READON STATS '.center(119, '=') +
@@ -353,6 +469,8 @@ class Backtesting(object):
                 self._generate_text_table_sell_reason(data, results)
 
             )
+        else:
+            logger.info("no sell reasons available!")
 
         logger.info(
             '\n' +
@@ -377,7 +495,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]:
     # Ensure we do not use Exchange credentials
     config['exchange']['key'] = ''
     config['exchange']['secret'] = ''
-
+    config['backslap'] = args.backslap
     if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
         raise DependencyException('stake amount could not be "%s" for backtesting' %
                                   constants.UNLIMITED_STAKE_AMOUNT)

requirements.txt

@@ -23,3 +23,10 @@ scikit-optimize==0.5.2
 
 # Required for plotting data
 #plotly==3.0.0
+
+# find first, C search in arrays
+py_find_1st==1.1.1
+
+#Load ticker files 30% faster
+ujson==1.35
+

setup.py

@@ -37,6 +37,8 @@ setup(name='freqtrade',
           'cachetools',
           'coinmarketcap',
           'scikit-optimize',
+          'ujson',
+          'py_find_1st'
       ],
       include_package_data=True,
       zip_safe=False,