edge positioning put into package

config.json.example

@@ -50,6 +50,13 @@
         "sell_profit_only": false,
         "ignore_roi_if_buy_signal": false
     },
+    "edge": {
+        "enabled": false,
+        "process_throttle_secs": 1800,
+        "maximum_winrate_to_consider": 0.80, 
+        "remove_pumps": true,
+        "minimum_delta_to_consider": 1
+    },
     "telegram": {
         "enabled": true,
         "token": "your_telegram_token",

freqtrade/edge/__init__.py

@@ -0,0 +1,929 @@
+import logging
+import operator
+import sys
+from argparse import Namespace
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple
+
+import arrow
+from pandas import DataFrame, to_datetime
+from tabulate import tabulate
+import numpy as np
+
+import freqtrade.optimize as optimize
+from freqtrade import DependencyException, constants
+from freqtrade.arguments import Arguments
+from freqtrade.configuration import Configuration
+from freqtrade.exchange import Exchange
+from freqtrade.misc import file_dump_json
+from freqtrade.persistence import Trade
+from freqtrade.strategy.interface import SellType
+from freqtrade.strategy.resolver import IStrategy, StrategyResolver
+from freqtrade.optimize.backtesting import Backtesting
+from collections import OrderedDict
+import timeit
+from time import sleep
+
+import pdb
+
+logger = logging.getLogger(__name__)
+
+class Edge():
+
+    config: Dict = {}
+
+    def __init__(self, config: Dict[str, Any], exchange = None) -> None:
+        """
+        constructor
+        """
+        self.config = config
+        self.strategy: IStrategy = StrategyResolver(self.config).strategy
+        self.ticker_interval = self.strategy.ticker_interval
+        self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe
+        self.get_timeframe = Backtesting.get_timeframe
+        self.populate_buy_trend = self.strategy.populate_buy_trend
+        self.populate_sell_trend = self.strategy.populate_sell_trend
+
+        self._last_updated = None
+        self._cached_pairs = []
+        self._total_capital = self.config['edge']['total_capital_in_stake_currency']
+        self._allowed_risk = self.config['edge']['allowed_risk']
+
+        ###
+        #
+        ###
+        if exchange is None:
+            self.config['exchange']['secret'] = ''
+            self.config['exchange']['password'] = ''
+            self.config['exchange']['uid'] = ''
+            self.config['dry_run'] = True
+            self.exchange = Exchange(self.config)
+        else:
+            self.exchange = exchange
+
+        self.fee = self.exchange.get_fee()
+
+        self.stop_loss_value = self.strategy.stoploss
+
+        #### backslap config
+        '''
+        Numpy arrays are used for 100x speed up
+        We requires setting Int values for
+        buy stop triggers and stop calculated on
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
+        '''
+        self.np_buy: int = 0
+        self.np_open: int = 1
+        self.np_close: int = 2
+        self.np_sell: int = 3
+        self.np_high: int = 4
+        self.np_low: int = 5
+        self.np_stop: int = 6
+        self.np_bto: int = self.np_close  # buys_triggered_on - should be close
+        self.np_bco: int = self.np_open  # buys calculated on - open of the next candle.
+        self.np_sto: int = self.np_low  # stops_triggered_on - Should be low, FT uses close
+        self.np_sco: int = self.np_stop  # stops_calculated_on - Should be stop, FT uses close
+        # self.np_sto: int = self.np_close  # stops_triggered_on - Should be low, FT uses close
+        # self.np_sco: int = self.np_close  # stops_calculated_on - Should be stop, FT uses close
+
+        self.debug = False  # Main debug enable, very print heavy, enable 2 loops recommended
+        self.debug_timing = False  # Stages within Backslap
+        self.debug_2loops = False  # Limit each pair to two loops, useful when debugging
+        self.debug_vector = False  # Debug vector calcs
+        self.debug_timing_main_loop = False  # print overall timing per pair - works in Backtest and Backslap
+
+        self.backslap_show_trades = False  # prints trades in addition to summary report
+        self.backslap_save_trades = True  # saves trades as a pretty table to backslap.txt
+
+        self.stop_stops: int = 9999  # stop back testing any pair with this many stops, set to 999999 to not hit
+
+
+    def calculate(self) -> bool:
+        pairs = self.config['exchange']['pair_whitelist']
+        heartbeat = self.config['edge']['process_throttle_secs']
+
+        if ((self._last_updated is not None) and (self._last_updated + heartbeat > arrow.utcnow().timestamp)):
+            return False
+
+        data = {}
+
+        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
+        logger.info('Using stake_amount: %s ...', self.config['stake_amount'])
+ 
+        logger.info('Using local backtesting data (using whitelist in given config) ...')
+
+        timerange = Arguments.parse_timerange(None if self.config.get(
+            'timerange') is None else str(self.config.get('timerange')))
+
+        data = optimize.load_data(
+            self.config['datadir'],
+            pairs=pairs,
+            ticker_interval=self.ticker_interval,
+            refresh_pairs=self.config.get('refresh_pairs', False),
+            exchange=self.exchange,
+            timerange=timerange
+        )
+
+        if not data:
+            logger.critical("No data found. Edge is stopped ...")
+            return
+        
+        preprocessed = self.tickerdata_to_dataframe(data)
+
+        # Print timeframe
+        min_date, max_date = self.get_timeframe(preprocessed)
+        logger.info(
+            'Measuring data from %s up to %s (%s days) ...',
+            min_date.isoformat(),
+            max_date.isoformat(),
+            (max_date - min_date).days
+        )
+
+
+        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
+
+        # Max open trades need not be considered in Edge positioning
+        max_open_trades = 0
+
+        realistic = False
+        stoploss_range = np.arange(-0.11, -0.00, 0.01)
+        trades = []
+        trade_count_lock: Dict = {}
+
+        ########################### Call out BSlap Loop instead of Original BT code
+        bslap_results: list = []
+        for pair, pair_data in preprocessed.items():
+            
+            # Sorting dataframe by date and reset index 
+            pair_data = pair_data.sort_values(by=['date'])
+            pair_data = pair_data.reset_index(drop=True)
+
+            ticker_data = self.populate_sell_trend(
+                self.populate_buy_trend(pair_data))[headers].copy()
+
+            # call backslap - results are a list of dicts
+            for stoploss in stoploss_range:
+                bslap_results += self.backslap_pair(ticker_data, pair, round(stoploss, 3))
+
+        # Switch List of Trade Dicts (bslap_results) to Dataframe
+        # Fill missing, calculable columns, profit, duration , abs etc.
+        bslap_results_df = DataFrame(bslap_results)
+
+        if len(bslap_results_df) > 0:  # Only post process a frame if it has a record
+            bslap_results_df = self.vector_fill_results_table(bslap_results_df)
+        else:
+            from freqtrade.optimize.backtesting import BacktestResult
+
+            bslap_results_df = []
+            bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields)
+ 
+        self._cached_pairs = self._process_result(data, bslap_results_df, stoploss_range)
+        self._last_updated = arrow.utcnow().timestamp
+        return True
+        
+    def sort_pairs(self, pairs) -> bool:
+        if len(self._cached_pairs) == 0:
+            self.calculate()
+        edge_sorted_pairs = [x[0] for x in self._cached_pairs]
+        return [x for _, x in sorted(zip(edge_sorted_pairs,pairs), key=lambda pair: pair[0])]
+        
+    
+    def vector_fill_results_table(self, bslap_results_df: DataFrame):
+        """
+        The Results frame contains a number of columns that are calculable
+        from othe columns. These are left blank till all rows are added,
+        to be populated in single vector calls.
+
+        Columns to be populated are:
+        - Profit
+        - trade duration
+        - profit abs
+        :param bslap_results Dataframe
+        :return: bslap_results Dataframe
+        """
+        import pandas as pd
+        import numpy as np
+        debug = self.debug_vector
+
+        # stake and fees
+        # stake = 0.015
+        # 0.05% is 0.0005
+        # fee = 0.001
+
+        stake = self.config.get('stake_amount')
+        fee = self.fee
+        open_fee = fee / 2
+        close_fee = fee / 2
+
+
+        bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time']
+        bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60))
+
+        ## Spends, Takes, Profit, Absolute Profit
+        # print(bslap_results_df)
+        # Buy Price
+        bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate']  # How many target are we buying
+        bslap_results_df['buy_fee'] = stake * open_fee
+        bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee']  # How much we're spending
+
+        # Sell price
+        bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate']
+        bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee
+        bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee']
+        # profit_percent
+        bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \
+                                             / bslap_results_df['buy_spend']
+        # Absolute profit
+        bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend']
+
+
+        return bslap_results_df
+
+    def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int,
+                          stop_stops_count: int):
+        import utils_find_1st as utf1st
+        """
+         The purpose of this def is to return the next "buy" = 1
+         after t_exit_ind.
+         
+         This function will also check is the stop limit for the pair has been reached. 
+         if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit.
+
+         t_exit_ind is the index the last trade exited on
+         or 0 if first time around this loop.
+         
+         stop_stops i
+         """
+        debug = self.debug
+
+        # Timers, to be called if in debug
+        def s():
+            st = timeit.default_timer()
+            return st
+
+        def f(st):
+            return (timeit.default_timer() - st)
+
+        st = s()
+        t_open_ind: int
+
+        """
+        Create a view on our buy index starting after last trade exit
+        Search for next buy
+        """
+        np_buy_arr_v = np_buy_arr[t_exit_ind:]
+        t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal)
+
+        '''
+        If -1 is returned no buy has been found, preserve the value
+        '''
+        if t_open_ind != -1:  # send back the -1 if no buys found. otherwise update index
+            t_open_ind = t_open_ind + t_exit_ind  # Align numpy index
+
+        if t_open_ind == np_buy_arr_len - 1:  # If buy found on last candle ignore, there is no OPEN in next to use
+            t_open_ind = -1  # -1 ends the loop
+
+        if stop_stops_count >= stop_stops:  # if maximum number of stops allowed in a pair is hit, exit loop
+            t_open_ind = -1  # -1 ends the loop
+            if debug:
+                print("Max stop limit ", stop_stops, "reached. Moving to next pair")
+
+        return t_open_ind
+
+    def _process_result(self, data: Dict[str, Dict], results: DataFrame, stoploss_range) -> str:
+        """
+        This is a temporary version of edge positioning calculation. 
+        The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and 
+        other indictaors related to money management periodically (each X minutes) and keep it in a storage. 
+        The calulation will be done per pair and per strategy. 
+        """
+
+        # Removing open trades from dataset
+        results = results[results.open_at_end == False]
+        ###################################
+
+        # Removing pairs having less than min_trades_number
+        min_trades_number = 50
+        results = results.groupby('pair').filter(lambda x: len(x) > min_trades_number)
+        ###################################
+
+
+        # Removing outliers (Only Pumps) from the dataset 
+        # The method to detect outliers is to calculate standard deviation
+        # Then every value more than (standard deviation + 2*average) is out (pump)
+        #
+        # Calculating standard deviation of profits
+        std = results[["profit_abs"]].std()
+        #
+        # Calculating average of profits
+        avg = results[["profit_abs"]].mean()
+        #
+        # Removing Pumps
+        results = results[results.profit_abs < float(avg + 2*std)]
+        ##########################################################################
+
+        # Removing trades having a duration more than X minutes (set in config)
+        max_trade_duration = 24*60
+        results = results[results.trade_duration < max_trade_duration]
+        #######################################################################
+
+
+        # Win Rate is the number of profitable trades
+        # Divided by number of trades
+        def winrate(x):
+            x = x[x > 0].count() / x.count()
+            return x
+        #############################
+
+        # Risk Reward Ratio
+        # 1 / ((loss money / losing trades) / (gained money / winning trades))
+        def risk_reward_ratio(x):
+            x = abs(1/ ((x[x<0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))
+            return x
+        ##############################
+
+        # Required Risk Reward
+        # (1/(winrate - 1)
+        def required_risk_reward(x):
+            x = (1/(x[x > 0].count()/x.count()) -1)
+            return x
+        ##############################
+        
+        # The difference between risk reward ratio and required risk reward
+        # We use it as an indicator to find the most interesting pair to trade
+        def delta(x):
+            x = (abs(1/ ((x[x < 0].sum() / x[x < 0].count()) / (x[x > 0].sum() / x[x > 0].count())))) - (1/(x[x > 0].count()/x.count()) -1)
+            return x
+        ##############################
+
+        
+        final = results.groupby(['pair', 'stoploss'])['profit_abs'].\
+            agg([winrate, risk_reward_ratio, required_risk_reward, delta]).\
+            reset_index().sort_values(by=['delta', 'stoploss'], ascending=False)\
+            .groupby('pair').first().sort_values(by=['delta'], ascending=False)
+        
+        # Returning an array of pairs in order of "delta"
+        return final.reset_index().values
+
+    def backslap_pair(self, ticker_data, pair, stoploss):
+        import pandas as pd
+        import numpy as np
+        import timeit
+        import utils_find_1st as utf1st
+        from datetime import datetime
+        
+        ### backslap debug wrap
+        # debug_2loops = False  # only loop twice, for faster debug
+        # debug_timing = False  # print timing for each step
+        # debug = False  # print values, to check accuracy
+        debug_2loops = self.debug_2loops  # only loop twice, for faster debug
+        debug_timing = self.debug_timing  # print timing for each step
+        debug = self.debug  # print values, to check accuracy
+
+        # Read Stop Loss Values and Stake
+        # pdb.set_trace()
+        #stop = self.stop_loss_value
+        stop = stoploss
+        p_stop = (stop + 1)  # What stop really means, e.g 0.01 is 0.99 of price
+
+        if debug:
+            print("Stop is ", stop, "value from stragey file")
+            print("p_stop is", p_stop, "value used to multiply to entry price")
+
+        if debug:
+            from pandas import set_option
+            set_option('display.max_rows', 5000)
+            set_option('display.max_columns', 8)
+            pd.set_option('display.width', 1000)
+            pd.set_option('max_colwidth', 40)
+            pd.set_option('precision', 12)
+
+        def s():
+            st = timeit.default_timer()
+            return st
+
+        def f(st):
+            return (timeit.default_timer() - st)
+
+        #### backslap config
+        '''
+        Numpy arrays are used for 100x speed up
+        We requires setting Int values for
+        buy stop triggers and stop calculated on
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
+        '''
+
+        #######
+        #  Use vars set at top of backtest
+        np_buy: int = self.np_buy
+        np_open: int = self.np_open
+        np_close: int = self.np_close
+        np_sell: int = self.np_sell
+        np_high: int = self.np_high
+        np_low: int = self.np_low
+        np_stop: int = self.np_stop
+        np_bto: int = self.np_bto  # buys_triggered_on - should be close
+        np_bco: int = self.np_bco  # buys calculated on - open of the next candle.
+        np_sto: int = self.np_sto  # stops_triggered_on - Should be low, FT uses close
+        np_sco: int = self.np_sco  # stops_calculated_on - Should be stop, FT uses close
+
+        ### End Config
+
+        pair: str = pair
+
+        # ticker_data: DataFrame = ticker_dfs[t_file]
+        bslap: DataFrame = ticker_data
+
+        # Build a single dimension numpy array from "buy" index for faster search
+        # (500x faster than pandas)
+        np_buy_arr = bslap['buy'].values
+        np_buy_arr_len: int = len(np_buy_arr)
+
+        # use numpy array for faster searches in loop, 20x faster than pandas
+        # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+        np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']])
+
+        # Build a numpy list of date-times.
+        # We use these when building the trade
+        # The rationale is to address a value from a pandas cell is thousands of
+        # times more expensive. Processing time went X25 when trying to use any data from pandas
+        np_bslap_dates = bslap['date'].values
+
+        loop: int = 0  # how many time around the loop
+        t_exit_ind = 0  # Start loop from first index
+        t_exit_last = 0  # To test for exit
+
+        stop_stops = self.stop_stops  # Int of stops within a pair to stop trading a pair at
+        stop_stops_count = 0  # stop counter per pair
+
+        st = s()  # Start timer for processing dataframe
+        if debug:
+            print('Processing:', pair)
+
+        # Results will be stored in a list of dicts
+        bslap_pair_results: list = []
+        bslap_result: dict = {}
+
+        while t_exit_ind < np_buy_arr_len:
+            loop = loop + 1
+            if debug or debug_timing:
+                print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair)
+            if debug_2loops:
+                if loop == 3:
+                    print(
+                        "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking")
+                    break
+            '''
+             Dev phases
+             Phase 1
+              1) Manage buy, sell, stop enter/exit
+               a) Find first buy index
+               b) Discover first stop and sell hit after buy index
+               c) Chose first instance as trade exit
+    
+             Phase 2
+              2) Manage dynamic Stop and ROI Exit
+               a) Create trade slice from 1
+               b) search within trade slice for dynamice stop hit
+               c) search within trade slice for ROI hit
+             '''
+
+            if debug_timing:
+                st = s()
+            '''
+            0 - Find next buy entry
+            Finds index for first (buy = 1) flag
+    
+            Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1"
+            Required: t_exit_ind - Either 0, first loop. Or The index we last exited on
+            Requires: np_buy_arr_len - length of pair array. 
+            Requires: stops_stops - number of stops allowed before stop trading a pair 
+            Requires: stop_stop_counts - count of stops hit in the pair
+            Provides: The next "buy" index after t_exit_ind
+    
+            If -1 is returned no buy has been found in remainder of array, skip to exit loop
+            '''
+            t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count)
+
+            if debug:
+                print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind)
+                print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop")
+            if debug_timing:
+                t_t = f(st)
+                print("0-numpy", str.format('{0:.17f}', t_t))
+                st = s()
+
+            if t_open_ind != -1:
+
+                """
+                1 - Create views to search within for our open trade
+    
+                The views are our search space for the next Stop or Sell
+                Numpy view is employed as:
+                1,000 faster than pandas searches
+                Pandas cannot assure it will always return a view, it may make a slow copy.
+    
+                The view contains columns:
+                buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+    
+                Requires: np_bslap is our numpy array of the ticker DataFrame
+                Requires: t_open_ind is the index row with the  buy.
+                Provides: np_t_open_v View of array after buy.
+                Provides: np_t_open_v_stop View of array after buy +1  
+                          (Stop will search in here to prevent stopping in the past)
+                """
+                np_t_open_v = np_bslap[t_open_ind:]
+                np_t_open_v_stop = np_bslap[t_open_ind + 1:]
+
+                if debug:
+                    print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind)
+                    print("Numpy View: Buy - Open - Close - Sell - High - Low")
+                    print("Row 0", np_t_open_v[0])
+                    print("Row 1", np_t_open_v[1], )
+                if debug_timing:
+                    t_t = f(st)
+                    print("2-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                2 - Calculate our stop-loss price
+    
+                As stop is based on buy price of our trade
+                - (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle
+                - (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle.
+                This is as we only see the CLOSE after it has happened.
+                The  back test assumption is we have bought at first available price, the OPEN
+    
+                Requires: np_bslap   - is our numpy array of the ticker DataFrame
+                Requires: t_open_ind - is the index row with the first buy.
+                Requires: p_stop     - is the stop rate, ie. 0.99 is -1%
+                Provides: np_t_stop_pri - The value stop-loss will be triggered on
+                '''
+                np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop)
+
+                if debug:
+                    print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri)
+                if debug_timing:
+                    t_t = f(st)
+                    print("2-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                3 -  Find candle STO is under Stop-Loss After Trade opened.
+    
+                where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri
+    
+                Requires: np_t_open_v_stop   Numpy view of ticker_data after buy row +1 (when trade was opened)
+                Requires: np_sto        User Var(STO)StopTriggeredOn. Typically set to "low" or "close"
+                Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop
+                Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss
+                '''
+                np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto],
+                                                np_t_stop_pri,
+                                                utf1st.cmp_smaller)
+
+                # plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out.
+                np_t_stop_ind = np_t_stop_ind + 1
+
+                if debug:
+                    print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is",
+                          np_t_stop_ind - 1,
+                          ". STO is using field", np_sto,
+                          "\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n")
+
+                    print(
+                        "If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage")
+                    print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind])
+                    print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1])
+                if debug_timing:
+                    t_t = f(st)
+                    print("3-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                4 - Find first sell index after trade open
+    
+                First index in the view np_t_open_v where ['sell'] = 1
+    
+                Requires: np_t_open_v - view of ticker_data from buy onwards
+                Requires: no_sell - integer '3', the buy column in the array
+                Provides: np_t_sell_ind index of view where first sell=1 after buy
+                '''
+                # Use numpy array for faster search for sell
+                # Sell uses column 3.
+                # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+                # Numpy searches 25-35x quicker than pandas on this data
+
+                np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell],
+                                                1, utf1st.cmp_equal)
+                if debug:
+                    print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind)
+                    print("If 0 or less is returned there is no sell found to end of view, then next lines garbage")
+                    print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind])
+                    print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1])
+                if debug_timing:
+                    t_t = f(st)
+                    print("4-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                '''
+                5 - Determine which was hit first a stop or sell
+                To then use as exit index price-field (sell on buy, stop on stop)
+    
+                STOP takes priority over SELL as would be 'in candle' from tick data
+                Sell would use Open from Next candle.
+                So in a draw Stop would be hit first on ticker data in live
+    
+                Validity of when types of trades may be executed can be summarised as:
+    
+                            Tick	View
+                            index	index	Buy Sell open	low	     close	 high   Stop price
+                open 2am	94	        -1	0	 0	-----	------	 ------  -----   -----
+                open 3am 	95	        0	1	 0	-----	------	 trg buy -----   -----
+                open 4am 	96	        1	0	 1	Enter	trgstop	 trg sel ROI out Stop out
+                open 5am 	97	        2	0	 0	Exit	------	 ------- -----   -----
+                open 6am	98	        3	0	 0	----- 	------	 ------- -----   -----
+    
+                -1 means not found till end of view i.e no valid Stop found. Exclude from match.
+                Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid.
+    
+                Buys and sells are triggered at candle close
+                Both will open their postions at the open of the next candle. i/e  + 1 index
+    
+                Stop and buy Indexes are on the view. To map to the ticker dataframe
+                the t_open_ind index should be summed.
+    
+                np_t_stop_ind: Stop Found index in view
+                t_exit_ind   : Sell found in view
+                t_open_ind   : Where view was started on ticker_data
+    
+                TODO: fix this frig for logic test,, case/switch/dictionary would be better...
+                      more so when later testing many options, dynamic stop / roi etc
+                cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found)
+                cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found)
+    
+                '''
+                if debug:
+                    print("\n(5) numpy debug\nStop or Sell Logic Processing")
+
+                # cludge for logic test (-1) means it was not found, set crazy high to lose < test
+                np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind
+                np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind
+
+                # Stoploss trigger found before a sell =1
+                if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind:
+                    t_exit_ind = t_open_ind + np_t_stop_ind  # Set Exit row index
+                    t_exit_type = SellType.STOP_LOSS  # Set Exit type (stop)
+                    np_t_exit_pri = np_sco  # The price field our STOP exit will use
+                    if debug:
+                        print("Type STOP is first exit condition. "
+                              "At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind)
+
+                # Buy = 1 found before a stoploss triggered
+                elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind:
+                    # move sell onto next candle, we only look back on sell
+                    # will use the open price later.
+                    t_exit_ind = t_open_ind + np_t_sell_ind # Set Exit row index
+                    t_exit_type = SellType.SELL_SIGNAL  # Set Exit type (sell)
+                    np_t_exit_pri = np_open  # The price field our SELL exit will use
+                    if debug:
+                        print("Type SELL is first exit condition. "
+                              "At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind)
+
+                # No stop or buy left in view - set t_exit_last -1 to handle gracefully
+                else:
+                    t_exit_last: int = -1  # Signal loop to exit, no buys or sells found.
+                    t_exit_type = SellType.NONE
+                    np_t_exit_pri = 999  # field price should be calculated on. 999 a non-existent column
+                    if debug:
+                        print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit")
+
+                # TODO: fix having to cludge/uncludge this ..
+                # Undo cludge
+                np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind
+                np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind
+
+                if debug_timing:
+                    t_t = f(st)
+                    print("5-logic", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                if debug:
+                    '''
+                    Print out the buys, stops, sells
+                    Include Line before and after to for easy
+                    Human verification
+                    '''
+                    # Combine the np_t_stop_pri value to bslap dataframe to make debug
+                    # life easy. This is the current stop price based on buy price_
+                    # This is slow but don't care about performance in debug
+                    #
+                    # When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1
+                    # as there is no data column in the numpy array.
+                    bslap['np_stop_pri'] = np_t_stop_pri
+
+                    # Buy
+                    print("\n\nDATAFRAME DEBUG =================== BUY ", pair)
+                    print("Numpy Array BUY Index is:", 0)
+                    print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n")
+                    print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1)
+                    op_is = t_open_ind - 1  # Print open index start, line before
+                    op_if = t_open_ind + 3  # Print open index finish, line after
+                    print(bslap.iloc[op_is:op_if], "\n")
+
+                    # Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On)
+                    if np_t_stop_ind < 0:
+                        print("DATAFRAME DEBUG =================== STOP  ", pair)
+                        print("No STOPS were found until the end of ticker data file\n")
+                    else:
+                        print("DATAFRAME DEBUG =================== STOP  ", pair)
+                        print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind)
+                        df_stop_index = (t_open_ind + np_t_stop_ind)
+
+                        print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n")
+                        print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ",
+                              df_stop_index, ": \n",
+                              str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]),
+                              "is less than", str.format('{0:.17f}', np_t_stop_pri))
+
+                        print("A stoploss exit will be calculated at rate:",
+                              str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1]))
+
+                        print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index,
+                              ": As live STOPs are not linked to O-C times")
+
+                        st_is = df_stop_index - 1  # Print stop index start, line before
+                        st_if = df_stop_index + 2  # Print stop index finish, line after
+                        print(bslap.iloc[st_is:st_if], "\n")
+
+                    # Sell
+                    if np_t_sell_ind < 0:
+                        print("DATAFRAME DEBUG =================== SELL ", pair)
+                        print("No SELLS were found till the end of ticker data file\n")
+                    else:
+                        print("DATAFRAME DEBUG =================== SELL ", pair)
+                        print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind)
+                        df_sell_index = (t_open_ind + np_t_sell_ind)
+
+                        print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n")
+                        print("First Sell Index after Trade open is in candle", df_sell_index)
+                        print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle",
+                              df_sell_index + 1)
+                        sl_is = df_sell_index - 1  # Print sell index start, line before
+                        sl_if = df_sell_index + 3  # Print sell index finish, line after
+                        print(bslap.iloc[sl_is:sl_if], "\n")
+
+                    # Chosen Exit (stop or sell)
+
+                    print("DATAFRAME DEBUG =================== EXIT ", pair)
+                    print("Exit type is :", t_exit_type)
+                    print("trade exit price field is", np_t_exit_pri, "\n")
+
+                if debug_timing:
+                    t_t = f(st)
+                    print("6-depra", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                ## use numpy view "np_t_open_v" for speed. Columns are
+                # buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
+                # exception is 6 which is use the stop value.
+
+                # TODO no! this is hard coded bleh fix this open
+                np_trade_enter_price = np_bslap[t_open_ind + 1, np_open]
+                if t_exit_type == SellType.STOP_LOSS:
+                    if np_t_exit_pri == 6:
+                        np_trade_exit_price = np_t_stop_pri
+                        t_exit_ind = t_exit_ind + 1
+                    else:
+                        np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri]
+                if t_exit_type == SellType.SELL_SIGNAL:
+                    np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri]
+
+                # Catch no exit found
+                if t_exit_type == SellType.NONE:
+                    np_trade_exit_price = 0
+
+                if debug_timing:
+                    t_t = f(st)
+                    print("7-numpy", str.format('{0:.17f}', t_t))
+                    st = s()
+
+                if debug:
+                    print("//////////////////////////////////////////////")
+                    print("+++++++++++++++++++++++++++++++++ Trade Enter ")
+                    print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price))
+                    print("--------------------------------- Trade Exit ")
+                    print("Trade Exit Type is ", t_exit_type)
+                    print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price))
+                    print("//////////////////////////////////////////////")
+
+            else:  # no buys were found, step 0 returned -1
+                # Gracefully exit the loop
+                t_exit_last == -1
+                if debug:
+                    print("\n(E) No buys were found in remaining ticker file. Exiting", pair)
+
+            # Loop control - catch no closed trades.
+            if debug:
+                print("---------------------------------------- end of loop", loop,
+                      " Dataframe Exit Index is: ", t_exit_ind)
+                print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind)
+
+            if t_exit_last >= t_exit_ind or t_exit_last == -1:
+                """
+                Break loop and go on to next pair.
+    
+                When last trade exit equals index of last exit, there is no
+                opportunity to close any more trades.
+                """
+                # TODO :add handing here to record none closed open trades
+                
+                if debug:
+                    print(bslap_pair_results)
+                break
+            else:
+                """
+                Add trade to backtest looking results list of dicts
+                Loop back to look for more trades.
+                """
+
+                # We added +1 to t_exit_ind if the exit was a stop-loss, to not exit early in the IF of this ELSE
+                # removing the +1 here so prices match.
+                if t_exit_type == SellType.STOP_LOSS:
+                    t_exit_ind = t_exit_ind - 1
+
+                # Build trade dictionary
+                ## In general if a field can be calculated later from other fields leave blank here
+                ## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time
+
+                # create a new dict
+                close_index: int = t_exit_ind
+                bslap_result = {}  # Must have at start or we end up with a list of multiple same last result
+                bslap_result["pair"] = pair
+                bslap_result["stoploss"] = stop
+                bslap_result["profit_percent"] = ""  # To be 1 vector calc across trades when loop complete
+                bslap_result["profit_abs"] = ""  # To be 1 vector calc across trades when loop complete
+                bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1]  # use numpy array, pandas 20x slower
+                bslap_result["close_time"] = np_bslap_dates[close_index]  # use numpy array, pandas 20x slower
+                bslap_result["open_index"] = t_open_ind + 1  # +1 as we buy on next.
+                bslap_result["close_index"] = close_index
+                bslap_result["trade_duration"] = ""  # To be 1 vector calc across trades when loop complete
+                bslap_result["open_at_end"] = False
+                bslap_result["open_rate"] = round(np_trade_enter_price, 15)
+                bslap_result["close_rate"] = round(np_trade_exit_price, 15)
+                bslap_result["exit_type"] = t_exit_type
+                bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care
+                # append the dict to the list and print list
+                bslap_pair_results.append(bslap_result)
+
+                if t_exit_type is SellType.STOP_LOSS:
+                    stop_stops_count = stop_stops_count + 1
+
+                if debug:
+                    print("The trade dict is: \n", bslap_result)
+                    print("Trades dicts in list after append are: \n ", bslap_pair_results)
+
+                """
+                Loop back to start. t_exit_last becomes where loop
+                will seek to open new trades from.
+                Push index on 1 to not open on close
+                """
+                t_exit_last = t_exit_ind + 1
+
+            if debug_timing:
+                t_t = f(st)
+                print("8+trade", str.format('{0:.17f}', t_t))
+
+        # Send back List of trade dicts
+        return bslap_pair_results
+
+    def stake_amount(self, pair: str) -> str:
+        info = [x for x in self._cached_pairs if x[0] == pair][0]
+        stoploss = info[1]
+        
+        allowed_capital_at_risk = round(self._total_capital * self._allowed_risk, 5)
+        position_size = abs(round((allowed_capital_at_risk / stoploss), 5))
+
+        return (allowed_dollars_at_risk / symbol_strategy_stop_loss)
+
+        ### stake amount is the same as position size
+        ### calculate position size
+        # print("\n~~~~~~~~ Position Size ~~~~~~~~")
+        # print("bid price is            ", bid_price)
+        # print("stop trigger is         ", stop_trigger_price)
+
+        # allowed_dollars_at_risk = total_capital * allowed_risk
+        # print("allowed capital at risk ", round(allowed_dollars_at_risk, 5))
+
+        # position_size = (allowed_dollars_at_risk / symbol_strategy_stop_loss)
+        # print("position_size in dollars", round(position_size, 5 ))
+
+        # buy_amount = position_size / bid_price
+        # print("amount of tokens to buy ", round(buy_amount,5))
+
+        # check_risk = (buy_amount * (bid_price - stop_trigger_price))
+        # print("check risk capital      ", round(check_risk, 5), "** Should not be more than allowed capital at risk")
+    
+    def stoploss(self, pair: str) -> float:
+        info = [x for x in self._cached_pairs if x[0] == pair][0]
+        return info[1]

freqtrade/freqtradebot.py

@@ -17,6 +17,7 @@ from cachetools import TTLCache, cached
 from freqtrade import (DependencyException, OperationalException,
                        TemporaryError, __version__, constants, persistence)
 from freqtrade.exchange import Exchange
+from freqtrade.edge import Edge
 from freqtrade.persistence import Trade
 from freqtrade.rpc import RPCManager, RPCMessageType
 from freqtrade.state import State
@@ -24,6 +25,8 @@ from freqtrade.strategy.interface import SellType
 from freqtrade.strategy.resolver import IStrategy, StrategyResolver
 from freqtrade.exchange.exchange_helpers import order_book_to_dataframe
 
+import pdb
+
 logger = logging.getLogger(__name__)
 
 
@@ -54,6 +57,7 @@ class FreqtradeBot(object):
         self.rpc: RPCManager = RPCManager(self)
         self.persistence = None
         self.exchange = Exchange(self.config)
+        self.edge = Edge(self.config, self.exchange)
         self._init_modules()
 
     def _init_modules(self) -> None:
@@ -185,6 +189,10 @@ class FreqtradeBot(object):
             # Refreshing candles
             self.exchange.refresh_tickers(final_list, self.strategy.ticker_interval)
 
+            # Calculating Edge positiong
+            if self.config['edge']['enabled']:
+                self.edge.calculate()
+
             # Query trades from persistence layer
             trades = Trade.query.filter(Trade.is_open.is_(True)).all()
 
@@ -307,13 +315,17 @@ class FreqtradeBot(object):
 
         return used_rate
 
-    def _get_trade_stake_amount(self) -> Optional[float]:
+    def _get_trade_stake_amount(self, pair="") -> Optional[float]:
         """
         Check if stake amount can be fulfilled with the available balance
         for the stake currency
         :return: float: Stake Amount
         """
-        stake_amount = self.config['stake_amount']
+        if self.config['edge']['enabled']:
+            stake_amount = self.edge.stake_amount(pair)
+        else:
+            stake_amount = self.config['stake_amount']
+
         avaliable_amount = self.exchange.get_balance(self.config['stake_currency'])
 
         if stake_amount == constants.UNLIMITED_STAKE_AMOUNT:
@@ -372,17 +384,8 @@ class FreqtradeBot(object):
         """
         interval = self.strategy.ticker_interval
 
-        # EDGE
+        logger.info('Checking buy signals to create a new trade: ...')
-        # STAKE AMOUNT SHOULD COME FROM EDGE
-        stake_amount = self._get_trade_stake_amount()
 
-        if not stake_amount:
-            return False
-
-        logger.info(
-            'Checking buy signals to create a new trade with stake_amount: %f ...',
-            stake_amount
-        )
         whitelist = copy.deepcopy(self.config['exchange']['pair_whitelist'])
 
         # Remove currently opened and latest pairs from whitelist
@@ -396,9 +399,13 @@ class FreqtradeBot(object):
 
         # running get_signal on historical data fetched
         # to find buy signals
+        if self.config['edge']['enabled']:
+            whitelist = self.edge.sort_pairs(whitelist)
+
         for _pair in whitelist:
             (buy, sell) = self.strategy.get_signal(_pair, interval, self.exchange.klines.get(_pair))
             if buy and not sell:
+                stake_amount = self._get_trade_stake_amount(_pair)
                 bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\
                     get('check_depth_of_market', {})
                 if (bidstrat_check_depth_of_market.get('enabled', False)) and\
@@ -436,11 +443,12 @@ class FreqtradeBot(object):
         """
         pair_s = pair.replace('_', '/')
         pair_url = self.exchange.get_pair_detail_url(pair)
-        stake_currency = self.config['stake_currency']
+
         fiat_currency = self.config.get('fiat_display_currency', None)
         
         # Calculate amount
         buy_limit = self.get_target_bid(pair, self.exchange.get_ticker(pair))
+        stake_currency = self.config['stake_currency']
 
         min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit)
         if min_stake_amount is not None and min_stake_amount > stake_amount:
@@ -622,7 +630,12 @@ class FreqtradeBot(object):
         return False
 
     def check_sell(self, trade: Trade, sell_rate: float, buy: bool, sell: bool) -> bool:
-        should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell)
+        if (self.config['edge']['enabled']):
+            stoploss = self.edge.stoploss(trade.pair)
+            should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell, stoploss)
+        else:
+            should_sell = self.strategy.should_sell(trade, sell_rate, datetime.utcnow(), buy, sell)
+
         if should_sell.sell_flag:
             self.execute_sell(trade, sell_rate, should_sell.sell_type)
             logger.info('excuted sell')

freqtrade/strategy/interface.py

@@ -203,7 +203,7 @@ class IStrategy(ABC):
         return buy, sell
 
     def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool,
-                    sell: bool) -> SellCheckTuple:
+                    sell: bool, force_stoploss=0) -> SellCheckTuple:
         """
         This function evaluate if on the condition required to trigger a sell has been reached
         if the threshold is reached and updates the trade record.
@@ -211,7 +211,7 @@ class IStrategy(ABC):
         """
         current_profit = trade.calc_profit_percent(rate)
         stoplossflag = self.stop_loss_reached(current_rate=rate, trade=trade, current_time=date,
-                                              current_profit=current_profit)
+                                              current_profit=current_profit, force_stoploss=force_stoploss)
         if stoplossflag.sell_flag:
             return stoplossflag
 
@@ -237,7 +237,7 @@ class IStrategy(ABC):
         return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE)
 
     def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime,
-                          current_profit: float) -> SellCheckTuple:
+                          current_profit: float, force_stoploss: float) -> SellCheckTuple:
         """
         Based on current profit of the trade and configured (trailing) stoploss,
         decides to sell or not
@@ -246,7 +246,10 @@ class IStrategy(ABC):
 
         trailing_stop = self.config.get('trailing_stop', False)
 
-        trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True)
+        if force_stoploss == 0:
+            trade.adjust_stop_loss(trade.open_rate, self.stoploss, initial=True)
+        else:
+            trade.adjust_stop_loss(trade.open_rate, force_stoploss, initial=True)
 
         # evaluate if the stoploss was hit
         if self.stoploss is not None and trade.stop_loss >= current_rate:

freqtrade/tests/conftest.py

@@ -127,6 +127,15 @@ def default_conf():
                 "NEO/BTC"
             ]
         },
+        "edge": {
+            "enabled": False,
+            "process_throttle_secs": 1800,
+            "total_capital_in_stake_currency": 0.5,
+            "allowed_risk": 0.01,
+            "maximum_winrate_to_consider": 0.80,
+            "remove_pumps": True,
+            "minimum_delta_to_consider": 1
+        },
         "telegram": {
             "enabled": True,
             "token": "token",