Merge pull request #1113 from berlinguyinca/backslap_develop
Backslap develop
This commit is contained in:
Matthias
GitHub
commit
4f5b530dcb
@ -161,6 +161,14 @@ class Arguments(object):
|
||||
dest='exportfilename',
|
||||
metavar='PATH',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--backslap',
|
||||
help="Utilize the Backslapping approach instead of the default Backtesting. This should provide more "
|
||||
"accurate results, unless you are utilizing Min/Max function in your strategy.",
|
||||
required=False,
|
||||
dest='backslap',
|
||||
action='store_true'
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def optimizer_shared_options(parser: argparse.ArgumentParser) -> None:
|
||||
|
||||
@ -10,9 +10,16 @@ import arrow
|
||||
from freqtrade import misc, constants, OperationalException
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.arguments import TimeRange
|
||||
import importlib
|
||||
ujson_found = importlib.util.find_spec("ujson")
|
||||
if ujson_found is not None:
|
||||
import ujson
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if ujson_found is not None:
|
||||
logger.debug('Loaded UltraJson ujson in optimize.py')
|
||||
|
||||
|
||||
def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]:
|
||||
if not tickerlist:
|
||||
@ -63,11 +70,17 @@ def load_tickerdata_file(
|
||||
if os.path.isfile(gzipfile):
|
||||
logger.debug('Loading ticker data from file %s', gzipfile)
|
||||
with gzip.open(gzipfile) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
if ujson_found is not None:
|
||||
pairdata = ujson.load(tickerdata, precise_float=True)
|
||||
else:
|
||||
pairdata = json.load(tickerdata)
|
||||
elif os.path.isfile(file):
|
||||
logger.debug('Loading ticker data from file %s', file)
|
||||
with open(file) as tickerdata:
|
||||
pairdata = json.load(tickerdata)
|
||||
if ujson_found is not None:
|
||||
pairdata = ujson.load(tickerdata, precise_float=True)
|
||||
else:
|
||||
pairdata = json.load(tickerdata)
|
||||
else:
|
||||
return None
|
||||
|
||||
@ -163,7 +176,10 @@ def load_cached_data_for_updating(filename: str,
|
||||
# read the cached file
|
||||
if os.path.isfile(filename):
|
||||
with open(filename, "rt") as file:
|
||||
data = json.load(file)
|
||||
if ujson_found is not None:
|
||||
data = ujson.load(file, precise_float=True)
|
||||
else:
|
||||
data = json.load(file)
|
||||
# remove the last item, because we are not sure if it is correct
|
||||
# it could be fetched when the candle was incompleted
|
||||
if data:
|
||||
@ -226,4 +242,4 @@ def download_backtesting_testdata(datadir: str,
|
||||
logger.debug("New Start: %s", misc.format_ms_time(data[0][0]))
|
||||
logger.debug("New End: %s", misc.format_ms_time(data[-1][0]))
|
||||
|
||||
misc.file_dump_json(filename, data)
|
||||
misc.file_dump_json(filename, data)
|
||||
789
freqtrade/optimize/backslapping.py
Normal file
789
freqtrade/optimize/backslapping.py
Normal file
@ -0,0 +1,789 @@
|
||||
import timeit
|
||||
from typing import Dict, Any
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.strategy import IStrategy
|
||||
from freqtrade.strategy.interface import SellType
|
||||
from freqtrade.strategy.resolver import StrategyResolver
|
||||
|
||||
|
||||
class Backslapping:
|
||||
"""
|
||||
provides a quick way to evaluate strategies over a longer term of time
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any], exchange = None) -> None:
|
||||
"""
|
||||
constructor
|
||||
"""
|
||||
|
||||
self.config = config
|
||||
self.strategy: IStrategy = StrategyResolver(self.config).strategy
|
||||
self.ticker_interval = self.strategy.ticker_interval
|
||||
self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe
|
||||
self.populate_buy_trend = self.strategy.populate_buy_trend
|
||||
self.populate_sell_trend = self.strategy.populate_sell_trend
|
||||
|
||||
###
|
||||
#
|
||||
###
|
||||
if exchange is None:
|
||||
self.config['exchange']['secret'] = ''
|
||||
self.config['exchange']['password'] = ''
|
||||
self.config['exchange']['uid'] = ''
|
||||
self.config['dry_run'] = True
|
||||
self.exchange = Exchange(self.config)
|
||||
else:
|
||||
self.exchange = exchange
|
||||
|
||||
self.fee = self.exchange.get_fee()
|
||||
|
||||
self.stop_loss_value = self.strategy.stoploss
|
||||
|
||||
#### backslap config
|
||||
'''
|
||||
Numpy arrays are used for 100x speed up
|
||||
We requires setting Int values for
|
||||
buy stop triggers and stop calculated on
|
||||
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
|
||||
'''
|
||||
self.np_buy: int = 0
|
||||
self.np_open: int = 1
|
||||
self.np_close: int = 2
|
||||
self.np_sell: int = 3
|
||||
self.np_high: int = 4
|
||||
self.np_low: int = 5
|
||||
self.np_stop: int = 6
|
||||
self.np_bto: int = self.np_close # buys_triggered_on - should be close
|
||||
self.np_bco: int = self.np_open # buys calculated on - open of the next candle.
|
||||
self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close
|
||||
self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close
|
||||
# self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close
|
||||
# self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close
|
||||
|
||||
self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended
|
||||
self.debug_timing = False # Stages within Backslap
|
||||
self.debug_2loops = False # Limit each pair to two loops, useful when debugging
|
||||
self.debug_vector = False # Debug vector calcs
|
||||
self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap
|
||||
|
||||
self.backslap_show_trades = False # prints trades in addition to summary report
|
||||
self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt
|
||||
|
||||
self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit
|
||||
|
||||
def s(self):
|
||||
st = timeit.default_timer()
|
||||
return st
|
||||
|
||||
def f(self, st):
|
||||
return (timeit.default_timer() - st)
|
||||
|
||||
def run(self,args):
|
||||
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
|
||||
processed = args['processed']
|
||||
max_open_trades = args.get('max_open_trades', 0)
|
||||
realistic = args.get('realistic', False)
|
||||
trades = []
|
||||
trade_count_lock: Dict = {}
|
||||
|
||||
########################### Call out BSlap Loop instead of Original BT code
|
||||
bslap_results: list = []
|
||||
for pair, pair_data in processed.items():
|
||||
if self.debug_timing: # Start timer
|
||||
fl = self.s()
|
||||
|
||||
ticker_data = self.populate_sell_trend(
|
||||
self.populate_buy_trend(pair_data))[headers].copy()
|
||||
|
||||
if self.debug_timing: # print time taken
|
||||
flt = self.f(fl)
|
||||
# print("populate_buy_trend:", pair, round(flt, 10))
|
||||
st = self.s()
|
||||
|
||||
# #dump same DFs to disk for offline testing in scratch
|
||||
# f_pair:str = pair
|
||||
# csv = f_pair.replace("/", "_")
|
||||
# csv="/Users/creslin/PycharmProjects/freqtrade_new/frames/" + csv
|
||||
# ticker_data.to_csv(csv, sep='\t', encoding='utf-8')
|
||||
|
||||
# call bslap - results are a list of dicts
|
||||
bslap_pair_results = self.backslap_pair(ticker_data, pair)
|
||||
last_bslap_results = bslap_results
|
||||
bslap_results = last_bslap_results + bslap_pair_results
|
||||
|
||||
if self.debug_timing: # print time taken
|
||||
tt = self.f(st)
|
||||
print("Time to BackSlap :", pair, round(tt, 10))
|
||||
print("-----------------------")
|
||||
|
||||
# Switch List of Trade Dicts (bslap_results) to Dataframe
|
||||
# Fill missing, calculable columns, profit, duration , abs etc.
|
||||
bslap_results_df = DataFrame(bslap_results)
|
||||
|
||||
if len(bslap_results_df) > 0: # Only post process a frame if it has a record
|
||||
# bslap_results_df['open_time'] = to_datetime(bslap_results_df['open_time'])
|
||||
# bslap_results_df['close_time'] = to_datetime(bslap_results_df['close_time'])
|
||||
# if debug:
|
||||
# print("open_time and close_time converted to datetime columns")
|
||||
|
||||
bslap_results_df = self.vector_fill_results_table(bslap_results_df, pair)
|
||||
else:
|
||||
from freqtrade.optimize.backtesting import BacktestResult
|
||||
|
||||
bslap_results_df = []
|
||||
bslap_results_df = DataFrame.from_records(bslap_results_df, columns=BacktestResult._fields)
|
||||
|
||||
return bslap_results_df
|
||||
|
||||
def vector_fill_results_table(self, bslap_results_df: DataFrame, pair: str):
|
||||
"""
|
||||
The Results frame contains a number of columns that are calculable
|
||||
from othe columns. These are left blank till all rows are added,
|
||||
to be populated in single vector calls.
|
||||
|
||||
Columns to be populated are:
|
||||
- Profit
|
||||
- trade duration
|
||||
- profit abs
|
||||
:param bslap_results Dataframe
|
||||
:return: bslap_results Dataframe
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
debug = self.debug_vector
|
||||
|
||||
# stake and fees
|
||||
# stake = 0.015
|
||||
# 0.05% is 0.0005
|
||||
# fee = 0.001
|
||||
|
||||
stake = self.config.get('stake_amount')
|
||||
fee = self.fee
|
||||
open_fee = fee / 2
|
||||
close_fee = fee / 2
|
||||
|
||||
if debug:
|
||||
print("Stake is,", stake, "the sum of currency to spend per trade")
|
||||
print("The open fee is", open_fee, "The close fee is", close_fee)
|
||||
if debug:
|
||||
from pandas import set_option
|
||||
set_option('display.max_rows', 5000)
|
||||
set_option('display.max_columns', 20)
|
||||
pd.set_option('display.width', 1000)
|
||||
pd.set_option('max_colwidth', 40)
|
||||
pd.set_option('precision', 12)
|
||||
|
||||
# # Get before
|
||||
# csv = "cryptosher_before_debug"
|
||||
# bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8')
|
||||
|
||||
# bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8')
|
||||
|
||||
bslap_results_df['trade_duration'] = bslap_results_df['close_time'] - bslap_results_df['open_time']
|
||||
bslap_results_df['trade_duration'] = bslap_results_df['trade_duration'].map(lambda x: int(x.total_seconds() / 60))
|
||||
|
||||
## Spends, Takes, Profit, Absolute Profit
|
||||
# print(bslap_results_df)
|
||||
# Buy Price
|
||||
bslap_results_df['buy_vol'] = stake / bslap_results_df['open_rate'] # How many target are we buying
|
||||
bslap_results_df['buy_fee'] = stake * open_fee
|
||||
bslap_results_df['buy_spend'] = stake + bslap_results_df['buy_fee'] # How much we're spending
|
||||
|
||||
# Sell price
|
||||
bslap_results_df['sell_sum'] = bslap_results_df['buy_vol'] * bslap_results_df['close_rate']
|
||||
bslap_results_df['sell_fee'] = bslap_results_df['sell_sum'] * close_fee
|
||||
bslap_results_df['sell_take'] = bslap_results_df['sell_sum'] - bslap_results_df['sell_fee']
|
||||
# profit_percent
|
||||
bslap_results_df['profit_percent'] = (bslap_results_df['sell_take'] - bslap_results_df['buy_spend']) \
|
||||
/ bslap_results_df['buy_spend']
|
||||
# Absolute profit
|
||||
bslap_results_df['profit_abs'] = bslap_results_df['sell_take'] - bslap_results_df['buy_spend']
|
||||
|
||||
# # Get After
|
||||
# csv="cryptosher_after_debug"
|
||||
# bslap_results_df.to_csv(csv, sep='\t', encoding='utf-8')
|
||||
|
||||
if debug:
|
||||
print("\n")
|
||||
print(bslap_results_df[
|
||||
['buy_vol', 'buy_fee', 'buy_spend', 'sell_sum', 'sell_fee', 'sell_take', 'profit_percent',
|
||||
'profit_abs', 'exit_type']])
|
||||
|
||||
return bslap_results_df
|
||||
|
||||
def np_get_t_open_ind(self, np_buy_arr, t_exit_ind: int, np_buy_arr_len: int, stop_stops: int,
|
||||
stop_stops_count: int):
|
||||
import utils_find_1st as utf1st
|
||||
"""
|
||||
The purpose of this def is to return the next "buy" = 1
|
||||
after t_exit_ind.
|
||||
|
||||
This function will also check is the stop limit for the pair has been reached.
|
||||
if stop_stops is the limit and stop_stops_count it the number of times the stop has been hit.
|
||||
|
||||
t_exit_ind is the index the last trade exited on
|
||||
or 0 if first time around this loop.
|
||||
|
||||
stop_stops i
|
||||
"""
|
||||
debug = self.debug
|
||||
|
||||
# Timers, to be called if in debug
|
||||
def s():
|
||||
st = timeit.default_timer()
|
||||
return st
|
||||
|
||||
def f(st):
|
||||
return (timeit.default_timer() - st)
|
||||
|
||||
st = s()
|
||||
t_open_ind: int
|
||||
|
||||
"""
|
||||
Create a view on our buy index starting after last trade exit
|
||||
Search for next buy
|
||||
"""
|
||||
np_buy_arr_v = np_buy_arr[t_exit_ind:]
|
||||
t_open_ind = utf1st.find_1st(np_buy_arr_v, 1, utf1st.cmp_equal)
|
||||
|
||||
'''
|
||||
If -1 is returned no buy has been found, preserve the value
|
||||
'''
|
||||
if t_open_ind != -1: # send back the -1 if no buys found. otherwise update index
|
||||
t_open_ind = t_open_ind + t_exit_ind # Align numpy index
|
||||
|
||||
if t_open_ind == np_buy_arr_len - 1: # If buy found on last candle ignore, there is no OPEN in next to use
|
||||
t_open_ind = -1 # -1 ends the loop
|
||||
|
||||
if stop_stops_count >= stop_stops: # if maximum number of stops allowed in a pair is hit, exit loop
|
||||
t_open_ind = -1 # -1 ends the loop
|
||||
if debug:
|
||||
print("Max stop limit ", stop_stops, "reached. Moving to next pair")
|
||||
|
||||
return t_open_ind
|
||||
|
||||
def backslap_pair(self, ticker_data, pair):
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import timeit
|
||||
import utils_find_1st as utf1st
|
||||
from datetime import datetime
|
||||
|
||||
### backslap debug wrap
|
||||
# debug_2loops = False # only loop twice, for faster debug
|
||||
# debug_timing = False # print timing for each step
|
||||
# debug = False # print values, to check accuracy
|
||||
debug_2loops = self.debug_2loops # only loop twice, for faster debug
|
||||
debug_timing = self.debug_timing # print timing for each step
|
||||
debug = self.debug # print values, to check accuracy
|
||||
|
||||
# Read Stop Loss Values and Stake
|
||||
stop = self.stop_loss_value
|
||||
p_stop = (stop + 1) # What stop really means, e.g 0.01 is 0.99 of price
|
||||
|
||||
if debug:
|
||||
print("Stop is ", stop, "value from stragey file")
|
||||
print("p_stop is", p_stop, "value used to multiply to entry price")
|
||||
|
||||
if debug:
|
||||
from pandas import set_option
|
||||
set_option('display.max_rows', 5000)
|
||||
set_option('display.max_columns', 8)
|
||||
pd.set_option('display.width', 1000)
|
||||
pd.set_option('max_colwidth', 40)
|
||||
pd.set_option('precision', 12)
|
||||
|
||||
def s():
|
||||
st = timeit.default_timer()
|
||||
return st
|
||||
|
||||
def f(st):
|
||||
return (timeit.default_timer() - st)
|
||||
|
||||
#### backslap config
|
||||
'''
|
||||
Numpy arrays are used for 100x speed up
|
||||
We requires setting Int values for
|
||||
buy stop triggers and stop calculated on
|
||||
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
|
||||
'''
|
||||
|
||||
#######
|
||||
# Use vars set at top of backtest
|
||||
np_buy: int = self.np_buy
|
||||
np_open: int = self.np_open
|
||||
np_close: int = self.np_close
|
||||
np_sell: int = self.np_sell
|
||||
np_high: int = self.np_high
|
||||
np_low: int = self.np_low
|
||||
np_stop: int = self.np_stop
|
||||
np_bto: int = self.np_bto # buys_triggered_on - should be close
|
||||
np_bco: int = self.np_bco # buys calculated on - open of the next candle.
|
||||
np_sto: int = self.np_sto # stops_triggered_on - Should be low, FT uses close
|
||||
np_sco: int = self.np_sco # stops_calculated_on - Should be stop, FT uses close
|
||||
|
||||
### End Config
|
||||
|
||||
pair: str = pair
|
||||
|
||||
# ticker_data: DataFrame = ticker_dfs[t_file]
|
||||
bslap: DataFrame = ticker_data
|
||||
|
||||
# Build a single dimension numpy array from "buy" index for faster search
|
||||
# (500x faster than pandas)
|
||||
np_buy_arr = bslap['buy'].values
|
||||
np_buy_arr_len: int = len(np_buy_arr)
|
||||
|
||||
# use numpy array for faster searches in loop, 20x faster than pandas
|
||||
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
|
||||
np_bslap = np.array(bslap[['buy', 'open', 'close', 'sell', 'high', 'low']])
|
||||
|
||||
# Build a numpy list of date-times.
|
||||
# We use these when building the trade
|
||||
# The rationale is to address a value from a pandas cell is thousands of
|
||||
# times more expensive. Processing time went X25 when trying to use any data from pandas
|
||||
np_bslap_dates = bslap['date'].values
|
||||
|
||||
loop: int = 0 # how many time around the loop
|
||||
t_exit_ind = 0 # Start loop from first index
|
||||
t_exit_last = 0 # To test for exit
|
||||
|
||||
stop_stops = self.stop_stops # Int of stops within a pair to stop trading a pair at
|
||||
stop_stops_count = 0 # stop counter per pair
|
||||
|
||||
st = s() # Start timer for processing dataframe
|
||||
if debug:
|
||||
print('Processing:', pair)
|
||||
|
||||
# Results will be stored in a list of dicts
|
||||
bslap_pair_results: list = []
|
||||
bslap_result: dict = {}
|
||||
|
||||
while t_exit_ind < np_buy_arr_len:
|
||||
loop = loop + 1
|
||||
if debug or debug_timing:
|
||||
print("-- T_exit_Ind - Numpy Index is", t_exit_ind, " ----------------------- Loop", loop, pair)
|
||||
if debug_2loops:
|
||||
if loop == 3:
|
||||
print(
|
||||
"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Loop debug max met - breaking")
|
||||
break
|
||||
'''
|
||||
Dev phases
|
||||
Phase 1
|
||||
1) Manage buy, sell, stop enter/exit
|
||||
a) Find first buy index
|
||||
b) Discover first stop and sell hit after buy index
|
||||
c) Chose first instance as trade exit
|
||||
|
||||
Phase 2
|
||||
2) Manage dynamic Stop and ROI Exit
|
||||
a) Create trade slice from 1
|
||||
b) search within trade slice for dynamice stop hit
|
||||
c) search within trade slice for ROI hit
|
||||
'''
|
||||
|
||||
if debug_timing:
|
||||
st = s()
|
||||
'''
|
||||
0 - Find next buy entry
|
||||
Finds index for first (buy = 1) flag
|
||||
|
||||
Requires: np_buy_arr - a 1D array of the 'buy' column. To find next "1"
|
||||
Required: t_exit_ind - Either 0, first loop. Or The index we last exited on
|
||||
Requires: np_buy_arr_len - length of pair array.
|
||||
Requires: stops_stops - number of stops allowed before stop trading a pair
|
||||
Requires: stop_stop_counts - count of stops hit in the pair
|
||||
Provides: The next "buy" index after t_exit_ind
|
||||
|
||||
If -1 is returned no buy has been found in remainder of array, skip to exit loop
|
||||
'''
|
||||
t_open_ind = self.np_get_t_open_ind(np_buy_arr, t_exit_ind, np_buy_arr_len, stop_stops, stop_stops_count)
|
||||
|
||||
if debug:
|
||||
print("\n(0) numpy debug \nnp_get_t_open, has returned the next valid buy index as", t_open_ind)
|
||||
print("If -1 there are no valid buys in the remainder of ticker data. Skipping to end of loop")
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("0-numpy", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
if t_open_ind != -1:
|
||||
|
||||
"""
|
||||
1 - Create views to search within for our open trade
|
||||
|
||||
The views are our search space for the next Stop or Sell
|
||||
Numpy view is employed as:
|
||||
1,000 faster than pandas searches
|
||||
Pandas cannot assure it will always return a view, it may make a slow copy.
|
||||
|
||||
The view contains columns:
|
||||
buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
|
||||
|
||||
Requires: np_bslap is our numpy array of the ticker DataFrame
|
||||
Requires: t_open_ind is the index row with the buy.
|
||||
Provides: np_t_open_v View of array after buy.
|
||||
Provides: np_t_open_v_stop View of array after buy +1
|
||||
(Stop will search in here to prevent stopping in the past)
|
||||
"""
|
||||
np_t_open_v = np_bslap[t_open_ind:]
|
||||
np_t_open_v_stop = np_bslap[t_open_ind + 1:]
|
||||
|
||||
if debug:
|
||||
print("\n(1) numpy debug \nNumpy view row 0 is now Ticker_Data Index", t_open_ind)
|
||||
print("Numpy View: Buy - Open - Close - Sell - High - Low")
|
||||
print("Row 0", np_t_open_v[0])
|
||||
print("Row 1", np_t_open_v[1], )
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("2-numpy", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
'''
|
||||
2 - Calculate our stop-loss price
|
||||
|
||||
As stop is based on buy price of our trade
|
||||
- (BTO)Buys are Triggered On np_bto, typically the CLOSE of candle
|
||||
- (BCO)Buys are Calculated On np_bco, default is OPEN of the next candle.
|
||||
This is as we only see the CLOSE after it has happened.
|
||||
The back test assumption is we have bought at first available price, the OPEN
|
||||
|
||||
Requires: np_bslap - is our numpy array of the ticker DataFrame
|
||||
Requires: t_open_ind - is the index row with the first buy.
|
||||
Requires: p_stop - is the stop rate, ie. 0.99 is -1%
|
||||
Provides: np_t_stop_pri - The value stop-loss will be triggered on
|
||||
'''
|
||||
np_t_stop_pri = (np_bslap[t_open_ind + 1, np_bco] * p_stop)
|
||||
|
||||
if debug:
|
||||
print("\n(2) numpy debug\nStop-Loss has been calculated at:", np_t_stop_pri)
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("2-numpy", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
'''
|
||||
3 - Find candle STO is under Stop-Loss After Trade opened.
|
||||
|
||||
where [np_sto] (stop tiggered on variable: "close", "low" etc) < np_t_stop_pri
|
||||
|
||||
Requires: np_t_open_v_stop Numpy view of ticker_data after buy row +1 (when trade was opened)
|
||||
Requires: np_sto User Var(STO)StopTriggeredOn. Typically set to "low" or "close"
|
||||
Requires: np_t_stop_pri The stop-loss price STO must fall under to trigger stop
|
||||
Provides: np_t_stop_ind The first candle after trade open where STO is under stop-loss
|
||||
'''
|
||||
np_t_stop_ind = utf1st.find_1st(np_t_open_v_stop[:, np_sto],
|
||||
np_t_stop_pri,
|
||||
utf1st.cmp_smaller)
|
||||
|
||||
# plus 1 as np_t_open_v_stop is 1 ahead of view np_t_open_v, used from here on out.
|
||||
np_t_stop_ind = np_t_stop_ind + 1
|
||||
|
||||
if debug:
|
||||
print("\n(3) numpy debug\nNext view index with STO (stop trigger on) under Stop-Loss is",
|
||||
np_t_stop_ind - 1,
|
||||
". STO is using field", np_sto,
|
||||
"\nFrom key: buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5\n")
|
||||
|
||||
print(
|
||||
"If -1 or 0 returned there is no stop found to end of view, then next two array lines are garbage")
|
||||
print("Row", np_t_stop_ind - 1, np_t_open_v[np_t_stop_ind])
|
||||
print("Row", np_t_stop_ind, np_t_open_v[np_t_stop_ind + 1])
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("3-numpy", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
'''
|
||||
4 - Find first sell index after trade open
|
||||
|
||||
First index in the view np_t_open_v where ['sell'] = 1
|
||||
|
||||
Requires: np_t_open_v - view of ticker_data from buy onwards
|
||||
Requires: no_sell - integer '3', the buy column in the array
|
||||
Provides: np_t_sell_ind index of view where first sell=1 after buy
|
||||
'''
|
||||
# Use numpy array for faster search for sell
|
||||
# Sell uses column 3.
|
||||
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
|
||||
# Numpy searches 25-35x quicker than pandas on this data
|
||||
|
||||
np_t_sell_ind = utf1st.find_1st(np_t_open_v[:, np_sell],
|
||||
1, utf1st.cmp_equal)
|
||||
if debug:
|
||||
print("\n(4) numpy debug\nNext view index with sell = 1 is ", np_t_sell_ind)
|
||||
print("If 0 or less is returned there is no sell found to end of view, then next lines garbage")
|
||||
print("Row", np_t_sell_ind, np_t_open_v[np_t_sell_ind])
|
||||
print("Row", np_t_sell_ind + 1, np_t_open_v[np_t_sell_ind + 1])
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("4-numpy", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
'''
|
||||
5 - Determine which was hit first a stop or sell
|
||||
To then use as exit index price-field (sell on buy, stop on stop)
|
||||
|
||||
STOP takes priority over SELL as would be 'in candle' from tick data
|
||||
Sell would use Open from Next candle.
|
||||
So in a draw Stop would be hit first on ticker data in live
|
||||
|
||||
Validity of when types of trades may be executed can be summarised as:
|
||||
|
||||
Tick View
|
||||
index index Buy Sell open low close high Stop price
|
||||
open 2am 94 -1 0 0 ----- ------ ------ ----- -----
|
||||
open 3am 95 0 1 0 ----- ------ trg buy ----- -----
|
||||
open 4am 96 1 0 1 Enter trgstop trg sel ROI out Stop out
|
||||
open 5am 97 2 0 0 Exit ------ ------- ----- -----
|
||||
open 6am 98 3 0 0 ----- ------ ------- ----- -----
|
||||
|
||||
-1 means not found till end of view i.e no valid Stop found. Exclude from match.
|
||||
Stop tiggering and closing in 96-1, the candle we bought at OPEN in, is valid.
|
||||
|
||||
Buys and sells are triggered at candle close
|
||||
Both will open their postions at the open of the next candle. i/e + 1 index
|
||||
|
||||
Stop and buy Indexes are on the view. To map to the ticker dataframe
|
||||
the t_open_ind index should be summed.
|
||||
|
||||
np_t_stop_ind: Stop Found index in view
|
||||
t_exit_ind : Sell found in view
|
||||
t_open_ind : Where view was started on ticker_data
|
||||
|
||||
TODO: fix this frig for logic test,, case/switch/dictionary would be better...
|
||||
more so when later testing many options, dynamic stop / roi etc
|
||||
cludge - Setting np_t_sell_ind as 9999999999 when -1 (not found)
|
||||
cludge - Setting np_t_stop_ind as 9999999999 when -1 (not found)
|
||||
|
||||
'''
|
||||
if debug:
|
||||
print("\n(5) numpy debug\nStop or Sell Logic Processing")
|
||||
|
||||
# cludge for logic test (-1) means it was not found, set crazy high to lose < test
|
||||
np_t_sell_ind = 99999999 if np_t_sell_ind <= 0 else np_t_sell_ind
|
||||
np_t_stop_ind = 99999999 if np_t_stop_ind <= 0 else np_t_stop_ind
|
||||
|
||||
# Stoploss trigger found before a sell =1
|
||||
if np_t_stop_ind < 99999999 and np_t_stop_ind <= np_t_sell_ind:
|
||||
t_exit_ind = t_open_ind + np_t_stop_ind # Set Exit row index
|
||||
t_exit_type = SellType.STOP_LOSS # Set Exit type (stop)
|
||||
np_t_exit_pri = np_sco # The price field our STOP exit will use
|
||||
if debug:
|
||||
print("Type STOP is first exit condition. "
|
||||
"At view index:", np_t_stop_ind, ". Ticker data exit index is", t_exit_ind)
|
||||
|
||||
# Buy = 1 found before a stoploss triggered
|
||||
elif np_t_sell_ind < 99999999 and np_t_sell_ind < np_t_stop_ind:
|
||||
# move sell onto next candle, we only look back on sell
|
||||
# will use the open price later.
|
||||
t_exit_ind = t_open_ind + np_t_sell_ind + 1 # Set Exit row index
|
||||
t_exit_type = SellType.SELL_SIGNAL # Set Exit type (sell)
|
||||
np_t_exit_pri = np_open # The price field our SELL exit will use
|
||||
if debug:
|
||||
print("Type SELL is first exit condition. "
|
||||
"At view index", np_t_sell_ind, ". Ticker data exit index is", t_exit_ind)
|
||||
|
||||
# No stop or buy left in view - set t_exit_last -1 to handle gracefully
|
||||
else:
|
||||
t_exit_last: int = -1 # Signal loop to exit, no buys or sells found.
|
||||
t_exit_type = SellType.NONE
|
||||
np_t_exit_pri = 999 # field price should be calculated on. 999 a non-existent column
|
||||
if debug:
|
||||
print("No valid STOP or SELL found. Signalling t_exit_last to gracefully exit")
|
||||
|
||||
# TODO: fix having to cludge/uncludge this ..
|
||||
# Undo cludge
|
||||
np_t_sell_ind = -1 if np_t_sell_ind == 99999999 else np_t_sell_ind
|
||||
np_t_stop_ind = -1 if np_t_stop_ind == 99999999 else np_t_stop_ind
|
||||
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("5-logic", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
if debug:
|
||||
'''
|
||||
Print out the buys, stops, sells
|
||||
Include Line before and after to for easy
|
||||
Human verification
|
||||
'''
|
||||
# Combine the np_t_stop_pri value to bslap dataframe to make debug
|
||||
# life easy. This is the current stop price based on buy price_
|
||||
# This is slow but don't care about performance in debug
|
||||
#
|
||||
# When referencing equiv np_column, as example np_sto, its 5 in numpy and 6 in df, so +1
|
||||
# as there is no data column in the numpy array.
|
||||
bslap['np_stop_pri'] = np_t_stop_pri
|
||||
|
||||
# Buy
|
||||
print("\n\nDATAFRAME DEBUG =================== BUY ", pair)
|
||||
print("Numpy Array BUY Index is:", 0)
|
||||
print("DataFrame BUY Index is:", t_open_ind, "displaying DF \n")
|
||||
print("HINT, BUY trade should use OPEN price from next candle, i.e ", t_open_ind + 1)
|
||||
op_is = t_open_ind - 1 # Print open index start, line before
|
||||
op_if = t_open_ind + 3 # Print open index finish, line after
|
||||
print(bslap.iloc[op_is:op_if], "\n")
|
||||
|
||||
# Stop - Stops trigger price np_sto (+1 for pandas column), and price received np_sco +1. (Stop Trigger|Calculated On)
|
||||
if np_t_stop_ind < 0:
|
||||
print("DATAFRAME DEBUG =================== STOP ", pair)
|
||||
print("No STOPS were found until the end of ticker data file\n")
|
||||
else:
|
||||
print("DATAFRAME DEBUG =================== STOP ", pair)
|
||||
print("Numpy Array STOP Index is:", np_t_stop_ind, "View starts at index", t_open_ind)
|
||||
df_stop_index = (t_open_ind + np_t_stop_ind)
|
||||
|
||||
print("DataFrame STOP Index is:", df_stop_index, "displaying DF \n")
|
||||
print("First Stoploss trigger after Trade entered at OPEN in candle", t_open_ind + 1, "is ",
|
||||
df_stop_index, ": \n",
|
||||
str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sto + 1]),
|
||||
"is less than", str.format('{0:.17f}', np_t_stop_pri))
|
||||
|
||||
print("A stoploss exit will be calculated at rate:",
|
||||
str.format('{0:.17f}', bslap.iloc[df_stop_index][np_sco + 1]))
|
||||
|
||||
print("\nHINT, STOPs should exit in-candle, i.e", df_stop_index,
|
||||
": As live STOPs are not linked to O-C times")
|
||||
|
||||
st_is = df_stop_index - 1 # Print stop index start, line before
|
||||
st_if = df_stop_index + 2 # Print stop index finish, line after
|
||||
print(bslap.iloc[st_is:st_if], "\n")
|
||||
|
||||
# Sell
|
||||
if np_t_sell_ind < 0:
|
||||
print("DATAFRAME DEBUG =================== SELL ", pair)
|
||||
print("No SELLS were found till the end of ticker data file\n")
|
||||
else:
|
||||
print("DATAFRAME DEBUG =================== SELL ", pair)
|
||||
print("Numpy View SELL Index is:", np_t_sell_ind, "View starts at index", t_open_ind)
|
||||
df_sell_index = (t_open_ind + np_t_sell_ind)
|
||||
|
||||
print("DataFrame SELL Index is:", df_sell_index, "displaying DF \n")
|
||||
print("First Sell Index after Trade open is in candle", df_sell_index)
|
||||
print("HINT, if exit is SELL (not stop) trade should use OPEN price from next candle",
|
||||
df_sell_index + 1)
|
||||
sl_is = df_sell_index - 1 # Print sell index start, line before
|
||||
sl_if = df_sell_index + 3 # Print sell index finish, line after
|
||||
print(bslap.iloc[sl_is:sl_if], "\n")
|
||||
|
||||
# Chosen Exit (stop or sell)
|
||||
|
||||
print("DATAFRAME DEBUG =================== EXIT ", pair)
|
||||
print("Exit type is :", t_exit_type)
|
||||
print("trade exit price field is", np_t_exit_pri, "\n")
|
||||
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("6-depra", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
## use numpy view "np_t_open_v" for speed. Columns are
|
||||
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5
|
||||
# exception is 6 which is use the stop value.
|
||||
|
||||
# TODO no! this is hard coded bleh fix this open
|
||||
np_trade_enter_price = np_bslap[t_open_ind + 1, np_open]
|
||||
if t_exit_type == SellType.STOP_LOSS:
|
||||
if np_t_exit_pri == 6:
|
||||
np_trade_exit_price = np_t_stop_pri
|
||||
else:
|
||||
np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri]
|
||||
if t_exit_type == SellType.SELL_SIGNAL:
|
||||
np_trade_exit_price = np_bslap[t_exit_ind, np_t_exit_pri]
|
||||
|
||||
# Catch no exit found
|
||||
if t_exit_type == SellType.NONE:
|
||||
np_trade_exit_price = 0
|
||||
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("7-numpy", str.format('{0:.17f}', t_t))
|
||||
st = s()
|
||||
|
||||
if debug:
|
||||
print("//////////////////////////////////////////////")
|
||||
print("+++++++++++++++++++++++++++++++++ Trade Enter ")
|
||||
print("np_trade Enter Price is ", str.format('{0:.17f}', np_trade_enter_price))
|
||||
print("--------------------------------- Trade Exit ")
|
||||
print("Trade Exit Type is ", t_exit_type)
|
||||
print("np_trade Exit Price is", str.format('{0:.17f}', np_trade_exit_price))
|
||||
print("//////////////////////////////////////////////")
|
||||
|
||||
else: # no buys were found, step 0 returned -1
|
||||
# Gracefully exit the loop
|
||||
t_exit_last == -1
|
||||
if debug:
|
||||
print("\n(E) No buys were found in remaining ticker file. Exiting", pair)
|
||||
|
||||
# Loop control - catch no closed trades.
|
||||
if debug:
|
||||
print("---------------------------------------- end of loop", loop,
|
||||
" Dataframe Exit Index is: ", t_exit_ind)
|
||||
print("Exit Index Last, Exit Index Now Are: ", t_exit_last, t_exit_ind)
|
||||
|
||||
if t_exit_last >= t_exit_ind or t_exit_last == -1:
|
||||
"""
|
||||
Break loop and go on to next pair.
|
||||
|
||||
When last trade exit equals index of last exit, there is no
|
||||
opportunity to close any more trades.
|
||||
"""
|
||||
# TODO :add handing here to record none closed open trades
|
||||
|
||||
if debug:
|
||||
print(bslap_pair_results)
|
||||
break
|
||||
else:
|
||||
"""
|
||||
Add trade to backtest looking results list of dicts
|
||||
Loop back to look for more trades.
|
||||
"""
|
||||
# Build trade dictionary
|
||||
## In general if a field can be calculated later from other fields leave blank here
|
||||
## Its X(number of trades faster) to calc all in a single vector than 1 trade at a time
|
||||
|
||||
# create a new dict
|
||||
close_index: int = t_exit_ind
|
||||
bslap_result = {} # Must have at start or we end up with a list of multiple same last result
|
||||
bslap_result["pair"] = pair
|
||||
bslap_result["profit_percent"] = "" # To be 1 vector calc across trades when loop complete
|
||||
bslap_result["profit_abs"] = "" # To be 1 vector calc across trades when loop complete
|
||||
bslap_result["open_time"] = np_bslap_dates[t_open_ind + 1] # use numpy array, pandas 20x slower
|
||||
bslap_result["close_time"] = np_bslap_dates[close_index] # use numpy array, pandas 20x slower
|
||||
bslap_result["open_index"] = t_open_ind + 1 # +1 as we buy on next.
|
||||
bslap_result["close_index"] = close_index
|
||||
bslap_result["trade_duration"] = "" # To be 1 vector calc across trades when loop complete
|
||||
bslap_result["open_at_end"] = False
|
||||
bslap_result["open_rate"] = round(np_trade_enter_price, 15)
|
||||
bslap_result["close_rate"] = round(np_trade_exit_price, 15)
|
||||
bslap_result["exit_type"] = t_exit_type
|
||||
bslap_result["sell_reason"] = t_exit_type #duplicated, but I don't care
|
||||
# append the dict to the list and print list
|
||||
bslap_pair_results.append(bslap_result)
|
||||
|
||||
if t_exit_type is SellType.STOP_LOSS:
|
||||
stop_stops_count = stop_stops_count + 1
|
||||
|
||||
if debug:
|
||||
print("The trade dict is: \n", bslap_result)
|
||||
print("Trades dicts in list after append are: \n ", bslap_pair_results)
|
||||
|
||||
"""
|
||||
Loop back to start. t_exit_last becomes where loop
|
||||
will seek to open new trades from.
|
||||
Push index on 1 to not open on close
|
||||
"""
|
||||
t_exit_last = t_exit_ind + 1
|
||||
|
||||
if debug_timing:
|
||||
t_t = f(st)
|
||||
print("8+trade", str.format('{0:.17f}', t_t))
|
||||
|
||||
# Send back List of trade dicts
|
||||
return bslap_pair_results
|
||||
@ -10,7 +10,7 @@ from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
|
||||
|
||||
import arrow
|
||||
from pandas import DataFrame
|
||||
from pandas import DataFrame, to_datetime
|
||||
from tabulate import tabulate
|
||||
|
||||
import freqtrade.optimize as optimize
|
||||
@ -19,9 +19,14 @@ from freqtrade.arguments import Arguments
|
||||
from freqtrade.configuration import Configuration
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.misc import file_dump_json
|
||||
from freqtrade.optimize.backslapping import Backslapping
|
||||
from freqtrade.persistence import Trade
|
||||
from freqtrade.strategy.interface import SellType
|
||||
from freqtrade.strategy.resolver import IStrategy, StrategyResolver
|
||||
from profilehooks import profile
|
||||
from collections import OrderedDict
|
||||
import timeit
|
||||
from time import sleep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@ -52,6 +57,7 @@ class Backtesting(object):
|
||||
backtesting = Backtesting(config)
|
||||
backtesting.start()
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
self.config = config
|
||||
self.strategy: IStrategy = StrategyResolver(self.config).strategy
|
||||
@ -69,6 +75,49 @@ class Backtesting(object):
|
||||
self.exchange = Exchange(self.config)
|
||||
self.fee = self.exchange.get_fee()
|
||||
|
||||
self.stop_loss_value = self.strategy.stoploss
|
||||
|
||||
#### backslap config
|
||||
'''
|
||||
Numpy arrays are used for 100x speed up
|
||||
We requires setting Int values for
|
||||
buy stop triggers and stop calculated on
|
||||
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
|
||||
'''
|
||||
self.np_buy: int = 0
|
||||
self.np_open: int = 1
|
||||
self.np_close: int = 2
|
||||
self.np_sell: int = 3
|
||||
self.np_high: int = 4
|
||||
self.np_low: int = 5
|
||||
self.np_stop: int = 6
|
||||
self.np_bto: int = self.np_close # buys_triggered_on - should be close
|
||||
self.np_bco: int = self.np_open # buys calculated on - open of the next candle.
|
||||
self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close
|
||||
self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close
|
||||
# self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close
|
||||
# self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close
|
||||
|
||||
if 'backslap' in config:
|
||||
self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed.
|
||||
else:
|
||||
self.use_backslap = False
|
||||
|
||||
logger.info("using backslap: {}".format(self.use_backslap))
|
||||
|
||||
self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended
|
||||
self.debug_timing = False # Stages within Backslap
|
||||
self.debug_2loops = False # Limit each pair to two loops, useful when debugging
|
||||
self.debug_vector = False # Debug vector calcs
|
||||
self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap
|
||||
|
||||
self.backslap_show_trades = False # prints trades in addition to summary report
|
||||
self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt
|
||||
|
||||
self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit
|
||||
|
||||
self.backslap = Backslapping(config)
|
||||
|
||||
@staticmethod
|
||||
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
|
||||
"""
|
||||
@ -81,7 +130,7 @@ class Backtesting(object):
|
||||
for frame in data.values()
|
||||
]
|
||||
return min(timeframe, key=operator.itemgetter(0))[0], \
|
||||
max(timeframe, key=operator.itemgetter(1))[1]
|
||||
max(timeframe, key=operator.itemgetter(1))[1]
|
||||
|
||||
def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str:
|
||||
"""
|
||||
@ -126,10 +175,11 @@ class Backtesting(object):
|
||||
"""
|
||||
Generate small table outlining Backtest results
|
||||
"""
|
||||
|
||||
tabular_data = []
|
||||
headers = ['Sell Reason', 'Count']
|
||||
for reason, count in results['sell_reason'].value_counts().iteritems():
|
||||
tabular_data.append([reason.value, count])
|
||||
tabular_data.append([reason.value, count])
|
||||
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
|
||||
|
||||
def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None:
|
||||
@ -168,14 +218,13 @@ class Backtesting(object):
|
||||
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
|
||||
sell_row.sell)
|
||||
if sell.sell_flag:
|
||||
|
||||
return BacktestResult(pair=pair,
|
||||
profit_percent=trade.calc_profit_percent(rate=sell_row.open),
|
||||
profit_abs=trade.calc_profit(rate=sell_row.open),
|
||||
open_time=buy_row.date,
|
||||
close_time=sell_row.date,
|
||||
trade_duration=int((
|
||||
sell_row.date - buy_row.date).total_seconds() // 60),
|
||||
sell_row.date - buy_row.date).total_seconds() // 60),
|
||||
open_index=buy_row.Index,
|
||||
close_index=sell_row.Index,
|
||||
open_at_end=False,
|
||||
@ -192,7 +241,7 @@ class Backtesting(object):
|
||||
open_time=buy_row.date,
|
||||
close_time=sell_row.date,
|
||||
trade_duration=int((
|
||||
sell_row.date - buy_row.date).total_seconds() // 60),
|
||||
sell_row.date - buy_row.date).total_seconds() // 60),
|
||||
open_index=buy_row.Index,
|
||||
close_index=sell_row.Index,
|
||||
open_at_end=True,
|
||||
@ -205,6 +254,13 @@ class Backtesting(object):
|
||||
return btr
|
||||
return None
|
||||
|
||||
def s(self):
|
||||
st = timeit.default_timer()
|
||||
return st
|
||||
|
||||
def f(self, st):
|
||||
return (timeit.default_timer() - st)
|
||||
|
||||
def backtest(self, args: Dict) -> DataFrame:
|
||||
"""
|
||||
Implements backtesting functionality
|
||||
@ -220,32 +276,50 @@ class Backtesting(object):
|
||||
position_stacking: do we allow position stacking? (default: False)
|
||||
:return: DataFrame
|
||||
"""
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell']
|
||||
processed = args['processed']
|
||||
max_open_trades = args.get('max_open_trades', 0)
|
||||
position_stacking = args.get('position_stacking', False)
|
||||
trades = []
|
||||
trade_count_lock: Dict = {}
|
||||
for pair, pair_data in processed.items():
|
||||
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
|
||||
|
||||
ticker_data = self.advise_sell(
|
||||
use_backslap = self.use_backslap
|
||||
debug_timing = self.debug_timing_main_loop
|
||||
|
||||
if use_backslap: # Use Back Slap code
|
||||
return self.backslap.run(args)
|
||||
else: # use Original Back test code
|
||||
########################## Original BT loop
|
||||
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell']
|
||||
processed = args['processed']
|
||||
max_open_trades = args.get('max_open_trades', 0)
|
||||
position_stacking = args.get('position_stacking', False)
|
||||
trades = []
|
||||
trade_count_lock: Dict = {}
|
||||
|
||||
for pair, pair_data in processed.items():
|
||||
if debug_timing: # Start timer
|
||||
fl = self.s()
|
||||
|
||||
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
|
||||
|
||||
ticker_data = self.advise_sell(
|
||||
self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
|
||||
|
||||
# to avoid using data from future, we buy/sell with signal from previous candle
|
||||
ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
|
||||
ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
|
||||
# to avoid using data from future, we buy/sell with signal from previous candle
|
||||
ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
|
||||
ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
|
||||
|
||||
ticker_data.drop(ticker_data.head(1).index, inplace=True)
|
||||
ticker_data.drop(ticker_data.head(1).index, inplace=True)
|
||||
|
||||
# Convert from Pandas to list for performance reasons
|
||||
# (Looping Pandas is slow.)
|
||||
ticker = [x for x in ticker_data.itertuples()]
|
||||
if debug_timing: # print time taken
|
||||
flt = self.f(fl)
|
||||
# print("populate_buy_trend:", pair, round(flt, 10))
|
||||
st = self.s()
|
||||
|
||||
lock_pair_until = None
|
||||
for index, row in enumerate(ticker):
|
||||
if row.buy == 0 or row.sell == 1:
|
||||
continue # skip rows where no buy signal or that would immediately sell off
|
||||
# Convert from Pandas to list for performance reasons
|
||||
# (Looping Pandas is slow.)
|
||||
ticker = [x for x in ticker_data.itertuples()]
|
||||
|
||||
lock_pair_until = None
|
||||
for index, row in enumerate(ticker):
|
||||
if row.buy == 0 or row.sell == 1:
|
||||
continue # skip rows where no buy signal or that would immediately sell off
|
||||
|
||||
if not position_stacking:
|
||||
if lock_pair_until is not None and row.date <= lock_pair_until:
|
||||
@ -255,20 +329,26 @@ class Backtesting(object):
|
||||
if not trade_count_lock.get(row.date, 0) < max_open_trades:
|
||||
continue
|
||||
|
||||
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
|
||||
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
|
||||
|
||||
trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
|
||||
trade_count_lock, args)
|
||||
trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
|
||||
trade_count_lock, args)
|
||||
|
||||
if trade_entry:
|
||||
lock_pair_until = trade_entry.close_time
|
||||
trades.append(trade_entry)
|
||||
else:
|
||||
# Set lock_pair_until to end of testing period if trade could not be closed
|
||||
# This happens only if the buy-signal was with the last candle
|
||||
lock_pair_until = ticker_data.iloc[-1].date
|
||||
if trade_entry:
|
||||
lock_pair_until = trade_entry.close_time
|
||||
trades.append(trade_entry)
|
||||
else:
|
||||
# Set lock_pair_until to end of testing period if trade could not be closed
|
||||
# This happens only if the buy-signal was with the last candle
|
||||
lock_pair_until = ticker_data.iloc[-1].date
|
||||
|
||||
return DataFrame.from_records(trades, columns=BacktestResult._fields)
|
||||
if debug_timing: # print time taken
|
||||
tt = self.f(st)
|
||||
print("Time to BackTest :", pair, round(tt, 10))
|
||||
print("-----------------------")
|
||||
|
||||
return DataFrame.from_records(trades, columns=BacktestResult._fields)
|
||||
####################### Original BT loop end
|
||||
|
||||
def start(self) -> None:
|
||||
"""
|
||||
@ -289,6 +369,7 @@ class Backtesting(object):
|
||||
|
||||
timerange = Arguments.parse_timerange(None if self.config.get(
|
||||
'timerange') is None else str(self.config.get('timerange')))
|
||||
|
||||
data = optimize.load_data(
|
||||
self.config['datadir'],
|
||||
pairs=pairs,
|
||||
@ -298,6 +379,7 @@ class Backtesting(object):
|
||||
timerange=timerange
|
||||
)
|
||||
|
||||
ld_files = self.s()
|
||||
if not data:
|
||||
logger.critical("No data found. Terminating.")
|
||||
return
|
||||
@ -309,6 +391,8 @@ class Backtesting(object):
|
||||
max_open_trades = 0
|
||||
|
||||
preprocessed = self.tickerdata_to_dataframe(data)
|
||||
t_t = self.f(ld_files)
|
||||
print("Load from json to file to df in mem took", t_t)
|
||||
|
||||
# Print timeframe
|
||||
min_date, max_date = self.get_timeframe(preprocessed)
|
||||
@ -332,27 +416,61 @@ class Backtesting(object):
|
||||
if self.config.get('export', False):
|
||||
self._store_backtest_result(self.config.get('exportfilename'), results)
|
||||
|
||||
logger.info(
|
||||
'\n' + '=' * 49 +
|
||||
' BACKTESTING REPORT ' +
|
||||
'=' * 50 + '\n'
|
||||
'%s',
|
||||
self._generate_text_table(
|
||||
data,
|
||||
results
|
||||
if self.use_backslap:
|
||||
logger.info(
|
||||
'\n====================================================== '
|
||||
'BackSLAP REPORT'
|
||||
' =======================================================\n'
|
||||
'%s',
|
||||
self._generate_text_table(
|
||||
data,
|
||||
results
|
||||
)
|
||||
)
|
||||
)
|
||||
# logger.info(
|
||||
# results[['sell_reason']].groupby('sell_reason').count()
|
||||
# )
|
||||
# optional print trades
|
||||
if self.backslap_show_trades:
|
||||
TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
|
||||
'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
|
||||
|
||||
logger.info(
|
||||
'\n' +
|
||||
' SELL READON STATS '.center(119, '=') +
|
||||
'\n%s \n',
|
||||
self._generate_text_table_sell_reason(data, results)
|
||||
def to_fwf(df, fname):
|
||||
content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
|
||||
print(content)
|
||||
|
||||
)
|
||||
DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
|
||||
|
||||
# optional save trades
|
||||
if self.backslap_save_trades:
|
||||
TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
|
||||
'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
|
||||
|
||||
def to_fwf(df, fname):
|
||||
content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
|
||||
open(fname, "w").write(content)
|
||||
|
||||
DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
|
||||
|
||||
else:
|
||||
logger.info(
|
||||
'\n================================================= '
|
||||
'BACKTEST REPORT'
|
||||
' ==================================================\n'
|
||||
'%s',
|
||||
self._generate_text_table(
|
||||
data,
|
||||
results
|
||||
)
|
||||
)
|
||||
|
||||
if 'sell_reason' in results.columns:
|
||||
logger.info(
|
||||
'\n' +
|
||||
' SELL READON STATS '.center(119, '=') +
|
||||
'\n%s \n',
|
||||
self._generate_text_table_sell_reason(data, results)
|
||||
|
||||
)
|
||||
else:
|
||||
logger.info("no sell reasons available!")
|
||||
|
||||
logger.info(
|
||||
'\n' +
|
||||
@ -377,7 +495,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]:
|
||||
# Ensure we do not use Exchange credentials
|
||||
config['exchange']['key'] = ''
|
||||
config['exchange']['secret'] = ''
|
||||
|
||||
config['backslap'] = args.backslap
|
||||
if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
|
||||
raise DependencyException('stake amount could not be "%s" for backtesting' %
|
||||
constants.UNLIMITED_STAKE_AMOUNT)
|
||||
|
||||
@ -23,3 +23,10 @@ scikit-optimize==0.5.2
|
||||
|
||||
# Required for plotting data
|
||||
#plotly==3.0.0
|
||||
|
||||
# find first, C search in arrays
|
||||
py_find_1st==1.1.1
|
||||
|
||||
#Load ticker files 30% faster
|
||||
ujson==1.35
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user