adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backtesting rollbacked to develop branch

Browse Source
This commit is contained in:
misagh 2018-09-21 17:54:37 +02:00
parent 21f4b85c7f
commit 2d432bfa95
1 changed files with 141 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

442
freqtrade/optimize/backtesting.py
View File

@ -6,11 +6,13 @@ This module contains the backtesting logic
import logging
import operator
from argparse import Namespace
from copy import deepcopy
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
import arrow
from pandas import DataFrame, to_datetime
from pandas import DataFrame
from tabulate import tabulate
import freqtrade.optimize as optimize
@ -19,15 +21,9 @@ from freqtrade.arguments import Arguments
from freqtrade.configuration import Configuration
from freqtrade.exchange import Exchange
from freqtrade.misc import file_dump_json
from freqtrade.optimize.backslapping import Backslapping
from freqtrade.persistence import Trade
from freqtrade.strategy.interface import SellType
from freqtrade.strategy.resolver import IStrategy, StrategyResolver
from collections import OrderedDict
import timeit
from time import sleep
import pdb
logger = logging.getLogger(__name__)
@ -61,11 +57,6 @@ class Backtesting(object):
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
self.strategy: IStrategy = StrategyResolver(self.config).strategy
self.ticker_interval = self.strategy.ticker_interval
self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe
self.advise_buy = self.strategy.advise_buy
self.advise_sell = self.strategy.advise_sell
# Reset keys for backtesting
self.config['exchange']['key'] = ''
@ -73,51 +64,35 @@ class Backtesting(object):
self.config['exchange']['password'] = ''
self.config['exchange']['uid'] = ''
self.config['dry_run'] = True
self.strategylist: List[IStrategy] = []
if self.config.get('strategy_list', None):
# Force one interval
self.ticker_interval = str(self.config.get('ticker_interval'))
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
self.strategylist.append(StrategyResolver(stratconf).strategy)
else:
# only one strategy
strat = StrategyResolver(self.config).strategy
self.strategylist.append(StrategyResolver(self.config).strategy)
# Load one strategy
self._set_strategy(self.strategylist[0])
self.exchange = Exchange(self.config)
self.fee = self.exchange.get_fee()
self.stop_loss_value = self.strategy.stoploss
#### backslap config
'''
Numpy arrays are used for 100x speed up
We requires setting Int values for
buy stop triggers and stop calculated on
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6
'''
self.np_buy: int = 0
self.np_open: int = 1
self.np_close: int = 2
self.np_sell: int = 3
self.np_high: int = 4
self.np_low: int = 5
self.np_stop: int = 6
self.np_bto: int = self.np_close # buys_triggered_on - should be close
self.np_bco: int = self.np_open # buys calculated on - open of the next candle.
self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close
self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close
# self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close
# self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close
if 'backslap' in config:
self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed.
else:
self.use_backslap = False
logger.info("using backslap: {}".format(self.use_backslap))
self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended
self.debug_timing = False # Stages within Backslap
self.debug_2loops = False # Limit each pair to two loops, useful when debugging
self.debug_vector = False # Debug vector calcs
self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap
self.backslap_show_trades = False # prints trades in addition to summary report
self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt
self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit
self.backslap = Backslapping(config)
def _set_strategy(self, strategy):
"""
Load strategy into backtesting
"""
self.strategy = strategy
self.ticker_interval = self.config.get('ticker_interval')
self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe
self.advise_buy = strategy.advise_buy
self.advise_sell = strategy.advise_sell
@staticmethod
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
@ -131,7 +106,7 @@ class Backtesting(object):
for frame in data.values()
]
return min(timeframe, key=operator.itemgetter(0))[0], \
max(timeframe, key=operator.itemgetter(1))[1]
max(timeframe, key=operator.itemgetter(1))[1]
def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame) -> str:
"""
@ -142,13 +117,10 @@ class Backtesting(object):
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f')
tabular_data = []
# headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
# 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'total loss ab', 'total profit ab', 'Risk Reward Ratio', 'Win Rate']
headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'RRR', 'Win Rate %', 'Required RR']
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
for pair in data:
result = results[results.pair == pair]
win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None
tabular_data.append([
pair,
len(result.index),
@ -158,12 +130,7 @@ class Backtesting(object):
str(timedelta(
minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
len(result[result.profit_abs > 0]),
len(result[result.profit_abs < 0]),
# result[result.profit_abs < 0]['profit_abs'].sum(),
# result[result.profit_abs > 0]['profit_abs'].sum(),
abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))),
win_rate * 100 if win_rate else "nan",
((1 / win_rate) - 1) if win_rate else "nan"
len(result[result.profit_abs < 0])
])
# Append Total
@ -180,88 +147,42 @@ class Backtesting(object):
])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def _generate_text_table_edge_positioning(self, data: Dict[str, Dict], results: DataFrame) -> str:
"""
This is a temporary version of edge positioning calculation.
The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and
other indictaors related to money management periodically (each X minutes) and keep it in a storage.
The calulation will be done per pair and per strategy.
"""
tabular_data = []
headers = ['Number of trades', 'RRR', 'Win Rate %', 'Required RR']
###
# The algorithm should be:
# 1) Removing outliers from dataframe. i.e. all profit_percent which are outside (mean -+ (2 * (standard deviation))).
# 2) Removing pairs with less than X trades (X defined in config).
# 3) Calculating RRR and WR.
# 4) Removing pairs for which WR and RRR are not in an acceptable range (e.x. WR > 95%).
# 5) Sorting the result based on the delta between required RR and RRR.
# Here we assume initial data in order to calculate position size.
# these values will be replaced by exchange info or config
for pair in data:
result = results[results.pair == pair]
# WinRate is calculated as follows: (Number of profitable trades) / (Total Trades)
win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None
# Risk Reward Ratio is calculated as follows: 1 / ((total loss on losing trades / number of losing trades) / (total gain on profitable trades / number of winning trades))
risk_reward_ratio = abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0]))))
# Required Reward Ratio is (1 / WinRate) - 1
required_risk_reward = ((1 / win_rate) - 1) if win_rate else None
#pdb.set_trace()
tabular_data.append([
pair,
len(result.index),
risk_reward_ratio,
win_rate * 100 if win_rate else "nan",
required_risk_reward
])
# for pair in data:
# result = results[results.pair == pair]
# win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None
# tabular_data.append([
# pair,
# #len(result.index),
# #result.profit_percent.mean() * 100.0,
# #result.profit_percent.sum() * 100.0,
# #result.profit_abs.sum(),
# str(timedelta(
# minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
# len(result[result.profit_abs > 0]),
# len(result[result.profit_abs < 0]),
# # result[result.profit_abs < 0]['profit_abs'].sum(),
# # result[result.profit_abs > 0]['profit_abs'].sum(),
# abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))),
# win_rate * 100 if win_rate else "nan",
# ((1 / win_rate) - 1) if win_rate else "nan"
# ])
#return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str:
"""
Generate small table outlining Backtest results
"""
tabular_data = []
headers = ['Sell Reason', 'Count']
for reason, count in results['sell_reason'].value_counts().iteritems():
tabular_data.append([reason.value, count])
tabular_data.append([reason.value, count])
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None:
def _generate_text_table_strategy(self, all_results: dict) -> str:
"""
Generate summary table per strategy
"""
stake_currency = str(self.config.get('stake_currency'))
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f')
tabular_data = []
headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %',
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
for strategy, results in all_results.items():
tabular_data.append([
strategy,
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_abs.sum(),
str(timedelta(
minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def _store_backtest_result(self, recordfilename: str, results: DataFrame,
strategyname: Optional[str] = None) -> None:
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
@ -269,6 +190,11 @@ class Backtesting(object):
for index, t in results.iterrows()]
if records:
if strategyname:
# Inject strategyname to filename
recname = Path(recordfilename)
recordfilename = str(Path.joinpath(
recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix))
logger.info('Dumping backtest results to %s', recordfilename)
file_dump_json(recordfilename, records)
@ -297,13 +223,14 @@ class Backtesting(object):
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
sell_row.sell)
if sell.sell_flag:
return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open),
open_time=buy_row.date,
close_time=sell_row.date,
trade_duration=int((
sell_row.date - buy_row.date).total_seconds() // 60),
sell_row.date - buy_row.date).total_seconds() // 60),
open_index=buy_row.Index,
close_index=sell_row.Index,
open_at_end=False,
@ -320,7 +247,7 @@ class Backtesting(object):
open_time=buy_row.date,
close_time=sell_row.date,
trade_duration=int((
sell_row.date - buy_row.date).total_seconds() // 60),
sell_row.date - buy_row.date).total_seconds() // 60),
open_index=buy_row.Index,
close_index=sell_row.Index,
open_at_end=True,
@ -333,13 +260,6 @@ class Backtesting(object):
return btr
return None
def s(self):
st = timeit.default_timer()
return st
def f(self, st):
return (timeit.default_timer() - st)
def backtest(self, args: Dict) -> DataFrame:
"""
Implements backtesting functionality
@ -355,50 +275,32 @@ class Backtesting(object):
position_stacking: do we allow position stacking? (default: False)
:return: DataFrame
"""
headers = ['date', 'buy', 'open', 'close', 'sell']
processed = args['processed']
max_open_trades = args.get('max_open_trades', 0)
position_stacking = args.get('position_stacking', False)
trades = []
trade_count_lock: Dict = {}
for pair, pair_data in processed.items():
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
use_backslap = self.use_backslap
debug_timing = self.debug_timing_main_loop
if use_backslap: # Use Back Slap code
return self.backslap.run(args)
else: # use Original Back test code
########################## Original BT loop
headers = ['date', 'buy', 'open', 'close', 'sell']
processed = args['processed']
max_open_trades = args.get('max_open_trades', 0)
position_stacking = args.get('position_stacking', False)
trades = []
trade_count_lock: Dict = {}
for pair, pair_data in processed.items():
if debug_timing: # Start timer
fl = self.s()
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
ticker_data = self.advise_sell(
ticker_data = self.advise_sell(
self.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
# to avoid using data from future, we buy/sell with signal from previous candle
ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
# to avoid using data from future, we buy/sell with signal from previous candle
ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1)
ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1)
ticker_data.drop(ticker_data.head(1).index, inplace=True)
ticker_data.drop(ticker_data.head(1).index, inplace=True)
if debug_timing: # print time taken
flt = self.f(fl)
# print("populate_buy_trend:", pair, round(flt, 10))
st = self.s()
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
ticker = [x for x in ticker_data.itertuples()]
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
ticker = [x for x in ticker_data.itertuples()]
lock_pair_until = None
for index, row in enumerate(ticker):
if row.buy == 0 or row.sell == 1:
continue # skip rows where no buy signal or that would immediately sell off
lock_pair_until = None
for index, row in enumerate(ticker):
if row.buy == 0 or row.sell == 1:
continue # skip rows where no buy signal or that would immediately sell off
if not position_stacking:
if lock_pair_until is not None and row.date <= lock_pair_until:
@ -408,26 +310,20 @@ class Backtesting(object):
if not trade_count_lock.get(row.date, 0) < max_open_trades:
continue
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1
trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
trade_count_lock, args)
trade_entry = self._get_sell_trade_entry(pair, row, ticker[index + 1:],
trade_count_lock, args)
if trade_entry:
lock_pair_until = trade_entry.close_time
trades.append(trade_entry)
else:
# Set lock_pair_until to end of testing period if trade could not be closed
# This happens only if the buy-signal was with the last candle
lock_pair_until = ticker_data.iloc[-1].date
if trade_entry:
lock_pair_until = trade_entry.close_time
trades.append(trade_entry)
else:
# Set lock_pair_until to end of testing period if trade could not be closed
# This happens only if the buy-signal was with the last candle
lock_pair_until = ticker_data.iloc[-1].date
if debug_timing: # print time taken
tt = self.f(st)
print("Time to BackTest :", pair, round(tt, 10))
print("-----------------------")
return DataFrame.from_records(trades, columns=BacktestResult._fields)
####################### Original BT loop end
return DataFrame.from_records(trades, columns=BacktestResult._fields)
def start(self) -> None:
"""
@ -448,7 +344,6 @@ class Backtesting(object):
timerange = Arguments.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
data = optimize.load_data(
self.config['datadir'],
pairs=pairs,
@ -458,7 +353,6 @@ class Backtesting(object):
timerange=timerange
)
ld_files = self.s()
if not data:
logger.critical("No data found. Terminating.")
return
@ -468,109 +362,55 @@ class Backtesting(object):
else:
logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
max_open_trades = 0
all_results = {}
preprocessed = self.tickerdata_to_dataframe(data)
t_t = self.f(ld_files)
print("Load from json to file to df in mem took", t_t)
for strat in self.strategylist:
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
self._set_strategy(strat)
# Print timeframe
min_date, max_date = self.get_timeframe(preprocessed)
logger.info(
'Measuring data from %s up to %s (%s days)..',
min_date.isoformat(),
max_date.isoformat(),
(max_date - min_date).days
)
# Execute backtest and print results
results = self.backtest(
{
'stake_amount': self.config.get('stake_amount'),
'processed': preprocessed,
'max_open_trades': max_open_trades,
'position_stacking': self.config.get('position_stacking', False),
}
)
if self.config.get('export', False):
self._store_backtest_result(self.config.get('exportfilename'), results)
if self.use_backslap:
# logger.info(
# '\n====================================================== '
# 'BackSLAP REPORT'
# ' =======================================================\n'
# '%s',
# self._generate_text_table(
# data,
# results
# )
# )
# need to reprocess data every time to populate signals
preprocessed = self.tickerdata_to_dataframe(data)
# Print timeframe
min_date, max_date = self.get_timeframe(preprocessed)
logger.info(
'\n====================================================== '
'Edge positionning REPORT'
' =======================================================\n'
'%s',
self._generate_text_table_edge_positioning(
data,
results
)
)
# optional print trades
if self.backslap_show_trades:
TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
def to_fwf(df, fname):
content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
print(content)
DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
# optional save trades
if self.backslap_save_trades:
TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
def to_fwf(df, fname):
content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
open(fname, "w").write(content)
DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
else:
logger.info(
'\n================================================= '
'BACKTEST REPORT'
' ==================================================\n'
'%s',
self._generate_text_table(
data,
results
)
'Measuring data from %s up to %s (%s days)..',
min_date.isoformat(),
max_date.isoformat(),
(max_date - min_date).days
)
if 'sell_reason' in results.columns:
logger.info(
'\n' +
' SELL READON STATS '.center(119, '=') +
'\n%s \n',
self._generate_text_table_sell_reason(data, results)
# Execute backtest and print results
all_results[self.strategy.get_strategy_name()] = self.backtest(
{
'stake_amount': self.config.get('stake_amount'),
'processed': preprocessed,
'max_open_trades': max_open_trades,
'position_stacking': self.config.get('position_stacking', False),
}
)
else:
logger.info("no sell reasons available!")
logger.info(
'\n' +
' LEFT OPEN TRADES REPORT '.center(119, '=') +
'\n%s',
self._generate_text_table(
data,
results.loc[results.open_at_end]
)
)
for strategy, results in all_results.items():
if self.config.get('export', False):
self._store_backtest_result(self.config['exportfilename'], results,
strategy if len(self.strategylist) > 1 else None)
print(f"Result for strategy {strategy}")
print(' BACKTESTING REPORT '.center(119, '='))
print(self._generate_text_table(data, results))
print(' SELL REASON STATS '.center(119, '='))
print(self._generate_text_table_sell_reason(data, results))
print(' LEFT OPEN TRADES REPORT '.center(119, '='))
print(self._generate_text_table(data, results.loc[results.open_at_end]))
print()
if len(all_results) > 1:
# Print Strategy summary table
print(' Strategy Summary '.center(119, '='))
print(self._generate_text_table_strategy(all_results))
print('\nFor more details, please look at the detail tables above')
def setup_configuration(args: Namespace) -> Dict[str, Any]:
@ -585,7 +425,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]:
# Ensure we do not use Exchange credentials
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
config['backslap'] = args.backslap
if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
raise DependencyException('stake amount could not be "%s" for backtesting' %
constants.UNLIMITED_STAKE_AMOUNT)
@ -605,4 +445,4 @@ def start(args: Namespace) -> None:
# Initialize backtesting object
backtesting = Backtesting(config)
backtesting.start()
backtesting.start()