adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backtesting rollbacked to develop branch

Browse Source
This commit is contained in:
misagh 2018-09-21 17:54:37 +02:00
parent 21f4b85c7f
commit 2d432bfa95
1 changed files with 141 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

336
freqtrade/optimize/backtesting.py
View File

@ -6,11 +6,13 @@ This module contains the backtesting logic
import logging import logging
import operator import operator
from argparse import Namespace from argparse import Namespace
from copy import deepcopy
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional, Tuple from typing import Any, Dict, List, NamedTuple, Optional, Tuple
import arrow import arrow
from pandas import DataFrame, to_datetime from pandas import DataFrame
from tabulate import tabulate from tabulate import tabulate
import freqtrade.optimize as optimize import freqtrade.optimize as optimize
@ -19,15 +21,9 @@ from freqtrade.arguments import Arguments
from freqtrade.configuration import Configuration from freqtrade.configuration import Configuration
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
from freqtrade.misc import file_dump_json from freqtrade.misc import file_dump_json
from freqtrade.optimize.backslapping import Backslapping
from freqtrade.persistence import Trade from freqtrade.persistence import Trade
from freqtrade.strategy.interface import SellType from freqtrade.strategy.interface import SellType
from freqtrade.strategy.resolver import IStrategy, StrategyResolver from freqtrade.strategy.resolver import IStrategy, StrategyResolver
from collections import OrderedDict
import timeit
from time import sleep
import pdb
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -61,11 +57,6 @@ class Backtesting(object):
def __init__(self, config: Dict[str, Any]) -> None: def __init__(self, config: Dict[str, Any]) -> None:
self.config = config self.config = config
self.strategy: IStrategy = StrategyResolver(self.config).strategy
self.ticker_interval = self.strategy.ticker_interval
self.tickerdata_to_dataframe = self.strategy.tickerdata_to_dataframe
self.advise_buy = self.strategy.advise_buy
self.advise_sell = self.strategy.advise_sell
# Reset keys for backtesting # Reset keys for backtesting
self.config['exchange']['key'] = '' self.config['exchange']['key'] = ''
@ -73,51 +64,35 @@ class Backtesting(object):
self.config['exchange']['password'] = '' self.config['exchange']['password'] = ''
self.config['exchange']['uid'] = '' self.config['exchange']['uid'] = ''
self.config['dry_run'] = True self.config['dry_run'] = True
self.strategylist: List[IStrategy] = []
if self.config.get('strategy_list', None):
# Force one interval
self.ticker_interval = str(self.config.get('ticker_interval'))
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
self.strategylist.append(StrategyResolver(stratconf).strategy)
else:
# only one strategy
strat = StrategyResolver(self.config).strategy
self.strategylist.append(StrategyResolver(self.config).strategy)
# Load one strategy
self._set_strategy(self.strategylist[0])
self.exchange = Exchange(self.config) self.exchange = Exchange(self.config)
self.fee = self.exchange.get_fee() self.fee = self.exchange.get_fee()
self.stop_loss_value = self.strategy.stoploss def _set_strategy(self, strategy):
"""
#### backslap config Load strategy into backtesting
''' """
Numpy arrays are used for 100x speed up self.strategy = strategy
We requires setting Int values for self.ticker_interval = self.config.get('ticker_interval')
buy stop triggers and stop calculated on self.tickerdata_to_dataframe = strategy.tickerdata_to_dataframe
# buy 0 - open 1 - close 2 - sell 3 - high 4 - low 5 - stop 6 self.advise_buy = strategy.advise_buy
''' self.advise_sell = strategy.advise_sell
self.np_buy: int = 0
self.np_open: int = 1
self.np_close: int = 2
self.np_sell: int = 3
self.np_high: int = 4
self.np_low: int = 5
self.np_stop: int = 6
self.np_bto: int = self.np_close # buys_triggered_on - should be close
self.np_bco: int = self.np_open # buys calculated on - open of the next candle.
self.np_sto: int = self.np_low # stops_triggered_on - Should be low, FT uses close
self.np_sco: int = self.np_stop # stops_calculated_on - Should be stop, FT uses close
# self.np_sto: int = self.np_close # stops_triggered_on - Should be low, FT uses close
# self.np_sco: int = self.np_close # stops_calculated_on - Should be stop, FT uses close
if 'backslap' in config:
self.use_backslap = config['backslap'] # Enable backslap - if false Orginal code is executed.
else:
self.use_backslap = False
logger.info("using backslap: {}".format(self.use_backslap))
self.debug = False # Main debug enable, very print heavy, enable 2 loops recommended
self.debug_timing = False # Stages within Backslap
self.debug_2loops = False # Limit each pair to two loops, useful when debugging
self.debug_vector = False # Debug vector calcs
self.debug_timing_main_loop = False # print overall timing per pair - works in Backtest and Backslap
self.backslap_show_trades = False # prints trades in addition to summary report
self.backslap_save_trades = True # saves trades as a pretty table to backslap.txt
self.stop_stops: int = 9999 # stop back testing any pair with this many stops, set to 999999 to not hit
self.backslap = Backslapping(config)
@staticmethod @staticmethod
def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: def get_timeframe(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
@ -142,13 +117,10 @@ class Backtesting(object):
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f') floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f')
tabular_data = [] tabular_data = []
# headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
# 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'total loss ab', 'total profit ab', 'Risk Reward Ratio', 'Win Rate']
headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss', 'RRR', 'Win Rate %', 'Required RR'] 'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
for pair in data: for pair in data:
result = results[results.pair == pair] result = results[results.pair == pair]
win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None
tabular_data.append([ tabular_data.append([
pair, pair,
len(result.index), len(result.index),
@ -158,12 +130,7 @@ class Backtesting(object):
str(timedelta( str(timedelta(
minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
len(result[result.profit_abs > 0]), len(result[result.profit_abs > 0]),
len(result[result.profit_abs < 0]), len(result[result.profit_abs < 0])
# result[result.profit_abs < 0]['profit_abs'].sum(),
# result[result.profit_abs > 0]['profit_abs'].sum(),
abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))),
win_rate * 100 if win_rate else "nan",
((1 / win_rate) - 1) if win_rate else "nan"
]) ])
# Append Total # Append Total
@ -180,88 +147,42 @@ class Backtesting(object):
]) ])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe") return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def _generate_text_table_edge_positioning(self, data: Dict[str, Dict], results: DataFrame) -> str:
"""
This is a temporary version of edge positioning calculation.
The function will be eventually moved to a plugin called Edge in order to calculate necessary WR, RRR and
other indictaors related to money management periodically (each X minutes) and keep it in a storage.
The calulation will be done per pair and per strategy.
"""
tabular_data = []
headers = ['Number of trades', 'RRR', 'Win Rate %', 'Required RR']
###
# The algorithm should be:
# 1) Removing outliers from dataframe. i.e. all profit_percent which are outside (mean -+ (2 * (standard deviation))).
# 2) Removing pairs with less than X trades (X defined in config).
# 3) Calculating RRR and WR.
# 4) Removing pairs for which WR and RRR are not in an acceptable range (e.x. WR > 95%).
# 5) Sorting the result based on the delta between required RR and RRR.
# Here we assume initial data in order to calculate position size.
# these values will be replaced by exchange info or config
for pair in data:
result = results[results.pair == pair]
# WinRate is calculated as follows: (Number of profitable trades) / (Total Trades)
win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None
# Risk Reward Ratio is calculated as follows: 1 / ((total loss on losing trades / number of losing trades) / (total gain on profitable trades / number of winning trades))
risk_reward_ratio = abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0]))))
# Required Reward Ratio is (1 / WinRate) - 1
required_risk_reward = ((1 / win_rate) - 1) if win_rate else None
#pdb.set_trace()
tabular_data.append([
pair,
len(result.index),
risk_reward_ratio,
win_rate * 100 if win_rate else "nan",
required_risk_reward
])
# for pair in data:
# result = results[results.pair == pair]
# win_rate = (len(result[result.profit_abs > 0]) / len(result.index)) if (len(result.index) > 0) else None
# tabular_data.append([
# pair,
# #len(result.index),
# #result.profit_percent.mean() * 100.0,
# #result.profit_percent.sum() * 100.0,
# #result.profit_abs.sum(),
# str(timedelta(
# minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
# len(result[result.profit_abs > 0]),
# len(result[result.profit_abs < 0]),
# # result[result.profit_abs < 0]['profit_abs'].sum(),
# # result[result.profit_abs > 0]['profit_abs'].sum(),
# abs(1 / ((result[result.profit_abs < 0]['profit_abs'].sum() / len(result[result.profit_abs < 0])) / (result[result.profit_abs > 0]['profit_abs'].sum() / len(result[result.profit_abs > 0])))),
# win_rate * 100 if win_rate else "nan",
# ((1 / win_rate) - 1) if win_rate else "nan"
# ])
#return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str:
""" """
Generate small table outlining Backtest results Generate small table outlining Backtest results
""" """
tabular_data = [] tabular_data = []
headers = ['Sell Reason', 'Count'] headers = ['Sell Reason', 'Count']
for reason, count in results['sell_reason'].value_counts().iteritems(): for reason, count in results['sell_reason'].value_counts().iteritems():
tabular_data.append([reason.value, count]) tabular_data.append([reason.value, count])
return tabulate(tabular_data, headers=headers, tablefmt="pipe") return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def _store_backtest_result(self, recordfilename: Optional[str], results: DataFrame) -> None: def _generate_text_table_strategy(self, all_results: dict) -> str:
"""
Generate summary table per strategy
"""
stake_currency = str(self.config.get('stake_currency'))
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', 'd', '.1f', '.1f')
tabular_data = []
headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %',
'total profit ' + stake_currency, 'avg duration', 'profit', 'loss']
for strategy, results in all_results.items():
tabular_data.append([
strategy,
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_abs.sum(),
str(timedelta(
minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
return tabulate(tabular_data, headers=headers, floatfmt=floatfmt, tablefmt="pipe")
def _store_backtest_result(self, recordfilename: str, results: DataFrame,
strategyname: Optional[str] = None) -> None:
records = [(t.pair, t.profit_percent, t.open_time.timestamp(), records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
t.close_time.timestamp(), t.open_index - 1, t.trade_duration, t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
@ -269,6 +190,11 @@ class Backtesting(object):
for index, t in results.iterrows()] for index, t in results.iterrows()]
if records: if records:
if strategyname:
# Inject strategyname to filename
recname = Path(recordfilename)
recordfilename = str(Path.joinpath(
recname.parent, f'{recname.stem}-{strategyname}').with_suffix(recname.suffix))
logger.info('Dumping backtest results to %s', recordfilename) logger.info('Dumping backtest results to %s', recordfilename)
file_dump_json(recordfilename, records) file_dump_json(recordfilename, records)
@ -297,6 +223,7 @@ class Backtesting(object):
sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal, sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, buy_signal,
sell_row.sell) sell_row.sell)
if sell.sell_flag: if sell.sell_flag:
return BacktestResult(pair=pair, return BacktestResult(pair=pair,
profit_percent=trade.calc_profit_percent(rate=sell_row.open), profit_percent=trade.calc_profit_percent(rate=sell_row.open),
profit_abs=trade.calc_profit(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open),
@ -333,13 +260,6 @@ class Backtesting(object):
return btr return btr
return None return None
def s(self):
st = timeit.default_timer()
return st
def f(self, st):
return (timeit.default_timer() - st)
def backtest(self, args: Dict) -> DataFrame: def backtest(self, args: Dict) -> DataFrame:
""" """
Implements backtesting functionality Implements backtesting functionality
@ -355,26 +275,13 @@ class Backtesting(object):
position_stacking: do we allow position stacking? (default: False) position_stacking: do we allow position stacking? (default: False)
:return: DataFrame :return: DataFrame
""" """
use_backslap = self.use_backslap
debug_timing = self.debug_timing_main_loop
if use_backslap: # Use Back Slap code
return self.backslap.run(args)
else: # use Original Back test code
########################## Original BT loop
headers = ['date', 'buy', 'open', 'close', 'sell'] headers = ['date', 'buy', 'open', 'close', 'sell']
processed = args['processed'] processed = args['processed']
max_open_trades = args.get('max_open_trades', 0) max_open_trades = args.get('max_open_trades', 0)
position_stacking = args.get('position_stacking', False) position_stacking = args.get('position_stacking', False)
trades = [] trades = []
trade_count_lock: Dict = {} trade_count_lock: Dict = {}
for pair, pair_data in processed.items(): for pair, pair_data in processed.items():
if debug_timing: # Start timer
fl = self.s()
pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run pair_data['buy'], pair_data['sell'] = 0, 0 # cleanup from previous run
ticker_data = self.advise_sell( ticker_data = self.advise_sell(
@ -386,11 +293,6 @@ class Backtesting(object):
ticker_data.drop(ticker_data.head(1).index, inplace=True) ticker_data.drop(ticker_data.head(1).index, inplace=True)
if debug_timing: # print time taken
flt = self.f(fl)
# print("populate_buy_trend:", pair, round(flt, 10))
st = self.s()
# Convert from Pandas to list for performance reasons # Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.) # (Looping Pandas is slow.)
ticker = [x for x in ticker_data.itertuples()] ticker = [x for x in ticker_data.itertuples()]
@ -421,13 +323,7 @@ class Backtesting(object):
# This happens only if the buy-signal was with the last candle # This happens only if the buy-signal was with the last candle
lock_pair_until = ticker_data.iloc[-1].date lock_pair_until = ticker_data.iloc[-1].date
if debug_timing: # print time taken
tt = self.f(st)
print("Time to BackTest :", pair, round(tt, 10))
print("-----------------------")
return DataFrame.from_records(trades, columns=BacktestResult._fields) return DataFrame.from_records(trades, columns=BacktestResult._fields)
####################### Original BT loop end
def start(self) -> None: def start(self) -> None:
""" """
@ -448,7 +344,6 @@ class Backtesting(object):
timerange = Arguments.parse_timerange(None if self.config.get( timerange = Arguments.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange'))) 'timerange') is None else str(self.config.get('timerange')))
data = optimize.load_data( data = optimize.load_data(
self.config['datadir'], self.config['datadir'],
pairs=pairs, pairs=pairs,
@ -458,7 +353,6 @@ class Backtesting(object):
timerange=timerange timerange=timerange
) )
ld_files = self.s()
if not data: if not data:
logger.critical("No data found. Terminating.") logger.critical("No data found. Terminating.")
return return
@ -468,10 +362,14 @@ class Backtesting(object):
else: else:
logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...')
max_open_trades = 0 max_open_trades = 0
all_results = {}
for strat in self.strategylist:
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
self._set_strategy(strat)
# need to reprocess data every time to populate signals
preprocessed = self.tickerdata_to_dataframe(data) preprocessed = self.tickerdata_to_dataframe(data)
t_t = self.f(ld_files)
print("Load from json to file to df in mem took", t_t)
# Print timeframe # Print timeframe
min_date, max_date = self.get_timeframe(preprocessed) min_date, max_date = self.get_timeframe(preprocessed)
@ -483,7 +381,7 @@ class Backtesting(object):
) )
# Execute backtest and print results # Execute backtest and print results
results = self.backtest( all_results[self.strategy.get_strategy_name()] = self.backtest(
{ {
'stake_amount': self.config.get('stake_amount'), 'stake_amount': self.config.get('stake_amount'),
'processed': preprocessed, 'processed': preprocessed,
@ -492,85 +390,27 @@ class Backtesting(object):
} }
) )
for strategy, results in all_results.items():
if self.config.get('export', False): if self.config.get('export', False):
self._store_backtest_result(self.config.get('exportfilename'), results) self._store_backtest_result(self.config['exportfilename'], results,
strategy if len(self.strategylist) > 1 else None)
if self.use_backslap: print(f"Result for strategy {strategy}")
# logger.info( print(' BACKTESTING REPORT '.center(119, '='))
# '\n====================================================== ' print(self._generate_text_table(data, results))
# 'BackSLAP REPORT'
# ' =======================================================\n'
# '%s',
# self._generate_text_table(
# data,
# results
# )
# )
logger.info( print(' SELL REASON STATS '.center(119, '='))
'\n====================================================== ' print(self._generate_text_table_sell_reason(data, results))
'Edge positionning REPORT'
' =======================================================\n'
'%s',
self._generate_text_table_edge_positioning(
data,
results
)
)
# optional print trades
if self.backslap_show_trades:
TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
def to_fwf(df, fname): print(' LEFT OPEN TRADES REPORT '.center(119, '='))
content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql') print(self._generate_text_table(data, results.loc[results.open_at_end]))
print(content) print()
if len(all_results) > 1:
DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt") # Print Strategy summary table
print(' Strategy Summary '.center(119, '='))
# optional save trades print(self._generate_text_table_strategy(all_results))
if self.backslap_save_trades: print('\nFor more details, please look at the detail tables above')
TradesFrame = results.filter(['open_time', 'pair', 'exit_type', 'profit_percent', 'profit_abs',
'buy_spend', 'sell_take', 'trade_duration', 'close_time'], axis=1)
def to_fwf(df, fname):
content = tabulate(df.values.tolist(), list(df.columns), floatfmt=".8f", tablefmt='psql')
open(fname, "w").write(content)
DataFrame.to_fwf = to_fwf(TradesFrame, "backslap.txt")
else:
logger.info(
'\n================================================= '
'BACKTEST REPORT'
' ==================================================\n'
'%s',
self._generate_text_table(
data,
results
)
)
if 'sell_reason' in results.columns:
logger.info(
'\n' +
' SELL READON STATS '.center(119, '=') +
'\n%s \n',
self._generate_text_table_sell_reason(data, results)
)
else:
logger.info("no sell reasons available!")
logger.info(
'\n' +
' LEFT OPEN TRADES REPORT '.center(119, '=') +
'\n%s',
self._generate_text_table(
data,
results.loc[results.open_at_end]
)
)
def setup_configuration(args: Namespace) -> Dict[str, Any]: def setup_configuration(args: Namespace) -> Dict[str, Any]:
@ -585,7 +425,7 @@ def setup_configuration(args: Namespace) -> Dict[str, Any]:
# Ensure we do not use Exchange credentials # Ensure we do not use Exchange credentials
config['exchange']['key'] = '' config['exchange']['key'] = ''
config['exchange']['secret'] = '' config['exchange']['secret'] = ''
config['backslap'] = args.backslap
if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT:
raise DependencyException('stake amount could not be "%s" for backtesting' % raise DependencyException('stake amount could not be "%s" for backtesting' %
constants.UNLIMITED_STAKE_AMOUNT) constants.UNLIMITED_STAKE_AMOUNT)