adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2730 from freqtrade/extract_bt_reporting

Browse Source 
Extract backtest reporting
This commit is contained in:
Matthias 2020-01-09 06:09:05 +01:00 committed by GitHub
commit b25f28d1ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 226 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
freqtrade/optimize/backtest_reports.py Normal file
View File

@ -0,0 +1,107 @@
from datetime import timedelta
from typing import Dict
from pandas import DataFrame
from tabulate import tabulate
def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
results: DataFrame, skip_nan: bool = False) -> str:
"""
Generates and returns a text table for the given backtest data and the results dataframe
:param data: Dict of <pair: dataframe> containing data that was used during backtesting.
:param stake_currency: stake-currency - used to correctly name headers
:param max_open_trades: Maximum allowed open trades
:param results: Dataframe containing the backtest results
:param skip_nan: Print "left open" open trades
:return: pretty printed table with tabulate as string
"""
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
tabular_data = []
headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
f'tot profit {stake_currency}', 'tot profit %', 'avg duration',
'profit', 'loss']
for pair in data:
result = results[results.pair == pair]
if skip_nan and result.profit_abs.isnull().all():
continue
tabular_data.append([
pair,
len(result.index),
result.profit_percent.mean() * 100.0,
result.profit_percent.sum() * 100.0,
result.profit_abs.sum(),
result.profit_percent.sum() * 100.0 / max_open_trades,
str(timedelta(
minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
len(result[result.profit_abs > 0]),
len(result[result.profit_abs < 0])
])
# Append Total
tabular_data.append([
'TOTAL',
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_abs.sum(),
results.profit_percent.sum() * 100.0 / max_open_trades,
str(timedelta(
minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore
def generate_text_table_sell_reason(data: Dict[str, Dict], results: DataFrame) -> str:
"""
Generate small table outlining Backtest results
:param data: Dict of <pair: dataframe> containing data that was used during backtesting.
:param results: Dataframe containing the backtest results
:return: pretty printed table with tabulate as string
"""
tabular_data = []
headers = ['Sell Reason', 'Count', 'Profit', 'Loss']
for reason, count in results['sell_reason'].value_counts().iteritems():
profit = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] >= 0)])
loss = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] < 0)])
tabular_data.append([reason.value, count, profit, loss])
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def generate_text_table_strategy(stake_currency: str, max_open_trades: str,
all_results: Dict) -> str:
"""
Generate summary table per strategy
:param stake_currency: stake-currency - used to correctly name headers
:param max_open_trades: Maximum allowed open trades used for backtest
:param all_results: Dict of <Strategyname: BacktestResult> containing results for all strategies
:return: pretty printed table with tabulate as string
"""
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
tabular_data = []
headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %',
f'tot profit {stake_currency}', 'tot profit %', 'avg duration',
'profit', 'loss']
for strategy, results in all_results.items():
tabular_data.append([
strategy,
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_abs.sum(),
results.profit_percent.sum() * 100.0 / max_open_trades,
str(timedelta(
minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore

110
freqtrade/optimize/backtesting.py
View File

@ -10,7 +10,6 @@ from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional
from pandas import DataFrame
from tabulate import tabulate
from freqtrade.configuration import (TimeRange, remove_credentials,
validate_config_consistency)
@ -19,6 +18,9 @@ from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.misc import file_dump_json
from freqtrade.optimize.backtest_reports import (
generate_text_table, generate_text_table_sell_reason,
generate_text_table_strategy)
from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
@ -129,96 +131,6 @@ class Backtesting:
return data, timerange
def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame,
skip_nan: bool = False) -> str:
"""
Generates and returns a text table for the given backtest data and the results dataframe
:return: pretty printed table with tabulate as str
"""
stake_currency = str(self.config.get('stake_currency'))
max_open_trades = self.config.get('max_open_trades')
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
tabular_data = []
headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
'tot profit ' + stake_currency, 'tot profit %', 'avg duration',
'profit', 'loss']
for pair in data:
result = results[results.pair == pair]
if skip_nan and result.profit_abs.isnull().all():
continue
tabular_data.append([
pair,
len(result.index),
result.profit_percent.mean() * 100.0,
result.profit_percent.sum() * 100.0,
result.profit_abs.sum(),
result.profit_percent.sum() * 100.0 / max_open_trades,
str(timedelta(
minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
len(result[result.profit_abs > 0]),
len(result[result.profit_abs < 0])
])
# Append Total
tabular_data.append([
'TOTAL',
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_abs.sum(),
results.profit_percent.sum() * 100.0 / max_open_trades,
str(timedelta(
minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore
def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str:
"""
Generate small table outlining Backtest results
"""
tabular_data = []
headers = ['Sell Reason', 'Count', 'Profit', 'Loss']
for reason, count in results['sell_reason'].value_counts().iteritems():
profit = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] >= 0)])
loss = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] < 0)])
tabular_data.append([reason.value, count, profit, loss])
return tabulate(tabular_data, headers=headers, tablefmt="pipe")
def _generate_text_table_strategy(self, all_results: dict) -> str:
"""
Generate summary table per strategy
"""
stake_currency = str(self.config.get('stake_currency'))
max_open_trades = self.config.get('max_open_trades')
floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
tabular_data = []
headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %',
'tot profit ' + stake_currency, 'tot profit %', 'avg duration',
'profit', 'loss']
for strategy, results in all_results.items():
tabular_data.append([
strategy,
len(results.index),
results.profit_percent.mean() * 100.0,
results.profit_percent.sum() * 100.0,
results.profit_abs.sum(),
results.profit_percent.sum() * 100.0 / max_open_trades,
str(timedelta(
minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
len(results[results.profit_abs > 0]),
len(results[results.profit_abs < 0])
])
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(tabular_data, headers=headers,
floatfmt=floatfmt, tablefmt="pipe") # type: ignore
def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
strategyname: Optional[str] = None) -> None:
@ -509,16 +421,24 @@ class Backtesting:
print(f"Result for strategy {strategy}")
print(' BACKTESTING REPORT '.center(133, '='))
print(self._generate_text_table(data, results))
print(generate_text_table(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results))
print(' SELL REASON STATS '.center(133, '='))
print(self._generate_text_table_sell_reason(data, results))
print(generate_text_table_sell_reason(data, results))
print(' LEFT OPEN TRADES REPORT '.center(133, '='))
print(self._generate_text_table(data, results.loc[results.open_at_end], True))
print(generate_text_table(data,
stake_currency=self.config['stake_currency'],
max_open_trades=self.config['max_open_trades'],
results=results.loc[results.open_at_end], skip_nan=True))
print()
if len(all_results) > 1:
# Print Strategy summary table
print(' Strategy Summary '.center(133, '='))
print(self._generate_text_table_strategy(all_results))
print(generate_text_table_strategy(self.config['stake_currency'],
self.config['max_open_trades'],
all_results=all_results))
print('\nFor more details, please look at the detail tables above')

96
tests/optimize/test_backtest_reports.py Normal file
View File

@ -0,0 +1,96 @@
import pandas as pd
from freqtrade.optimize.backtest_reports import (
generate_text_table, generate_text_table_sell_reason,
generate_text_table_strategy)
from freqtrade.strategy.interface import SellType
def test_generate_text_table(default_conf, mocker):
results = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2],
'profit_abs': [0.2, 0.4],
'trade_duration': [10, 30],
'profit': [2, 0],
'loss': [0, 0]
}
)
result_str = (
'| pair | buy count | avg profit % | cum profit % | '
'tot profit BTC | tot profit % | avg duration | profit | loss |\n'
'|:--------|------------:|---------------:|---------------:|'
'-----------------:|---------------:|:---------------|---------:|-------:|\n'
'| ETH/BTC | 2 | 15.00 | 30.00 | '
'0.60000000 | 15.00 | 0:20:00 | 2 | 0 |\n'
'| TOTAL | 2 | 15.00 | 30.00 | '
'0.60000000 | 15.00 | 0:20:00 | 2 | 0 |'
)
assert generate_text_table(data={'ETH/BTC': {}},
stake_currency='BTC', max_open_trades=2,
results=results) == result_str
def test_generate_text_table_sell_reason(default_conf, mocker):
results = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2, -0.3],
'profit_abs': [0.2, 0.4, -0.5],
'trade_duration': [10, 30, 10],
'profit': [2, 0, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
result_str = (
'| Sell Reason | Count | Profit | Loss |\n'
'|:--------------|--------:|---------:|-------:|\n'
'| roi | 2 | 2 | 0 |\n'
'| stop_loss | 1 | 0 | 1 |'
)
assert generate_text_table_sell_reason(
data={'ETH/BTC': {}}, results=results) == result_str
def test_generate_text_table_strategy(default_conf, mocker):
results = {}
results['ETH/BTC'] = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2, 0.3],
'profit_abs': [0.2, 0.4, 0.5],
'trade_duration': [10, 30, 10],
'profit': [2, 0, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
results['LTC/BTC'] = pd.DataFrame(
{
'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'],
'profit_percent': [0.4, 0.2, 0.3],
'profit_abs': [0.4, 0.4, 0.5],
'trade_duration': [15, 30, 15],
'profit': [4, 1, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
result_str = (
'| Strategy | buy count | avg profit % | cum profit % '
'| tot profit BTC | tot profit % | avg duration | profit | loss |\n'
'|:-----------|------------:|---------------:|---------------:'
'|-----------------:|---------------:|:---------------|---------:|-------:|\n'
'| ETH/BTC | 3 | 20.00 | 60.00 '
'| 1.10000000 | 30.00 | 0:17:00 | 3 | 0 |\n'
'| LTC/BTC | 3 | 30.00 | 90.00 '
'| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |'
)
assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str

121
tests/optimize/test_backtesting.py
View File

@ -358,105 +358,6 @@ def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC'])
def test_generate_text_table(default_conf, mocker):
patch_exchange(mocker)
default_conf['max_open_trades'] = 2
backtesting = Backtesting(default_conf)
results = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2],
'profit_abs': [0.2, 0.4],
'trade_duration': [10, 30],
'profit': [2, 0],
'loss': [0, 0]
}
)
result_str = (
'| pair | buy count | avg profit % | cum profit % | '
'tot profit BTC | tot profit % | avg duration | profit | loss |\n'
'|:--------|------------:|---------------:|---------------:|'
'-----------------:|---------------:|:---------------|---------:|-------:|\n'
'| ETH/BTC | 2 | 15.00 | 30.00 | '
'0.60000000 | 15.00 | 0:20:00 | 2 | 0 |\n'
'| TOTAL | 2 | 15.00 | 30.00 | '
'0.60000000 | 15.00 | 0:20:00 | 2 | 0 |'
)
assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str
def test_generate_text_table_sell_reason(default_conf, mocker):
patch_exchange(mocker)
backtesting = Backtesting(default_conf)
results = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2, -0.3],
'profit_abs': [0.2, 0.4, -0.5],
'trade_duration': [10, 30, 10],
'profit': [2, 0, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
result_str = (
'| Sell Reason | Count | Profit | Loss |\n'
'|:--------------|--------:|---------:|-------:|\n'
'| roi | 2 | 2 | 0 |\n'
'| stop_loss | 1 | 0 | 1 |'
)
assert backtesting._generate_text_table_sell_reason(
data={'ETH/BTC': {}}, results=results) == result_str
def test_generate_text_table_strategyn(default_conf, mocker):
"""
Test Backtesting.generate_text_table_sell_reason() method
"""
patch_exchange(mocker)
default_conf['max_open_trades'] = 2
backtesting = Backtesting(default_conf)
results = {}
results['ETH/BTC'] = pd.DataFrame(
{
'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
'profit_percent': [0.1, 0.2, 0.3],
'profit_abs': [0.2, 0.4, 0.5],
'trade_duration': [10, 30, 10],
'profit': [2, 0, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
results['LTC/BTC'] = pd.DataFrame(
{
'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'],
'profit_percent': [0.4, 0.2, 0.3],
'profit_abs': [0.4, 0.4, 0.5],
'trade_duration': [15, 30, 15],
'profit': [4, 1, 0],
'loss': [0, 0, 1],
'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
}
)
result_str = (
'| Strategy | buy count | avg profit % | cum profit % '
'| tot profit BTC | tot profit % | avg duration | profit | loss |\n'
'|:-----------|------------:|---------------:|---------------:'
'|-----------------:|---------------:|:---------------|---------:|-------:|\n'
'| ETH/BTC | 3 | 20.00 | 60.00 '
'| 1.10000000 | 30.00 | 0:17:00 | 3 | 0 |\n'
'| LTC/BTC | 3 | 30.00 | 90.00 '
'| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |'
)
assert backtesting._generate_text_table_strategy(all_results=results) == result_str
def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
def get_timerange(input1):
return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
@ -465,11 +366,8 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
patch_exchange(mocker)
mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting',
backtest=MagicMock(),
_generate_text_table=MagicMock(return_value='1'),
)
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
default_conf['ticker_interval'] = '1m'
@ -498,11 +396,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
patch_exchange(mocker)
mocker.patch.multiple(
'freqtrade.optimize.backtesting.Backtesting',
backtest=MagicMock(),
_generate_text_table=MagicMock(return_value='1'),
)
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
default_conf['ticker_interval'] = "1m"
@ -813,7 +708,8 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
patch_exchange(mocker, api_mock)
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock())
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock())
patched_configuration_load_config_file(mocker, default_conf)
args = [
@ -859,10 +755,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
backtestmock = MagicMock()
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
gen_table_mock = MagicMock()
mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock)
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', gen_table_mock)
gen_strattable_mock = MagicMock()
mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy',
gen_strattable_mock)
mocker.patch('freqtrade.optimize.backtesting.generate_text_table_strategy', gen_strattable_mock)
patched_configuration_load_config_file(mocker, default_conf)
args = [