Merge pull request #3066 from freqtrade/backtest_small_improvements
Backtest small improvements
This commit is contained in:
hroff-1902
GitHub
commit
0847652df0
@ -6,8 +6,7 @@ This module contains the backtesting logic
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, NamedTuple, Optional
|
||||
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
|
||||
|
||||
import arrow
|
||||
from pandas import DataFrame
|
||||
@ -19,10 +18,8 @@ from freqtrade.data.converter import trim_dataframe
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
|
||||
from freqtrade.misc import file_dump_json
|
||||
from freqtrade.optimize.optimize_reports import (
|
||||
generate_text_table, generate_text_table_sell_reason,
|
||||
generate_text_table_strategy)
|
||||
from freqtrade.optimize.optimize_reports import (show_backtest_results,
|
||||
store_backtest_result)
|
||||
from freqtrade.persistence import Trade
|
||||
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
|
||||
from freqtrade.state import RunMode
|
||||
@ -108,7 +105,7 @@ class Backtesting:
|
||||
# And the regular "stoploss" function would not apply to that case
|
||||
self.strategy.order_types['stoploss_on_exchange'] = False
|
||||
|
||||
def load_bt_data(self):
|
||||
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
|
||||
timerange = TimeRange.parse_timerange(None if self.config.get(
|
||||
'timerange') is None else str(self.config.get('timerange')))
|
||||
|
||||
@ -134,23 +131,6 @@ class Backtesting:
|
||||
|
||||
return data, timerange
|
||||
|
||||
def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
|
||||
strategyname: Optional[str] = None) -> None:
|
||||
|
||||
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
|
||||
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
|
||||
t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
|
||||
for index, t in results.iterrows()]
|
||||
|
||||
if records:
|
||||
if strategyname:
|
||||
# Inject strategyname to filename
|
||||
recordfilename = Path.joinpath(
|
||||
recordfilename.parent,
|
||||
f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix)
|
||||
logger.info(f'Dumping backtest results to {recordfilename}')
|
||||
file_dump_json(recordfilename, records)
|
||||
|
||||
def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]:
|
||||
"""
|
||||
Helper function to convert a processed dataframes into lists for performance reasons.
|
||||
@ -418,44 +398,7 @@ class Backtesting:
|
||||
position_stacking=position_stacking,
|
||||
)
|
||||
|
||||
for strategy, results in all_results.items():
|
||||
|
||||
if self.config.get('export', False):
|
||||
self._store_backtest_result(self.config['exportfilename'], results,
|
||||
strategy if len(self.strategylist) > 1 else None)
|
||||
|
||||
print(f"Result for strategy {strategy}")
|
||||
table = generate_text_table(data, stake_currency=self.config['stake_currency'],
|
||||
max_open_trades=self.config['max_open_trades'],
|
||||
results=results)
|
||||
if isinstance(table, str):
|
||||
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = generate_text_table_sell_reason(data,
|
||||
stake_currency=self.config['stake_currency'],
|
||||
max_open_trades=self.config['max_open_trades'],
|
||||
results=results)
|
||||
if isinstance(table, str):
|
||||
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = generate_text_table(data,
|
||||
stake_currency=self.config['stake_currency'],
|
||||
max_open_trades=self.config['max_open_trades'],
|
||||
results=results.loc[results.open_at_end], skip_nan=True)
|
||||
if isinstance(table, str):
|
||||
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
if isinstance(table, str):
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
print()
|
||||
if len(all_results) > 1:
|
||||
# Print Strategy summary table
|
||||
table = generate_text_table_strategy(self.config['stake_currency'],
|
||||
self.config['max_open_trades'],
|
||||
all_results=all_results)
|
||||
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
print('\nFor more details, please look at the detail tables above')
|
||||
if self.config.get('export', False):
|
||||
store_backtest_result(self.config['exportfilename'], all_results)
|
||||
# Show backtest results
|
||||
show_backtest_results(self.config, data, all_results)
|
||||
|
||||
@ -1,9 +1,37 @@
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from pandas import DataFrame
|
||||
from tabulate import tabulate
|
||||
|
||||
from freqtrade.misc import file_dump_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None:
|
||||
"""
|
||||
Stores backtest results to file (one file per strategy)
|
||||
:param recordfilename: Destination filename
|
||||
:param all_results: Dict of Dataframes, one results dataframe per strategy
|
||||
"""
|
||||
for strategy, results in all_results.items():
|
||||
records = [(t.pair, t.profit_percent, t.open_time.timestamp(),
|
||||
t.close_time.timestamp(), t.open_index - 1, t.trade_duration,
|
||||
t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value)
|
||||
for index, t in results.iterrows()]
|
||||
|
||||
if records:
|
||||
if len(all_results) > 1:
|
||||
# Inject strategy to filename
|
||||
recordfilename = Path.joinpath(
|
||||
recordfilename.parent,
|
||||
f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix)
|
||||
logger.info(f'Dumping backtest results to {recordfilename}')
|
||||
file_dump_json(recordfilename, records)
|
||||
|
||||
|
||||
def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
|
||||
results: DataFrame, skip_nan: bool = False) -> str:
|
||||
@ -69,12 +97,12 @@ def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_tra
|
||||
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
|
||||
|
||||
|
||||
def generate_text_table_sell_reason(
|
||||
data: Dict[str, Dict], stake_currency: str, max_open_trades: int, results: DataFrame
|
||||
) -> str:
|
||||
def generate_text_table_sell_reason(stake_currency: str, max_open_trades: int,
|
||||
results: DataFrame) -> str:
|
||||
"""
|
||||
Generate small table outlining Backtest results
|
||||
:param data: Dict of <pair: dataframe> containing data that was used during backtesting.
|
||||
:param stake_currency: Stakecurrency used
|
||||
:param max_open_trades: Max_open_trades parameter
|
||||
:param results: Dataframe containing the backtest results
|
||||
:return: pretty printed table with tabulate as string
|
||||
"""
|
||||
@ -173,3 +201,43 @@ def generate_edge_table(results: dict) -> str:
|
||||
# Ignore type as floatfmt does allow tuples but mypy does not know that
|
||||
return tabulate(tabular_data, headers=headers,
|
||||
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
|
||||
|
||||
|
||||
def show_backtest_results(config: Dict, btdata: Dict[str, DataFrame],
|
||||
all_results: Dict[str, DataFrame]):
|
||||
for strategy, results in all_results.items():
|
||||
|
||||
print(f"Result for strategy {strategy}")
|
||||
table = generate_text_table(btdata, stake_currency=config['stake_currency'],
|
||||
max_open_trades=config['max_open_trades'],
|
||||
results=results)
|
||||
if isinstance(table, str):
|
||||
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = generate_text_table_sell_reason(stake_currency=config['stake_currency'],
|
||||
max_open_trades=config['max_open_trades'],
|
||||
results=results)
|
||||
if isinstance(table, str):
|
||||
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = generate_text_table(btdata,
|
||||
stake_currency=config['stake_currency'],
|
||||
max_open_trades=config['max_open_trades'],
|
||||
results=results.loc[results.open_at_end], skip_nan=True)
|
||||
if isinstance(table, str):
|
||||
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
if isinstance(table, str):
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
print()
|
||||
if len(all_results) > 1:
|
||||
# Print Strategy summary table
|
||||
table = generate_text_table_strategy(config['stake_currency'],
|
||||
config['max_open_trades'],
|
||||
all_results=all_results)
|
||||
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
print('\nFor more details, please look at the detail tables above')
|
||||
|
||||
@ -331,8 +331,8 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
|
||||
|
||||
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
|
||||
patch_exchange(mocker)
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
|
||||
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
|
||||
mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
|
||||
|
||||
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
|
||||
default_conf['ticker_interval'] = '1m'
|
||||
@ -361,8 +361,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
|
||||
MagicMock(return_value=pd.DataFrame()))
|
||||
mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
|
||||
patch_exchange(mocker)
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
|
||||
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest')
|
||||
mocker.patch('freqtrade.optimize.backtesting.show_backtest_results')
|
||||
|
||||
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
|
||||
default_conf['ticker_interval'] = "1m"
|
||||
@ -507,7 +507,6 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir):
|
||||
|
||||
def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
|
||||
mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock())
|
||||
backtest_conf = _make_backtest_conf(mocker, conf=default_conf,
|
||||
pair='UNITTEST/BTC', datadir=testdatadir)
|
||||
default_conf['ticker_interval'] = '1m'
|
||||
@ -515,7 +514,6 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir):
|
||||
backtesting.strategy.advise_buy = _trend_alternate # Override
|
||||
backtesting.strategy.advise_sell = _trend_alternate # Override
|
||||
results = backtesting.backtest(**backtest_conf)
|
||||
backtesting._store_backtest_result("test_.json", results)
|
||||
# 200 candles in backtest data
|
||||
# won't buy on first (shifted by 1)
|
||||
# 100 buys signals
|
||||
@ -586,84 +584,12 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir)
|
||||
assert len(evaluate_result_multi(results, '5m', 1)) == 0
|
||||
|
||||
|
||||
def test_backtest_record(default_conf, fee, mocker):
|
||||
names = []
|
||||
records = []
|
||||
patch_exchange(mocker)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
|
||||
mocker.patch(
|
||||
'freqtrade.optimize.backtesting.file_dump_json',
|
||||
new=lambda n, r: (names.append(n), records.append(r))
|
||||
)
|
||||
|
||||
backtesting = Backtesting(default_conf)
|
||||
results = pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
|
||||
"UNITTEST/BTC", "UNITTEST/BTC"],
|
||||
"profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
|
||||
"profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
|
||||
"open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
|
||||
Arrow(2017, 11, 14, 21, 36, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 12, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 44, 00).datetime],
|
||||
"close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 10, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 43, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 58, 00).datetime],
|
||||
"open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
|
||||
"close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
|
||||
"open_index": [1, 119, 153, 185],
|
||||
"close_index": [118, 151, 184, 199],
|
||||
"trade_duration": [123, 34, 31, 14],
|
||||
"open_at_end": [False, False, False, True],
|
||||
"sell_reason": [SellType.ROI, SellType.STOP_LOSS,
|
||||
SellType.ROI, SellType.FORCE_SELL]
|
||||
})
|
||||
backtesting._store_backtest_result("backtest-result.json", results)
|
||||
assert len(results) == 4
|
||||
# Assert file_dump_json was only called once
|
||||
assert names == ['backtest-result.json']
|
||||
records = records[0]
|
||||
# Ensure records are of correct type
|
||||
assert len(records) == 4
|
||||
|
||||
# reset test to test with strategy name
|
||||
names = []
|
||||
records = []
|
||||
backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat")
|
||||
assert len(results) == 4
|
||||
# Assert file_dump_json was only called once
|
||||
assert names == [Path('backtest-result-DefStrat.json')]
|
||||
records = records[0]
|
||||
# Ensure records are of correct type
|
||||
assert len(records) == 4
|
||||
|
||||
# ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117)
|
||||
# Below follows just a typecheck of the schema/type of trade-records
|
||||
oix = None
|
||||
for (pair, profit, date_buy, date_sell, buy_index, dur,
|
||||
openr, closer, open_at_end, sell_reason) in records:
|
||||
assert pair == 'UNITTEST/BTC'
|
||||
assert isinstance(profit, float)
|
||||
# FIX: buy/sell should be converted to ints
|
||||
assert isinstance(date_buy, float)
|
||||
assert isinstance(date_sell, float)
|
||||
assert isinstance(openr, float)
|
||||
assert isinstance(closer, float)
|
||||
assert isinstance(open_at_end, bool)
|
||||
assert isinstance(sell_reason, str)
|
||||
isinstance(buy_index, pd._libs.tslib.Timestamp)
|
||||
if oix:
|
||||
assert buy_index > oix
|
||||
oix = buy_index
|
||||
assert dur > 0
|
||||
|
||||
|
||||
def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
|
||||
default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
|
||||
|
||||
patch_exchange(mocker)
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
|
||||
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock())
|
||||
mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock())
|
||||
|
||||
patched_configuration_load_config_file(mocker, default_conf)
|
||||
|
||||
@ -705,9 +631,10 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
|
||||
backtestmock = MagicMock()
|
||||
mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
|
||||
gen_table_mock = MagicMock()
|
||||
mocker.patch('freqtrade.optimize.backtesting.generate_text_table', gen_table_mock)
|
||||
mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table', gen_table_mock)
|
||||
gen_strattable_mock = MagicMock()
|
||||
mocker.patch('freqtrade.optimize.backtesting.generate_text_table_strategy', gen_strattable_mock)
|
||||
mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table_strategy',
|
||||
gen_strattable_mock)
|
||||
patched_configuration_load_config_file(mocker, default_conf)
|
||||
|
||||
args = [
|
||||
|
||||
@ -1,10 +1,14 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from arrow import Arrow
|
||||
|
||||
from freqtrade.edge import PairInfo
|
||||
from freqtrade.optimize.optimize_reports import (
|
||||
generate_edge_table, generate_text_table, generate_text_table_sell_reason,
|
||||
generate_text_table_strategy)
|
||||
generate_text_table_strategy, store_backtest_result)
|
||||
from freqtrade.strategy.interface import SellType
|
||||
from tests.conftest import patch_exchange
|
||||
|
||||
|
||||
def test_generate_text_table(default_conf, mocker):
|
||||
@ -61,10 +65,8 @@ def test_generate_text_table_sell_reason(default_conf, mocker):
|
||||
'| stop_loss | 1 | 0 | 0 | 1 |'
|
||||
' -10 | -10 | -0.2 | -5 |'
|
||||
)
|
||||
assert generate_text_table_sell_reason(
|
||||
data={'ETH/BTC': {}},
|
||||
stake_currency='BTC', max_open_trades=2,
|
||||
results=results) == result_str
|
||||
assert generate_text_table_sell_reason(stake_currency='BTC', max_open_trades=2,
|
||||
results=results) == result_str
|
||||
|
||||
|
||||
def test_generate_text_table_strategy(default_conf, mocker):
|
||||
@ -115,3 +117,73 @@ def test_generate_edge_table(edge_conf, mocker):
|
||||
assert generate_edge_table(results).count('| ETH/BTC |') == 1
|
||||
assert generate_edge_table(results).count(
|
||||
'| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1
|
||||
|
||||
|
||||
def test_backtest_record(default_conf, fee, mocker):
|
||||
names = []
|
||||
records = []
|
||||
patch_exchange(mocker)
|
||||
mocker.patch('freqtrade.exchange.Exchange.get_fee', fee)
|
||||
mocker.patch(
|
||||
'freqtrade.optimize.optimize_reports.file_dump_json',
|
||||
new=lambda n, r: (names.append(n), records.append(r))
|
||||
)
|
||||
|
||||
results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC",
|
||||
"UNITTEST/BTC", "UNITTEST/BTC"],
|
||||
"profit_percent": [0.003312, 0.010801, 0.013803, 0.002780],
|
||||
"profit_abs": [0.000003, 0.000011, 0.000014, 0.000003],
|
||||
"open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime,
|
||||
Arrow(2017, 11, 14, 21, 36, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 12, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 44, 00).datetime],
|
||||
"close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 10, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 43, 00).datetime,
|
||||
Arrow(2017, 11, 14, 22, 58, 00).datetime],
|
||||
"open_rate": [0.002543, 0.003003, 0.003089, 0.003214],
|
||||
"close_rate": [0.002546, 0.003014, 0.003103, 0.003217],
|
||||
"open_index": [1, 119, 153, 185],
|
||||
"close_index": [118, 151, 184, 199],
|
||||
"trade_duration": [123, 34, 31, 14],
|
||||
"open_at_end": [False, False, False, True],
|
||||
"sell_reason": [SellType.ROI, SellType.STOP_LOSS,
|
||||
SellType.ROI, SellType.FORCE_SELL]
|
||||
})}
|
||||
store_backtest_result(Path("backtest-result.json"), results)
|
||||
# Assert file_dump_json was only called once
|
||||
assert names == [Path('backtest-result.json')]
|
||||
records = records[0]
|
||||
# Ensure records are of correct type
|
||||
assert len(records) == 4
|
||||
|
||||
# reset test to test with strategy name
|
||||
names = []
|
||||
records = []
|
||||
results['Strat'] = pd.DataFrame()
|
||||
store_backtest_result(Path("backtest-result.json"), results)
|
||||
# Assert file_dump_json was only called once
|
||||
assert names == [Path('backtest-result-DefStrat.json')]
|
||||
records = records[0]
|
||||
# Ensure records are of correct type
|
||||
assert len(records) == 4
|
||||
|
||||
# ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117)
|
||||
# Below follows just a typecheck of the schema/type of trade-records
|
||||
oix = None
|
||||
for (pair, profit, date_buy, date_sell, buy_index, dur,
|
||||
openr, closer, open_at_end, sell_reason) in records:
|
||||
assert pair == 'UNITTEST/BTC'
|
||||
assert isinstance(profit, float)
|
||||
# FIX: buy/sell should be converted to ints
|
||||
assert isinstance(date_buy, float)
|
||||
assert isinstance(date_sell, float)
|
||||
assert isinstance(openr, float)
|
||||
assert isinstance(closer, float)
|
||||
assert isinstance(open_at_end, bool)
|
||||
assert isinstance(sell_reason, str)
|
||||
isinstance(buy_index, pd._libs.tslib.Timestamp)
|
||||
if oix:
|
||||
assert buy_index > oix
|
||||
oix = buy_index
|
||||
assert dur > 0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user