Merge pull request #2730 from freqtrade/extract_bt_reporting

Extract backtest reporting

freqtrade/optimize/backtest_reports.py

@@ -0,0 +1,107 @@
+from datetime import timedelta
+from typing import Dict
+
+from pandas import DataFrame
+from tabulate import tabulate
+
+
+def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int,
+                        results: DataFrame, skip_nan: bool = False) -> str:
+    """
+    Generates and returns a text table for the given backtest data and the results dataframe
+    :param data: Dict of <pair: dataframe> containing data that was used during backtesting.
+    :param stake_currency: stake-currency - used to correctly name headers
+    :param max_open_trades: Maximum allowed open trades
+    :param results: Dataframe containing the backtest results
+    :param skip_nan: Print "left open" open trades
+    :return: pretty printed table with tabulate as string
+    """
+
+    floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
+    tabular_data = []
+    headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
+               f'tot profit {stake_currency}', 'tot profit %', 'avg duration',
+               'profit', 'loss']
+    for pair in data:
+        result = results[results.pair == pair]
+        if skip_nan and result.profit_abs.isnull().all():
+            continue
+
+        tabular_data.append([
+            pair,
+            len(result.index),
+            result.profit_percent.mean() * 100.0,
+            result.profit_percent.sum() * 100.0,
+            result.profit_abs.sum(),
+            result.profit_percent.sum() * 100.0 / max_open_trades,
+            str(timedelta(
+                minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
+            len(result[result.profit_abs > 0]),
+            len(result[result.profit_abs < 0])
+        ])
+
+    # Append Total
+    tabular_data.append([
+        'TOTAL',
+        len(results.index),
+        results.profit_percent.mean() * 100.0,
+        results.profit_percent.sum() * 100.0,
+        results.profit_abs.sum(),
+        results.profit_percent.sum() * 100.0 / max_open_trades,
+        str(timedelta(
+            minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
+        len(results[results.profit_abs > 0]),
+        len(results[results.profit_abs < 0])
+    ])
+    # Ignore type as floatfmt does allow tuples but mypy does not know that
+    return tabulate(tabular_data, headers=headers,
+                    floatfmt=floatfmt, tablefmt="pipe")  # type: ignore
+
+
+def generate_text_table_sell_reason(data: Dict[str, Dict], results: DataFrame) -> str:
+    """
+    Generate small table outlining Backtest results
+    :param data: Dict of <pair: dataframe> containing data that was used during backtesting.
+    :param results: Dataframe containing the backtest results
+    :return: pretty printed table with tabulate as string
+    """
+    tabular_data = []
+    headers = ['Sell Reason', 'Count', 'Profit', 'Loss']
+    for reason, count in results['sell_reason'].value_counts().iteritems():
+        profit = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] >= 0)])
+        loss = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] < 0)])
+        tabular_data.append([reason.value, count, profit, loss])
+    return tabulate(tabular_data, headers=headers, tablefmt="pipe")
+
+
+def generate_text_table_strategy(stake_currency: str, max_open_trades: str,
+                                 all_results: Dict) -> str:
+    """
+    Generate summary table per strategy
+    :param stake_currency: stake-currency - used to correctly name headers
+    :param max_open_trades: Maximum allowed open trades used for backtest
+    :param all_results: Dict of <Strategyname: BacktestResult> containing results for all strategies
+    :return: pretty printed table with tabulate as string
+    """
+
+    floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
+    tabular_data = []
+    headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %',
+               f'tot profit {stake_currency}', 'tot profit %', 'avg duration',
+               'profit', 'loss']
+    for strategy, results in all_results.items():
+        tabular_data.append([
+            strategy,
+            len(results.index),
+            results.profit_percent.mean() * 100.0,
+            results.profit_percent.sum() * 100.0,
+            results.profit_abs.sum(),
+            results.profit_percent.sum() * 100.0 / max_open_trades,
+            str(timedelta(
+                minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
+            len(results[results.profit_abs > 0]),
+            len(results[results.profit_abs < 0])
+        ])
+    # Ignore type as floatfmt does allow tuples but mypy does not know that
+    return tabulate(tabular_data, headers=headers,
+                    floatfmt=floatfmt, tablefmt="pipe")  # type: ignore

freqtrade/optimize/backtesting.py

@@ -10,7 +10,6 @@ from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, Optional
 
 from pandas import DataFrame
-from tabulate import tabulate
 
 from freqtrade.configuration import (TimeRange, remove_credentials,
                                      validate_config_consistency)
@@ -19,6 +18,9 @@ from freqtrade.data.dataprovider import DataProvider
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.misc import file_dump_json
+from freqtrade.optimize.backtest_reports import (
+    generate_text_table, generate_text_table_sell_reason,
+    generate_text_table_strategy)
 from freqtrade.persistence import Trade
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -129,96 +131,6 @@ class Backtesting:
 
         return data, timerange
 
-    def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame,
-                             skip_nan: bool = False) -> str:
-        """
-        Generates and returns a text table for the given backtest data and the results dataframe
-        :return: pretty printed table with tabulate as str
-        """
-        stake_currency = str(self.config.get('stake_currency'))
-        max_open_trades = self.config.get('max_open_trades')
-
-        floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
-        tabular_data = []
-        headers = ['pair', 'buy count', 'avg profit %', 'cum profit %',
-                   'tot profit ' + stake_currency, 'tot profit %', 'avg duration',
-                   'profit', 'loss']
-        for pair in data:
-            result = results[results.pair == pair]
-            if skip_nan and result.profit_abs.isnull().all():
-                continue
-
-            tabular_data.append([
-                pair,
-                len(result.index),
-                result.profit_percent.mean() * 100.0,
-                result.profit_percent.sum() * 100.0,
-                result.profit_abs.sum(),
-                result.profit_percent.sum() * 100.0 / max_open_trades,
-                str(timedelta(
-                    minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00',
-                len(result[result.profit_abs > 0]),
-                len(result[result.profit_abs < 0])
-            ])
-
-        # Append Total
-        tabular_data.append([
-            'TOTAL',
-            len(results.index),
-            results.profit_percent.mean() * 100.0,
-            results.profit_percent.sum() * 100.0,
-            results.profit_abs.sum(),
-            results.profit_percent.sum() * 100.0 / max_open_trades,
-            str(timedelta(
-                minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
-            len(results[results.profit_abs > 0]),
-            len(results[results.profit_abs < 0])
-        ])
-        # Ignore type as floatfmt does allow tuples but mypy does not know that
-        return tabulate(tabular_data, headers=headers,
-                        floatfmt=floatfmt, tablefmt="pipe")  # type: ignore
-
-    def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str:
-        """
-        Generate small table outlining Backtest results
-        """
-        tabular_data = []
-        headers = ['Sell Reason', 'Count', 'Profit', 'Loss']
-        for reason, count in results['sell_reason'].value_counts().iteritems():
-            profit = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] >= 0)])
-            loss = len(results[(results['sell_reason'] == reason) & (results['profit_abs'] < 0)])
-            tabular_data.append([reason.value, count, profit, loss])
-        return tabulate(tabular_data, headers=headers, tablefmt="pipe")
-
-    def _generate_text_table_strategy(self, all_results: dict) -> str:
-        """
-        Generate summary table per strategy
-        """
-        stake_currency = str(self.config.get('stake_currency'))
-        max_open_trades = self.config.get('max_open_trades')
-
-        floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f')
-        tabular_data = []
-        headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %',
-                   'tot profit ' + stake_currency, 'tot profit %', 'avg duration',
-                   'profit', 'loss']
-        for strategy, results in all_results.items():
-            tabular_data.append([
-                strategy,
-                len(results.index),
-                results.profit_percent.mean() * 100.0,
-                results.profit_percent.sum() * 100.0,
-                results.profit_abs.sum(),
-                results.profit_percent.sum() * 100.0 / max_open_trades,
-                str(timedelta(
-                    minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00',
-                len(results[results.profit_abs > 0]),
-                len(results[results.profit_abs < 0])
-            ])
-        # Ignore type as floatfmt does allow tuples but mypy does not know that
-        return tabulate(tabular_data, headers=headers,
-                        floatfmt=floatfmt, tablefmt="pipe")  # type: ignore
-
     def _store_backtest_result(self, recordfilename: Path, results: DataFrame,
                                strategyname: Optional[str] = None) -> None:
 
@@ -509,16 +421,24 @@ class Backtesting:
 
             print(f"Result for strategy {strategy}")
             print(' BACKTESTING REPORT '.center(133, '='))
-            print(self._generate_text_table(data, results))
+            print(generate_text_table(data,
+                                      stake_currency=self.config['stake_currency'],
+                                      max_open_trades=self.config['max_open_trades'],
+                                      results=results))
 
             print(' SELL REASON STATS '.center(133, '='))
-            print(self._generate_text_table_sell_reason(data, results))
+            print(generate_text_table_sell_reason(data, results))
 
             print(' LEFT OPEN TRADES REPORT '.center(133, '='))
-            print(self._generate_text_table(data, results.loc[results.open_at_end], True))
+            print(generate_text_table(data,
+                                      stake_currency=self.config['stake_currency'],
+                                      max_open_trades=self.config['max_open_trades'],
+                                      results=results.loc[results.open_at_end], skip_nan=True))
             print()
         if len(all_results) > 1:
             # Print Strategy summary table
             print(' Strategy Summary '.center(133, '='))
-            print(self._generate_text_table_strategy(all_results))
+            print(generate_text_table_strategy(self.config['stake_currency'],
+                                               self.config['max_open_trades'],
+                                               all_results=all_results))
             print('\nFor more details, please look at the detail tables above')

tests/optimize/test_backtest_reports.py

@@ -0,0 +1,96 @@
+import pandas as pd
+
+from freqtrade.optimize.backtest_reports import (
+    generate_text_table, generate_text_table_sell_reason,
+    generate_text_table_strategy)
+from freqtrade.strategy.interface import SellType
+
+
+def test_generate_text_table(default_conf, mocker):
+
+    results = pd.DataFrame(
+        {
+            'pair': ['ETH/BTC', 'ETH/BTC'],
+            'profit_percent': [0.1, 0.2],
+            'profit_abs': [0.2, 0.4],
+            'trade_duration': [10, 30],
+            'profit': [2, 0],
+            'loss': [0, 0]
+        }
+    )
+
+    result_str = (
+        '| pair    |   buy count |   avg profit % |   cum profit % |   '
+        'tot profit BTC |   tot profit % | avg duration   |   profit |   loss |\n'
+        '|:--------|------------:|---------------:|---------------:|'
+        '-----------------:|---------------:|:---------------|---------:|-------:|\n'
+        '| ETH/BTC |           2 |          15.00 |          30.00 |       '
+        '0.60000000 |          15.00 | 0:20:00        |        2 |      0 |\n'
+        '| TOTAL   |           2 |          15.00 |          30.00 |       '
+        '0.60000000 |          15.00 | 0:20:00        |        2 |      0 |'
+    )
+    assert generate_text_table(data={'ETH/BTC': {}},
+                               stake_currency='BTC', max_open_trades=2,
+                               results=results) == result_str
+
+
+def test_generate_text_table_sell_reason(default_conf, mocker):
+
+    results = pd.DataFrame(
+        {
+            'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
+            'profit_percent': [0.1, 0.2, -0.3],
+            'profit_abs': [0.2, 0.4, -0.5],
+            'trade_duration': [10, 30, 10],
+            'profit': [2, 0, 0],
+            'loss': [0, 0, 1],
+            'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
+        }
+    )
+
+    result_str = (
+        '| Sell Reason   |   Count |   Profit |   Loss |\n'
+        '|:--------------|--------:|---------:|-------:|\n'
+        '| roi           |       2 |        2 |      0 |\n'
+        '| stop_loss     |       1 |        0 |      1 |'
+    )
+    assert generate_text_table_sell_reason(
+        data={'ETH/BTC': {}}, results=results) == result_str
+
+
+def test_generate_text_table_strategy(default_conf, mocker):
+    results = {}
+    results['ETH/BTC'] = pd.DataFrame(
+        {
+            'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
+            'profit_percent': [0.1, 0.2, 0.3],
+            'profit_abs': [0.2, 0.4, 0.5],
+            'trade_duration': [10, 30, 10],
+            'profit': [2, 0, 0],
+            'loss': [0, 0, 1],
+            'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
+        }
+    )
+    results['LTC/BTC'] = pd.DataFrame(
+        {
+            'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'],
+            'profit_percent': [0.4, 0.2, 0.3],
+            'profit_abs': [0.4, 0.4, 0.5],
+            'trade_duration': [15, 30, 15],
+            'profit': [4, 1, 0],
+            'loss': [0, 0, 1],
+            'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
+        }
+    )
+
+    result_str = (
+        '| Strategy   |   buy count |   avg profit % |   cum profit % '
+        '|   tot profit BTC |   tot profit % | avg duration   |   profit |   loss |\n'
+        '|:-----------|------------:|---------------:|---------------:'
+        '|-----------------:|---------------:|:---------------|---------:|-------:|\n'
+        '| ETH/BTC    |           3 |          20.00 |          60.00 '
+        '|       1.10000000 |          30.00 | 0:17:00        |        3 |      0 |\n'
+        '| LTC/BTC    |           3 |          30.00 |          90.00 '
+        '|       1.30000000 |          45.00 | 0:20:00        |        3 |      0 |'
+    )
+    assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str

tests/optimize/test_backtesting.py

@@ -358,105 +358,6 @@ def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None:
     assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC'])
 
 
-def test_generate_text_table(default_conf, mocker):
-    patch_exchange(mocker)
-    default_conf['max_open_trades'] = 2
-    backtesting = Backtesting(default_conf)
-
-    results = pd.DataFrame(
-        {
-            'pair': ['ETH/BTC', 'ETH/BTC'],
-            'profit_percent': [0.1, 0.2],
-            'profit_abs': [0.2, 0.4],
-            'trade_duration': [10, 30],
-            'profit': [2, 0],
-            'loss': [0, 0]
-        }
-    )
-
-    result_str = (
-        '| pair    |   buy count |   avg profit % |   cum profit % |   '
-        'tot profit BTC |   tot profit % | avg duration   |   profit |   loss |\n'
-        '|:--------|------------:|---------------:|---------------:|'
-        '-----------------:|---------------:|:---------------|---------:|-------:|\n'
-        '| ETH/BTC |           2 |          15.00 |          30.00 |       '
-        '0.60000000 |          15.00 | 0:20:00        |        2 |      0 |\n'
-        '| TOTAL   |           2 |          15.00 |          30.00 |       '
-        '0.60000000 |          15.00 | 0:20:00        |        2 |      0 |'
-    )
-    assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str
-
-
-def test_generate_text_table_sell_reason(default_conf, mocker):
-    patch_exchange(mocker)
-    backtesting = Backtesting(default_conf)
-
-    results = pd.DataFrame(
-        {
-            'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
-            'profit_percent': [0.1, 0.2, -0.3],
-            'profit_abs': [0.2, 0.4, -0.5],
-            'trade_duration': [10, 30, 10],
-            'profit': [2, 0, 0],
-            'loss': [0, 0, 1],
-            'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
-        }
-    )
-
-    result_str = (
-        '| Sell Reason   |   Count |   Profit |   Loss |\n'
-        '|:--------------|--------:|---------:|-------:|\n'
-        '| roi           |       2 |        2 |      0 |\n'
-        '| stop_loss     |       1 |        0 |      1 |'
-    )
-    assert backtesting._generate_text_table_sell_reason(
-        data={'ETH/BTC': {}}, results=results) == result_str
-
-
-def test_generate_text_table_strategyn(default_conf, mocker):
-    """
-    Test Backtesting.generate_text_table_sell_reason() method
-    """
-    patch_exchange(mocker)
-    default_conf['max_open_trades'] = 2
-    backtesting = Backtesting(default_conf)
-    results = {}
-    results['ETH/BTC'] = pd.DataFrame(
-        {
-            'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'],
-            'profit_percent': [0.1, 0.2, 0.3],
-            'profit_abs': [0.2, 0.4, 0.5],
-            'trade_duration': [10, 30, 10],
-            'profit': [2, 0, 0],
-            'loss': [0, 0, 1],
-            'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
-        }
-    )
-    results['LTC/BTC'] = pd.DataFrame(
-        {
-            'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'],
-            'profit_percent': [0.4, 0.2, 0.3],
-            'profit_abs': [0.4, 0.4, 0.5],
-            'trade_duration': [15, 30, 15],
-            'profit': [4, 1, 0],
-            'loss': [0, 0, 1],
-            'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS]
-        }
-    )
-
-    result_str = (
-        '| Strategy   |   buy count |   avg profit % |   cum profit % '
-        '|   tot profit BTC |   tot profit % | avg duration   |   profit |   loss |\n'
-        '|:-----------|------------:|---------------:|---------------:'
-        '|-----------------:|---------------:|:---------------|---------:|-------:|\n'
-        '| ETH/BTC    |           3 |          20.00 |          60.00 '
-        '|       1.10000000 |          30.00 | 0:17:00        |        3 |      0 |\n'
-        '| LTC/BTC    |           3 |          30.00 |          90.00 '
-        '|       1.30000000 |          45.00 | 0:20:00        |        3 |      0 |'
-    )
-    assert backtesting._generate_text_table_strategy(all_results=results) == result_str
-
-
 def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     def get_timerange(input1):
         return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59)
@@ -465,11 +366,8 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
-    mocker.patch.multiple(
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
-        'freqtrade.optimize.backtesting.Backtesting',
+    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
-        backtest=MagicMock(),
-        _generate_text_table=MagicMock(return_value='1'),
-    )
 
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
     default_conf['ticker_interval'] = '1m'
@@ -498,11 +396,8 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) ->
     mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
     mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock())
     patch_exchange(mocker)
-    mocker.patch.multiple(
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
-        'freqtrade.optimize.backtesting.Backtesting',
+    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock(return_value=1))
-        backtest=MagicMock(),
-        _generate_text_table=MagicMock(return_value='1'),
-    )
 
     default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC']
     default_conf['ticker_interval'] = "1m"
@@ -813,7 +708,8 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
 
     patch_exchange(mocker, api_mock)
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock())
-    mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock())
+    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', MagicMock())
+
     patched_configuration_load_config_file(mocker, default_conf)
 
     args = [
@@ -859,10 +755,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
     backtestmock = MagicMock()
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock)
     gen_table_mock = MagicMock()
-    mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock)
+    mocker.patch('freqtrade.optimize.backtesting.generate_text_table', gen_table_mock)
     gen_strattable_mock = MagicMock()
-    mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy',
+    mocker.patch('freqtrade.optimize.backtesting.generate_text_table_strategy', gen_strattable_mock)
-                 gen_strattable_mock)
     patched_configuration_load_config_file(mocker, default_conf)
 
     args = [