Merge pull request #1635 from freqtrade/feat/btanlaysis

BTAnalysis - simplify backtest result analysis

docs/backtesting.md

@@ -245,6 +245,26 @@ On the other hand, if you set a too high `minimal_roi` like `"0":  0.55`
 profit. Hence, keep in mind that your performance is a mix of your
 strategies, your configuration, and the crypto-currency you have set up.
 
+### Further backtest-result analysis
+
+To further analyze your backtest results, you can [export the trades](#exporting-trades-to-file).
+You can then load the trades to perform further analysis.
+
+A good way for this is using Jupyter (notebook or lab) - which provides an interactive environment to analyze the data.
+
+Freqtrade provides an easy to load the backtest results, which is `load_backtest_data` - and takes a path to the backtest-results file.
+
+``` python
+from freqtrade.data.btanalysis import load_backtest_data
+df = load_backtest_data("user_data/backtest-result.json")
+
+# Show value-counts per pair
+df.groupby("pair")["sell_reason"].value_counts()
+
+```
+
+This will allow you to drill deeper into your backtest results, and perform analysis which would make the regular backtest-output unreadable. 
+
 ## Backtesting multiple strategies
 
 To backtest multiple strategies, a list of Strategies can be provided.

freqtrade/data/btanalysis.py

@@ -0,0 +1,67 @@
+"""
+Helpers when analyzing backtest data
+"""
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from freqtrade.misc import json_load
+
+# must align with columns in backtest.py
+BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "duration",
+                   "open_rate", "close_rate", "open_at_end", "sell_reason"]
+
+
+def load_backtest_data(filename) -> pd.DataFrame:
+    """
+    Load backtest data file.
+    :param filename: pathlib.Path object, or string pointing to the file.
+    :return a dataframe with the analysis results
+    """
+    if isinstance(filename, str):
+        filename = Path(filename)
+
+    if not filename.is_file():
+        raise ValueError("File {filename} does not exist.")
+
+    with filename.open() as file:
+        data = json_load(file)
+
+    df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
+
+    df['open_time'] = pd.to_datetime(df['open_time'],
+                                     unit='s',
+                                     utc=True,
+                                     infer_datetime_format=True
+                                     )
+    df['close_time'] = pd.to_datetime(df['close_time'],
+                                      unit='s',
+                                      utc=True,
+                                      infer_datetime_format=True
+                                      )
+    df['profitabs'] = df['close_rate'] - df['open_rate']
+    df = df.sort_values("open_time").reset_index(drop=True)
+    return df
+
+
+def evaluate_result_multi(results: pd.DataFrame, freq: str, max_open_trades: int) -> pd.DataFrame:
+    """
+    Find overlapping trades by expanding each trade once per period it was open
+    and then counting overlaps
+    :param results: Results Dataframe - can be loaded
+    :param freq: Frequency used for the backtest
+    :param max_open_trades: parameter max_open_trades used during backtest run
+    :return: dataframe with open-counts per time-period in freq
+    """
+    dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq))
+             for row in results[['open_time', 'close_time']].iterrows()]
+    deltas = [len(x) for x in dates]
+    dates = pd.Series(pd.concat(dates).values, name='date')
+    df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
+
+    df2 = df2.astype(dtype={"open_time": "datetime64", "close_time": "datetime64"})
+    df2 = pd.concat([dates, df2], axis=1)
+    df2 = df2.set_index('date')
+    df_final = df2.resample(freq)[['pair']].count()
+    return df_final[df_final['pair'] > max_open_trades]

freqtrade/tests/data/test_btanalysis.py

@@ -0,0 +1,21 @@
+import pytest
+from pandas import DataFrame
+
+from freqtrade.data.btanalysis import BT_DATA_COLUMNS, load_backtest_data
+from freqtrade.data.history import make_testdata_path
+
+
+def test_load_backtest_data():
+
+    filename = make_testdata_path(None) / "backtest-result_test.json"
+    bt_data = load_backtest_data(filename)
+    assert isinstance(bt_data, DataFrame)
+    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"]
+    assert len(bt_data) == 179
+
+    # Test loading from string (must yield same result)
+    bt_data2 = load_backtest_data(str(filename))
+    assert bt_data.equals(bt_data2)
+
+    with pytest.raises(ValueError, match=r"File .* does not exist\."):
+        load_backtest_data(str("filename") + "nofile")

freqtrade/tests/optimize/test_backtesting.py

@@ -14,6 +14,7 @@ from arrow import Arrow
 from freqtrade import DependencyException, constants
 from freqtrade.arguments import Arguments, TimeRange
 from freqtrade.data import history
+from freqtrade.data.btanalysis import evaluate_result_multi
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.optimize import get_timeframe
 from freqtrade.optimize.backtesting import (Backtesting, setup_configuration,
@@ -684,21 +685,6 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker):
 
 def test_backtest_multi_pair(default_conf, fee, mocker):
 
-    def evaluate_result_multi(results, freq, max_open_trades):
-        # Find overlapping trades by expanding each trade once per period
-        # and then counting overlaps
-        dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, freq=freq))
-                 for row in results[['open_time', 'close_time']].iterrows()]
-        deltas = [len(x) for x in dates]
-        dates = pd.Series(pd.concat(dates).values, name='date')
-        df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns)
-
-        df2 = df2.astype(dtype={"open_time": "datetime64", "close_time": "datetime64"})
-        df2 = pd.concat([dates, df2], axis=1)
-        df2 = df2.set_index('date')
-        df_final = df2.resample(freq)[['pair']].count()
-        return df_final[df_final['pair'] > max_open_trades]
-
     def _trend_alternate_hold(dataframe=None, metadata=None):
         """
         Buy every 8th candle - sell every other 8th -2 (hold on to pairs a bit)

scripts/plot_dataframe.py

@@ -41,6 +41,7 @@ from plotly.offline import plot
 from freqtrade import persistence
 from freqtrade.arguments import Arguments, TimeRange
 from freqtrade.data import history
+from freqtrade.data.btanalysis import load_backtest_data, BT_DATA_COLUMNS
 from freqtrade.exchange import Exchange
 from freqtrade.optimize.backtesting import setup_configuration
 from freqtrade.persistence import Trade
@@ -56,7 +57,8 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram
     trades: pd.DataFrame = pd.DataFrame()
     if args.db_url:
         persistence.init(_CONF)
-        columns = ["pair", "profit", "opents", "closets", "open_rate", "close_rate", "duration"]
+        columns = ["pair", "profit", "open_time", "close_time",
+                   "open_rate", "close_rate", "duration"]
 
         for x in Trade.query.all():
             print("date: {}".format(x.open_date))
@@ -71,33 +73,13 @@ def load_trades(args: Namespace, pair: str, timerange: TimeRange) -> pd.DataFram
                               columns=columns)
 
     elif args.exportfilename:
-        file = Path(args.exportfilename)
-        # must align with columns in backtest.py
-        columns = ["pair", "profit", "opents", "closets", "index", "duration",
-                   "open_rate", "close_rate", "open_at_end", "sell_reason"]
-        if file.exists():
-            with file.open() as f:
-                data = json.load(f)
-                trades = pd.DataFrame(data, columns=columns)
-            trades = trades.loc[trades["pair"] == pair]
-            if timerange:
-                if timerange.starttype == 'date':
-                    trades = trades.loc[trades["opents"] >= timerange.startts]
-                if timerange.stoptype == 'date':
-                    trades = trades.loc[trades["opents"] <= timerange.stopts]
 
-            trades['opents'] = pd.to_datetime(
+        file = Path(args.exportfilename)
-                                            trades['opents'],
+        if file.exists():
-                                            unit='s',
+            load_backtest_data(file)
-                                            utc=True,
+
-                                            infer_datetime_format=True)
-            trades['closets'] = pd.to_datetime(
-                                            trades['closets'],
-                                            unit='s',
-                                            utc=True,
-                                            infer_datetime_format=True)
         else:
-            trades = pd.DataFrame([], columns=columns)
+            trades = pd.DataFrame([], columns=BT_DATA_COLUMNS)
 
     return trades
 
@@ -206,7 +188,7 @@ def extract_trades_of_period(dataframe, trades) -> pd.DataFrame:
     Compare trades and backtested pair DataFrames to get trades performed on backtested period
     :return: the DataFrame of a trades of period
     """
-    trades = trades.loc[trades['opents'] >= dataframe.iloc[0]['date']]
+    trades = trades.loc[trades['open_time'] >= dataframe.iloc[0]['date']]
     return trades
 
 
@@ -279,7 +261,7 @@ def generate_graph(
     )
 
     trade_buys = go.Scattergl(
-        x=trades["opents"],
+        x=trades["open_time"],
         y=trades["open_rate"],
         mode='markers',
         name='trade_buy',
@@ -291,7 +273,7 @@ def generate_graph(
         )
     )
     trade_sells = go.Scattergl(
-        x=trades["closets"],
+        x=trades["close_time"],
         y=trades["close_rate"],
         mode='markers',
         name='trade_sell',