Streamline trade to dataframe conversion

freqtrade/data/btanalysis.py

@@ -2,9 +2,8 @@
 Helpers when analyzing backtest data
 """
 import logging
-from datetime import timezone
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -16,9 +15,21 @@ from freqtrade.persistence import Trade, init_db
 
 logger = logging.getLogger(__name__)
 
-# must align with columns in backtest.py
+# Old format - maybe remove?
-BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration",
+BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
-                   "open_rate", "close_rate", "open_at_end", "sell_reason"]
+                       "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
+
+# Mid-term format, crated by BacktestResult Named Tuple
+BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
+                       'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
+                       'fee_close', 'amount', 'profit_abs', 'profit_ratio']
+
+# Newest format
+BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
+                   'fee_open', 'fee_close', 'trade_duration',
+                   'profit_ratio', 'profit_abs', 'sell_reason',
+                   'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
+                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
 
 
 def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
@@ -154,7 +165,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                           )
     else:
         # old format - only with lists.
-        df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
+        df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
 
         df['open_date'] = pd.to_datetime(df['open_date'],
                                          unit='s',
@@ -166,7 +177,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                           utc=True,
                                           infer_datetime_format=True
                                           )
+        # Create compatibility with new format
         df['profit_abs'] = df['close_rate'] - df['open_rate']
+    if 'profit_ratio' not in df.columns:
+        df['profit_ratio'] = df['profit_percent']
     df = df.sort_values("open_date").reset_index(drop=True)
     return df
 
@@ -209,6 +223,19 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     return df_final[df_final['open_trades'] > max_open_trades]
 
 
+def trade_list_to_dataframe(trades: List[Trade]) -> pd.DataFrame:
+    """
+    Convert list of Trade objects to pandas Dataframe
+    :param trades: List of trade objects
+    :return: Dataframe with BT_DATA_COLUMNS
+    """
+    df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS)
+    if len(df) > 0:
+        df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True)
+        df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True)
+    return df
+
+
 def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame:
     """
     Load trades from a DB (using dburl)
@@ -219,36 +246,10 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
     """
     init_db(db_url, clean_open_orders=False)
 
-    columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
-               "open_rate", "close_rate", "amount", "trade_duration", "sell_reason",
-               "fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
-               "stake_amount", "max_rate", "min_rate", "id", "exchange",
-               "stop_loss", "initial_stop_loss", "strategy", "timeframe"]
-
     filters = []
     if strategy:
         filters.append(Trade.strategy == strategy)
-
+    trades = trade_list_to_dataframe(Trade.get_trades(filters).all())
-    trades = pd.DataFrame([(t.pair,
-                            t.open_date.replace(tzinfo=timezone.utc),
-                            t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
-                            t.calc_profit(), t.calc_profit_ratio(),
-                            t.open_rate, t.close_rate, t.amount,
-                            (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
-                             if t.close_date else None),
-                            t.sell_reason,
-                            t.fee_open, t.fee_close,
-                            t.open_rate_requested,
-                            t.close_rate_requested,
-                            t.stake_amount,
-                            t.max_rate,
-                            t.min_rate,
-                            t.id, t.exchange,
-                            t.stop_loss, t.initial_stop_loss,
-                            t.strategy, t.timeframe
-                            )
-                           for t in Trade.get_trades(filters).all()],
-                          columns=columns)
 
     return trades
 

freqtrade/optimize/backtesting.py

@@ -3,7 +3,7 @@
 """
 This module contains the backtesting logic
 """
-from freqtrade.data.btanalysis import BT_DATA_COLUMNS
+from freqtrade.data.btanalysis import BT_DATA_COLUMNS, trade_list_to_dataframe
 import logging
 from collections import defaultdict
 from copy import deepcopy
@@ -385,11 +385,7 @@ class Backtesting:
 
         trades += self.handle_left_open(open_trades, data=data)
 
-        df = DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS)
+        return trade_list_to_dataframe(trades)
-        if len(df) > 0:
-            df.loc[:, 'close_date'] = to_datetime(df['close_date'], utc=True)
-            df.loc[:, 'open_date'] = to_datetime(df['open_date'], utc=True)
-        return df
 
     def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
         logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())

tests/data/test_btanalysis.py

@@ -7,14 +7,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import LAST_BT_RESULT_FN
-from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism,
+from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_MID, BT_DATA_COLUMNS_OLD,
-                                       calculate_market_change, calculate_max_drawdown,
+                                       analyze_trade_parallelism, calculate_market_change,
+                                       calculate_max_drawdown,
                                        combine_dataframes_with_mean, create_cum_profit,
                                        extract_trades_of_period, get_latest_backtest_filename,
                                        get_latest_hyperopt_file, load_backtest_data, load_trades,
                                        load_trades_from_db)
 from freqtrade.data.history import load_data, load_pair_history
-from freqtrade.optimize.backtesting import BacktestResult
 from tests.conftest import create_mock_trades
 from tests.conftest_trades import MOCK_TRADE_COUNT
 
@@ -55,7 +55,7 @@ def test_load_backtest_data_old_format(testdatadir):
     filename = testdatadir / "backtest-result_test.json"
     bt_data = load_backtest_data(filename)
     assert isinstance(bt_data, DataFrame)
-    assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit_abs"]
+    assert list(bt_data.columns) == BT_DATA_COLUMNS_OLD + ['profit_abs', 'profit_ratio']
     assert len(bt_data) == 179
 
     # Test loading from string (must yield same result)
@@ -71,7 +71,7 @@ def test_load_backtest_data_new_format(testdatadir):
     filename = testdatadir / "backtest-result_new.json"
     bt_data = load_backtest_data(filename)
     assert isinstance(bt_data, DataFrame)
-    assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"])
+    assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID)
     assert len(bt_data) == 179
 
     # Test loading from string (must yield same result)
@@ -95,7 +95,7 @@ def test_load_backtest_data_multi(testdatadir):
     for strategy in ('DefaultStrategy', 'TestStrategy'):
         bt_data = load_backtest_data(filename, strategy=strategy)
         assert isinstance(bt_data, DataFrame)
-        assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"])
+        assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID)
         assert len(bt_data) == 179
 
         # Test loading from string (must yield same result)
@@ -122,7 +122,7 @@ def test_load_trades_from_db(default_conf, fee, mocker):
     assert isinstance(trades, DataFrame)
     assert "pair" in trades.columns
     assert "open_date" in trades.columns
-    assert "profit_percent" in trades.columns
+    assert "profit_ratio" in trades.columns
 
     for col in BT_DATA_COLUMNS:
         if col not in ['index', 'open_at_end']: