Streamline trade to dataframe conversion

freqtrade/data/btanalysis.py

@@ -2,9 +2,8 @@
 Helpers when analyzing backtest data
 """
 import logging
-from datetime import timezone
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
@@ -16,9 +15,21 @@ from freqtrade.persistence import Trade, init_db
 
 logger = logging.getLogger(__name__)
 
-# must align with columns in backtest.py
-BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration",
-                   "open_rate", "close_rate", "open_at_end", "sell_reason"]
+# Old format - maybe remove?
+BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
+                       "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
+
+# Mid-term format, crated by BacktestResult Named Tuple
+BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
+                       'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
+                       'fee_close', 'amount', 'profit_abs', 'profit_ratio']
+
+# Newest format
+BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
+                   'fee_open', 'fee_close', 'trade_duration',
+                   'profit_ratio', 'profit_abs', 'sell_reason',
+                   'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
+                   'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
 
 
 def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
@@ -154,7 +165,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                           )
     else:
         # old format - only with lists.
-        df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
+        df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
 
         df['open_date'] = pd.to_datetime(df['open_date'],
                                          unit='s',
@@ -166,7 +177,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
                                           utc=True,
                                           infer_datetime_format=True
                                           )
+        # Create compatibility with new format
         df['profit_abs'] = df['close_rate'] - df['open_rate']
+    if 'profit_ratio' not in df.columns:
+        df['profit_ratio'] = df['profit_percent']
     df = df.sort_values("open_date").reset_index(drop=True)
     return df
 
@@ -209,6 +223,19 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
     return df_final[df_final['open_trades'] > max_open_trades]
 
 
+def trade_list_to_dataframe(trades: List[Trade]) -> pd.DataFrame:
+    """
+    Convert list of Trade objects to pandas Dataframe
+    :param trades: List of trade objects
+    :return: Dataframe with BT_DATA_COLUMNS
+    """
+    df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS)
+    if len(df) > 0:
+        df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True)
+        df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True)
+    return df
+
+
 def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame:
     """
     Load trades from a DB (using dburl)
@@ -219,36 +246,10 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
     """
     init_db(db_url, clean_open_orders=False)
 
-    columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
-               "open_rate", "close_rate", "amount", "trade_duration", "sell_reason",
-               "fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
-               "stake_amount", "max_rate", "min_rate", "id", "exchange",
-               "stop_loss", "initial_stop_loss", "strategy", "timeframe"]
-
     filters = []
     if strategy:
         filters.append(Trade.strategy == strategy)
-
-    trades = pd.DataFrame([(t.pair,
-                            t.open_date.replace(tzinfo=timezone.utc),
-                            t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
-                            t.calc_profit(), t.calc_profit_ratio(),
-                            t.open_rate, t.close_rate, t.amount,
-                            (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
-                             if t.close_date else None),
-                            t.sell_reason,
-                            t.fee_open, t.fee_close,
-                            t.open_rate_requested,
-                            t.close_rate_requested,
-                            t.stake_amount,
-                            t.max_rate,
-                            t.min_rate,
-                            t.id, t.exchange,
-                            t.stop_loss, t.initial_stop_loss,
-                            t.strategy, t.timeframe
-                            )
-                           for t in Trade.get_trades(filters).all()],
-                          columns=columns)
+    trades = trade_list_to_dataframe(Trade.get_trades(filters).all())
 
     return trades