From deb8432d3395b6db8c03991e1321e3d144d2a580 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 23 Jan 2021 20:49:49 +0100 Subject: [PATCH] Streamline trade to dataframe conversion --- freqtrade/data/btanalysis.py | 67 ++++++++++++++++--------------- freqtrade/optimize/backtesting.py | 8 +--- tests/data/test_btanalysis.py | 14 +++---- 3 files changed, 43 insertions(+), 46 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 2b51f5371..20977e447 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -2,9 +2,8 @@ Helpers when analyzing backtest data """ import logging -from datetime import timezone from pathlib import Path -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np import pandas as pd @@ -16,9 +15,21 @@ from freqtrade.persistence import Trade, init_db logger = logging.getLogger(__name__) -# must align with columns in backtest.py -BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration", - "open_rate", "close_rate", "open_at_end", "sell_reason"] +# Old format - maybe remove? +BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index", + "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"] + +# Mid-term format, crated by BacktestResult Named Tuple +BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration', + 'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open', + 'fee_close', 'amount', 'profit_abs', 'profit_ratio'] + +# Newest format +BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', + 'fee_open', 'fee_close', 'trade_duration', + 'profit_ratio', 'profit_abs', 'sell_reason', + 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', + 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ] def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str: @@ -154,7 +165,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non ) else: # old format - only with lists. - df = pd.DataFrame(data, columns=BT_DATA_COLUMNS) + df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD) df['open_date'] = pd.to_datetime(df['open_date'], unit='s', @@ -166,7 +177,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non utc=True, infer_datetime_format=True ) + # Create compatibility with new format df['profit_abs'] = df['close_rate'] - df['open_rate'] + if 'profit_ratio' not in df.columns: + df['profit_ratio'] = df['profit_percent'] df = df.sort_values("open_date").reset_index(drop=True) return df @@ -209,6 +223,19 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str, return df_final[df_final['open_trades'] > max_open_trades] +def trade_list_to_dataframe(trades: List[Trade]) -> pd.DataFrame: + """ + Convert list of Trade objects to pandas Dataframe + :param trades: List of trade objects + :return: Dataframe with BT_DATA_COLUMNS + """ + df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS) + if len(df) > 0: + df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) + df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) + return df + + def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame: """ Load trades from a DB (using dburl) @@ -219,36 +246,10 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF """ init_db(db_url, clean_open_orders=False) - columns = ["pair", "open_date", "close_date", "profit", "profit_percent", - "open_rate", "close_rate", "amount", "trade_duration", "sell_reason", - "fee_open", "fee_close", "open_rate_requested", "close_rate_requested", - "stake_amount", "max_rate", "min_rate", "id", "exchange", - "stop_loss", "initial_stop_loss", "strategy", "timeframe"] - filters = [] if strategy: filters.append(Trade.strategy == strategy) - - trades = pd.DataFrame([(t.pair, - t.open_date.replace(tzinfo=timezone.utc), - t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None, - t.calc_profit(), t.calc_profit_ratio(), - t.open_rate, t.close_rate, t.amount, - (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) - if t.close_date else None), - t.sell_reason, - t.fee_open, t.fee_close, - t.open_rate_requested, - t.close_rate_requested, - t.stake_amount, - t.max_rate, - t.min_rate, - t.id, t.exchange, - t.stop_loss, t.initial_stop_loss, - t.strategy, t.timeframe - ) - for t in Trade.get_trades(filters).all()], - columns=columns) + trades = trade_list_to_dataframe(Trade.get_trades(filters).all()) return trades diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 08dbdffc4..875538731 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -3,7 +3,7 @@ """ This module contains the backtesting logic """ -from freqtrade.data.btanalysis import BT_DATA_COLUMNS +from freqtrade.data.btanalysis import BT_DATA_COLUMNS, trade_list_to_dataframe import logging from collections import defaultdict from copy import deepcopy @@ -385,11 +385,7 @@ class Backtesting: trades += self.handle_left_open(open_trades, data=data) - df = DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS) - if len(df) > 0: - df.loc[:, 'close_date'] = to_datetime(df['close_date'], utc=True) - df.loc[:, 'open_date'] = to_datetime(df['open_date'], utc=True) - return df + return trade_list_to_dataframe(trades) def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange): logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index cdd5c08d2..9d6a31955 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -7,14 +7,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, - calculate_market_change, calculate_max_drawdown, +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_MID, BT_DATA_COLUMNS_OLD, + analyze_trade_parallelism, calculate_market_change, + calculate_max_drawdown, combine_dataframes_with_mean, create_cum_profit, extract_trades_of_period, get_latest_backtest_filename, get_latest_hyperopt_file, load_backtest_data, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history -from freqtrade.optimize.backtesting import BacktestResult from tests.conftest import create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT @@ -55,7 +55,7 @@ def test_load_backtest_data_old_format(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit_abs"] + assert list(bt_data.columns) == BT_DATA_COLUMNS_OLD + ['profit_abs', 'profit_ratio'] assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -71,7 +71,7 @@ def test_load_backtest_data_new_format(testdatadir): filename = testdatadir / "backtest-result_new.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"]) + assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID) assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -95,7 +95,7 @@ def test_load_backtest_data_multi(testdatadir): for strategy in ('DefaultStrategy', 'TestStrategy'): bt_data = load_backtest_data(filename, strategy=strategy) assert isinstance(bt_data, DataFrame) - assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"]) + assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID) assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -122,7 +122,7 @@ def test_load_trades_from_db(default_conf, fee, mocker): assert isinstance(trades, DataFrame) assert "pair" in trades.columns assert "open_date" in trades.columns - assert "profit_percent" in trades.columns + assert "profit_ratio" in trades.columns for col in BT_DATA_COLUMNS: if col not in ['index', 'open_at_end']: