adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Streamline trade to dataframe conversion

Browse Source
This commit is contained in:
Matthias 2021-01-23 20:49:49 +01:00
parent 8ee264bc59
commit deb8432d33
3 changed files with 43 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
freqtrade/data/btanalysis.py
View File

@ -2,9 +2,8 @@
Helpers when analyzing backtest data
"""
import logging
from datetime import timezone
from pathlib import Path
from typing import Any, Dict, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple, Union
import numpy as np
import pandas as pd
@ -16,9 +15,21 @@ from freqtrade.persistence import Trade, init_db
logger = logging.getLogger(__name__)
# must align with columns in backtest.py
BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration",
"open_rate", "close_rate", "open_at_end", "sell_reason"]
# Old format - maybe remove?
BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
"trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
# Mid-term format, crated by BacktestResult Named Tuple
BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
'fee_close', 'amount', 'profit_abs', 'profit_ratio']
# Newest format
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'fee_open', 'fee_close', 'trade_duration',
'profit_ratio', 'profit_abs', 'sell_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
@ -154,7 +165,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
)
else:
# old format - only with lists.
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS)
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
df['open_date'] = pd.to_datetime(df['open_date'],
unit='s',
@ -166,7 +177,10 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
utc=True,
infer_datetime_format=True
)
# Create compatibility with new format
df['profit_abs'] = df['close_rate'] - df['open_rate']
if 'profit_ratio' not in df.columns:
df['profit_ratio'] = df['profit_percent']
df = df.sort_values("open_date").reset_index(drop=True)
return df
@ -209,6 +223,19 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
return df_final[df_final['open_trades'] > max_open_trades]
def trade_list_to_dataframe(trades: List[Trade]) -> pd.DataFrame:
"""
Convert list of Trade objects to pandas Dataframe
:param trades: List of trade objects
:return: Dataframe with BT_DATA_COLUMNS
"""
df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS)
if len(df) > 0:
df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True)
df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True)
return df
def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame:
"""
Load trades from a DB (using dburl)
@ -219,36 +246,10 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
"""
init_db(db_url, clean_open_orders=False)
columns = ["pair", "open_date", "close_date", "profit", "profit_percent",
"open_rate", "close_rate", "amount", "trade_duration", "sell_reason",
"fee_open", "fee_close", "open_rate_requested", "close_rate_requested",
"stake_amount", "max_rate", "min_rate", "id", "exchange",
"stop_loss", "initial_stop_loss", "strategy", "timeframe"]
filters = []
if strategy:
filters.append(Trade.strategy == strategy)
trades = pd.DataFrame([(t.pair,
t.open_date.replace(tzinfo=timezone.utc),
t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None,
t.calc_profit(), t.calc_profit_ratio(),
t.open_rate, t.close_rate, t.amount,
(round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2)
if t.close_date else None),
t.sell_reason,
t.fee_open, t.fee_close,
t.open_rate_requested,
t.close_rate_requested,
t.stake_amount,
t.max_rate,
t.min_rate,
t.id, t.exchange,
t.stop_loss, t.initial_stop_loss,
t.strategy, t.timeframe
)
for t in Trade.get_trades(filters).all()],
columns=columns)
trades = trade_list_to_dataframe(Trade.get_trades(filters).all())
return trades

8
freqtrade/optimize/backtesting.py
View File

@ -3,7 +3,7 @@
"""
This module contains the backtesting logic
"""
from freqtrade.data.btanalysis import BT_DATA_COLUMNS
from freqtrade.data.btanalysis import BT_DATA_COLUMNS, trade_list_to_dataframe
import logging
from collections import defaultdict
from copy import deepcopy
@ -385,11 +385,7 @@ class Backtesting:
trades += self.handle_left_open(open_trades, data=data)
df = DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS)
if len(df) > 0:
df.loc[:, 'close_date'] = to_datetime(df['close_date'], utc=True)
df.loc[:, 'open_date'] = to_datetime(df['open_date'], utc=True)
return df
return trade_list_to_dataframe(trades)
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())

14
tests/data/test_btanalysis.py
View File

@ -7,14 +7,14 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism,
calculate_market_change, calculate_max_drawdown,
from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_MID, BT_DATA_COLUMNS_OLD,
analyze_trade_parallelism, calculate_market_change,
calculate_max_drawdown,
combine_dataframes_with_mean, create_cum_profit,
extract_trades_of_period, get_latest_backtest_filename,
get_latest_hyperopt_file, load_backtest_data, load_trades,
load_trades_from_db)
from freqtrade.data.history import load_data, load_pair_history
from freqtrade.optimize.backtesting import BacktestResult
from tests.conftest import create_mock_trades
from tests.conftest_trades import MOCK_TRADE_COUNT
@ -55,7 +55,7 @@ def test_load_backtest_data_old_format(testdatadir):
filename = testdatadir / "backtest-result_test.json"
bt_data = load_backtest_data(filename)
assert isinstance(bt_data, DataFrame)
assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit_abs"]
assert list(bt_data.columns) == BT_DATA_COLUMNS_OLD + ['profit_abs', 'profit_ratio']
assert len(bt_data) == 179
# Test loading from string (must yield same result)
@ -71,7 +71,7 @@ def test_load_backtest_data_new_format(testdatadir):
filename = testdatadir / "backtest-result_new.json"
bt_data = load_backtest_data(filename)
assert isinstance(bt_data, DataFrame)
assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"])
assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID)
assert len(bt_data) == 179
# Test loading from string (must yield same result)
@ -95,7 +95,7 @@ def test_load_backtest_data_multi(testdatadir):
for strategy in ('DefaultStrategy', 'TestStrategy'):
bt_data = load_backtest_data(filename, strategy=strategy)
assert isinstance(bt_data, DataFrame)
assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"])
assert set(bt_data.columns) == set(BT_DATA_COLUMNS_MID)
assert len(bt_data) == 179
# Test loading from string (must yield same result)
@ -122,7 +122,7 @@ def test_load_trades_from_db(default_conf, fee, mocker):
assert isinstance(trades, DataFrame)
assert "pair" in trades.columns
assert "open_date" in trades.columns
assert "profit_percent" in trades.columns
assert "profit_ratio" in trades.columns
for col in BT_DATA_COLUMNS:
if col not in ['index', 'open_at_end']: