Merge pull request #6940 from freqtrade/bt_orders

Open orders should also be shown in the UI
This commit is contained in:
Matthias 2022-06-06 13:44:21 +02:00 committed by GitHub
commit 5007024f63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 102 additions and 31 deletions

View File

@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'profit_ratio', 'profit_abs', 'exit_reason', 'profit_ratio', 'profit_abs', 'exit_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag', 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag',
'is_short' 'is_short', 'open_timestamp', 'close_timestamp', 'orders'
] ]
@ -283,6 +283,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
if 'enter_tag' not in df.columns: if 'enter_tag' not in df.columns:
df['enter_tag'] = df['buy_tag'] df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1) df = df.drop(['buy_tag'], axis=1)
if 'orders' not in df.columns:
df.loc[:, 'orders'] = None
else: else:
# old format - only with lists. # old format - only with lists.
@ -337,7 +339,7 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame:
:param trades: List of trade objects :param trades: List of trade objects
:return: Dataframe with BT_DATA_COLUMNS :return: Dataframe with BT_DATA_COLUMNS
""" """
df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS) df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS)
if len(df) > 0: if len(df) > 0:
df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True)
df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True)

View File

@ -1094,6 +1094,7 @@ class Backtesting:
# 5. Process exit orders. # 5. Process exit orders.
order = trade.select_order(trade.exit_side, is_open=True) order = trade.select_order(trade.exit_side, is_open=True)
if order and self._get_order_filled(order.price, row): if order and self._get_order_filled(order.price, row):
order.close_bt_order(current_time, trade)
trade.open_order_id = None trade.open_order_id = None
trade.close_date = current_time trade.close_date = current_time
trade.close(order.price, show_msg=False) trade.close(order.price, show_msg=False)

View File

@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Union from typing import Any, Dict, List, Union
from numpy import int64
from pandas import DataFrame, to_datetime from pandas import DataFrame, to_datetime
from tabulate import tabulate from tabulate import tabulate
@ -417,9 +416,6 @@ def generate_strategy_stats(pairlist: List[str],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
if not results.empty:
results['open_timestamp'] = results['open_date'].view(int64) // 1e6
results['close_timestamp'] = results['close_date'].view(int64) // 1e6
backtest_days = (max_date - min_date).days or 1 backtest_days = (max_date - min_date).days or 1
strat_stats = { strat_stats = {

View File

@ -247,6 +247,35 @@ def set_sqlite_to_wal(engine):
connection.execute(text("PRAGMA journal_mode=wal")) connection.execute(text("PRAGMA journal_mode=wal"))
def fix_old_dry_orders(engine):
with engine.begin() as connection:
connection.execute(
text(
"""
update orders
set ft_is_open = 0
where ft_is_open = 1 and (ft_trade_id, order_id) not in (
select id, stoploss_order_id from trades where stoploss_order_id is not null
) and ft_order_side = 'stoploss'
and order_id like 'dry_%'
"""
)
)
connection.execute(
text(
"""
update orders
set ft_is_open = 0
where ft_is_open = 1
and (ft_trade_id, order_id) not in (
select id, open_order_id from trades where open_order_id is not null
) and ft_order_side != 'stoploss'
and order_id like 'dry_%'
"""
)
)
def check_migrate(engine, decl_base, previous_tables) -> None: def check_migrate(engine, decl_base, previous_tables) -> None:
""" """
Checks if migration is necessary and migrates if necessary Checks if migration is necessary and migrates if necessary
@ -288,3 +317,4 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
"start with a fresh database.") "start with a fresh database.")
set_sqlite_to_wal(engine) set_sqlite_to_wal(engine)
fix_old_dry_orders(engine)

View File

@ -137,17 +137,23 @@ class Order(_DECL_BASE):
'info': {}, 'info': {},
} }
def to_json(self, entry_side: str) -> Dict[str, Any]: def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]:
return { resp = {
'amount': self.amount,
'safe_price': self.safe_price,
'ft_order_side': self.ft_order_side,
'order_filled_timestamp': int(self.order_filled_date.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None,
'ft_is_entry': self.ft_order_side == entry_side,
}
if not minified:
resp.update({
'pair': self.ft_pair, 'pair': self.ft_pair,
'order_id': self.order_id, 'order_id': self.order_id,
'status': self.status, 'status': self.status,
'amount': self.amount,
'average': round(self.average, 8) if self.average else 0, 'average': round(self.average, 8) if self.average else 0,
'safe_price': self.safe_price,
'cost': self.cost if self.cost else 0, 'cost': self.cost if self.cost else 0,
'filled': self.filled, 'filled': self.filled,
'ft_order_side': self.ft_order_side,
'is_open': self.ft_is_open, 'is_open': self.ft_is_open,
'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT)
if self.order_date else None, if self.order_date else None,
@ -155,17 +161,16 @@ class Order(_DECL_BASE):
tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None,
'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT)
if self.order_filled_date else None, if self.order_filled_date else None,
'order_filled_timestamp': int(self.order_filled_date.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None,
'order_type': self.order_type, 'order_type': self.order_type,
'price': self.price, 'price': self.price,
'ft_is_entry': self.ft_order_side == entry_side,
'remaining': self.remaining, 'remaining': self.remaining,
} })
return resp
def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'):
self.order_filled_date = close_date self.order_filled_date = close_date
self.filled = self.amount self.filled = self.amount
self.remaining = 0
self.status = 'closed' self.status = 'closed'
self.ft_is_open = False self.ft_is_open = False
if (self.ft_order_side == trade.entry_side if (self.ft_order_side == trade.entry_side
@ -393,9 +398,9 @@ class LocalTrade():
f'open_rate={self.open_rate:.8f}, open_since={open_since})' f'open_rate={self.open_rate:.8f}, open_since={open_since})'
) )
def to_json(self) -> Dict[str, Any]: def to_json(self, minified: bool = False) -> Dict[str, Any]:
filled_orders = self.select_filled_orders() filled_orders = self.select_filled_or_open_orders()
orders = [order.to_json(self.entry_side) for order in filled_orders] orders = [order.to_json(self.entry_side, minified) for order in filled_orders]
return { return {
'trade_id': self.id, 'trade_id': self.id,
@ -897,6 +902,21 @@ class LocalTrade():
(o.filled or 0) > 0 and (o.filled or 0) > 0 and
o.status in NON_OPEN_EXCHANGE_STATES] o.status in NON_OPEN_EXCHANGE_STATES]
def select_filled_or_open_orders(self) -> List['Order']:
"""
Finds filled or open orders
:param order_side: Side of the order (either 'buy', 'sell', or None)
:return: array of Order objects
"""
return [o for o in self.orders if
(
o.ft_is_open is False
and (o.filled or 0) > 0
and o.status in NON_OPEN_EXCHANGE_STATES
)
or (o.ft_is_open is True and o.status is not None)
]
@property @property
def nr_of_successful_entries(self) -> int: def nr_of_successful_entries(self) -> int:
""" """

View File

@ -396,7 +396,7 @@ class Telegram(RPCHandler):
first_avg = filled_orders[0]["safe_price"] first_avg = filled_orders[0]["safe_price"]
for x, order in enumerate(filled_orders): for x, order in enumerate(filled_orders):
if not order['ft_is_entry']: if not order['ft_is_entry'] or order['is_open'] is True:
continue continue
cur_entry_datetime = arrow.get(order["order_filled_date"]) cur_entry_datetime = arrow.get(order["order_filled_date"])
cur_entry_amount = order["amount"] cur_entry_amount = order["amount"]

View File

@ -85,7 +85,7 @@ def test_load_backtest_data_new_format(testdatadir):
filename = testdatadir / "backtest_results/backtest-result_new.json" filename = testdatadir / "backtest_results/backtest-result_new.json"
bt_data = load_backtest_data(filename) bt_data = load_backtest_data(filename)
assert isinstance(bt_data, DataFrame) assert isinstance(bt_data, DataFrame)
assert set(bt_data.columns) == set(BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) assert set(bt_data.columns) == set(BT_DATA_COLUMNS)
assert len(bt_data) == 179 assert len(bt_data) == 179
# Test loading from string (must yield same result) # Test loading from string (must yield same result)
@ -110,7 +110,7 @@ def test_load_backtest_data_multi(testdatadir):
bt_data = load_backtest_data(filename, strategy=strategy) bt_data = load_backtest_data(filename, strategy=strategy)
assert isinstance(bt_data, DataFrame) assert isinstance(bt_data, DataFrame)
assert set(bt_data.columns) == set( assert set(bt_data.columns) == set(
BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) BT_DATA_COLUMNS)
assert len(bt_data) == 179 assert len(bt_data) == 179
# Test loading from string (must yield same result) # Test loading from string (must yield same result)

View File

@ -795,10 +795,27 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None:
'is_open': [False, False], 'is_open': [False, False],
'enter_tag': [None, None], 'enter_tag': [None, None],
"is_short": [False, False], "is_short": [False, False],
'open_timestamp': [1517251200000, 1517283000000],
'close_timestamp': [1517265300000, 1517285400000],
'orders': [
[
{'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy',
'order_filled_timestamp': 1517251200000, 'ft_is_entry': True},
{'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell',
'order_filled_timestamp': 1517265300000, 'ft_is_entry': False}
], [
{'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy',
'order_filled_timestamp': 1517283000000, 'ft_is_entry': True},
{'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell',
'order_filled_timestamp': 1517285400000, 'ft_is_entry': False}
]
]
}) })
pd.testing.assert_frame_equal(results, expected) pd.testing.assert_frame_equal(results, expected)
assert 'orders' in results.columns
data_pair = processed[pair] data_pair = processed[pair]
for _, t in results.iterrows(): for _, t in results.iterrows():
assert len(t['orders']) == 2
ln = data_pair.loc[data_pair["date"] == t["open_date"]] ln = data_pair.loc[data_pair["date"] == t["open_date"]]
# Check open trade rate alignes to open rate # Check open trade rate alignes to open rate
assert ln is not None assert ln is not None

View File

@ -70,9 +70,14 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) ->
'is_open': [False, False], 'is_open': [False, False],
'enter_tag': [None, None], 'enter_tag': [None, None],
'is_short': [False, False], 'is_short': [False, False],
'open_timestamp': [1517251200000, 1517283000000],
'close_timestamp': [1517265300000, 1517285400000],
}) })
pd.testing.assert_frame_equal(results, expected) pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected)
data_pair = processed[pair] data_pair = processed[pair]
assert len(results.iloc[0]['orders']) == 6
assert len(results.iloc[1]['orders']) == 2
for _, t in results.iterrows(): for _, t in results.iterrows():
ln = data_pair.loc[data_pair["date"] == t["open_date"]] ln = data_pair.loc[data_pair["date"] == t["open_date"]]
# Check open trade rate alignes to open rate # Check open trade rate alignes to open rate