diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index fef432576..9e38f6833 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'profit_ratio', 'profit_abs', 'exit_reason', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag', - 'is_short' + 'is_short', 'open_timestamp', 'close_timestamp', 'orders' ] @@ -283,6 +283,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non if 'enter_tag' not in df.columns: df['enter_tag'] = df['buy_tag'] df = df.drop(['buy_tag'], axis=1) + if 'orders' not in df.columns: + df.loc[:, 'orders'] = None else: # old format - only with lists. @@ -337,7 +339,7 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: :param trades: List of trade objects :return: Dataframe with BT_DATA_COLUMNS """ - df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS) + df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS) if len(df) > 0: df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index aebaecaca..8fe5f509e 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1094,6 +1094,7 @@ class Backtesting: # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): + order.close_bt_order(current_time, trade) trade.open_order_id = None trade.close_date = current_time trade.close(order.price, show_msg=False) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 93336fa3f..e3dd17411 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union -from numpy import int64 from pandas import DataFrame, to_datetime from tabulate import tabulate @@ -417,9 +416,6 @@ def generate_strategy_stats(pairlist: List[str], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - if not results.empty: - results['open_timestamp'] = results['open_date'].view(int64) // 1e6 - results['close_timestamp'] = results['close_date'].view(int64) // 1e6 backtest_days = (max_date - min_date).days or 1 strat_stats = { diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 53e35d9da..b0fdf0412 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -247,6 +247,35 @@ def set_sqlite_to_wal(engine): connection.execute(text("PRAGMA journal_mode=wal")) +def fix_old_dry_orders(engine): + with engine.begin() as connection: + connection.execute( + text( + """ + update orders + set ft_is_open = 0 + where ft_is_open = 1 and (ft_trade_id, order_id) not in ( + select id, stoploss_order_id from trades where stoploss_order_id is not null + ) and ft_order_side = 'stoploss' + and order_id like 'dry_%' + """ + ) + ) + connection.execute( + text( + """ + update orders + set ft_is_open = 0 + where ft_is_open = 1 + and (ft_trade_id, order_id) not in ( + select id, open_order_id from trades where open_order_id is not null + ) and ft_order_side != 'stoploss' + and order_id like 'dry_%' + """ + ) + ) + + def check_migrate(engine, decl_base, previous_tables) -> None: """ Checks if migration is necessary and migrates if necessary @@ -288,3 +317,4 @@ def check_migrate(engine, decl_base, previous_tables) -> None: "start with a fresh database.") set_sqlite_to_wal(engine) + fix_old_dry_orders(engine) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 45a16bfbd..0be9d22c1 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -137,35 +137,40 @@ class Order(_DECL_BASE): 'info': {}, } - def to_json(self, entry_side: str) -> Dict[str, Any]: - return { - 'pair': self.ft_pair, - 'order_id': self.order_id, - 'status': self.status, + def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]: + resp = { 'amount': self.amount, - 'average': round(self.average, 8) if self.average else 0, 'safe_price': self.safe_price, - 'cost': self.cost if self.cost else 0, - 'filled': self.filled, 'ft_order_side': self.ft_order_side, - 'is_open': self.ft_is_open, - 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_date else None, - 'order_timestamp': int(self.order_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, - 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_filled_date else None, 'order_filled_timestamp': int(self.order_filled_date.replace( tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, - 'order_type': self.order_type, - 'price': self.price, 'ft_is_entry': self.ft_order_side == entry_side, - 'remaining': self.remaining, } + if not minified: + resp.update({ + 'pair': self.ft_pair, + 'order_id': self.order_id, + 'status': self.status, + 'average': round(self.average, 8) if self.average else 0, + 'cost': self.cost if self.cost else 0, + 'filled': self.filled, + 'is_open': self.ft_is_open, + 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_date else None, + 'order_timestamp': int(self.order_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, + 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_filled_date else None, + 'order_type': self.order_type, + 'price': self.price, + 'remaining': self.remaining, + }) + return resp def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): self.order_filled_date = close_date self.filled = self.amount + self.remaining = 0 self.status = 'closed' self.ft_is_open = False if (self.ft_order_side == trade.entry_side @@ -393,9 +398,9 @@ class LocalTrade(): f'open_rate={self.open_rate:.8f}, open_since={open_since})' ) - def to_json(self) -> Dict[str, Any]: - filled_orders = self.select_filled_orders() - orders = [order.to_json(self.entry_side) for order in filled_orders] + def to_json(self, minified: bool = False) -> Dict[str, Any]: + filled_orders = self.select_filled_or_open_orders() + orders = [order.to_json(self.entry_side, minified) for order in filled_orders] return { 'trade_id': self.id, @@ -897,6 +902,21 @@ class LocalTrade(): (o.filled or 0) > 0 and o.status in NON_OPEN_EXCHANGE_STATES] + def select_filled_or_open_orders(self) -> List['Order']: + """ + Finds filled or open orders + :param order_side: Side of the order (either 'buy', 'sell', or None) + :return: array of Order objects + """ + return [o for o in self.orders if + ( + o.ft_is_open is False + and (o.filled or 0) > 0 + and o.status in NON_OPEN_EXCHANGE_STATES + ) + or (o.ft_is_open is True and o.status is not None) + ] + @property def nr_of_successful_entries(self) -> int: """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4a274002e..e456b1eef 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -396,7 +396,7 @@ class Telegram(RPCHandler): first_avg = filled_orders[0]["safe_price"] for x, order in enumerate(filled_orders): - if not order['ft_is_entry']: + if not order['ft_is_entry'] or order['is_open'] is True: continue cur_entry_datetime = arrow.get(order["order_filled_date"]) cur_entry_amount = order["amount"] diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 4157bd899..977140ebb 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -85,7 +85,7 @@ def test_load_backtest_data_new_format(testdatadir): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert set(bt_data.columns) == set(BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) + assert set(bt_data.columns) == set(BT_DATA_COLUMNS) assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -110,7 +110,7 @@ def test_load_backtest_data_multi(testdatadir): bt_data = load_backtest_data(filename, strategy=strategy) assert isinstance(bt_data, DataFrame) assert set(bt_data.columns) == set( - BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) + BT_DATA_COLUMNS) assert len(bt_data) == 179 # Test loading from string (must yield same result) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f169e0a35..6912184aa 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -795,10 +795,27 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'is_open': [False, False], 'enter_tag': [None, None], "is_short": [False, False], + 'open_timestamp': [1517251200000, 1517283000000], + 'close_timestamp': [1517265300000, 1517285400000], + 'orders': [ + [ + {'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy', + 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True}, + {'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell', + 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False} + ], [ + {'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy', + 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True}, + {'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell', + 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False} + ] + ] }) pd.testing.assert_frame_equal(results, expected) + assert 'orders' in results.columns data_pair = processed[pair] for _, t in results.iterrows(): + assert len(t['orders']) == 2 ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate assert ln is not None diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 94505e3ce..fca9c01b2 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -70,9 +70,14 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'is_open': [False, False], 'enter_tag': [None, None], 'is_short': [False, False], + 'open_timestamp': [1517251200000, 1517283000000], + 'close_timestamp': [1517265300000, 1517285400000], }) - pd.testing.assert_frame_equal(results, expected) + pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected) data_pair = processed[pair] + assert len(results.iloc[0]['orders']) == 6 + assert len(results.iloc[1]['orders']) == 2 + for _, t in results.iterrows(): ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate