stable/freqtrade/persistence/models.py

1116 lines
43 KiB
Python
Raw Normal View History

2018-03-02 15:22:00 +00:00
"""
This module contains the class to persist trades into SQLite
"""
import logging
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from typing import Any, Dict, List, Optional
2017-05-12 17:11:56 +00:00
2020-09-28 17:39:41 +00:00
from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
create_engine, desc, func, inspect)
2018-06-07 19:35:57 +00:00
from sqlalchemy.exc import NoSuchModuleError
2021-04-13 15:34:20 +00:00
from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker
2017-11-09 22:45:22 +00:00
from sqlalchemy.pool import StaticPool
2020-08-13 14:14:28 +00:00
from sqlalchemy.sql.schema import UniqueConstraint
2017-05-12 17:11:56 +00:00
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
from freqtrade.enums import SellType
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.misc import safe_value_fallback
2020-08-13 06:33:46 +00:00
from freqtrade.persistence.migrations import check_migrate
2018-06-07 19:35:57 +00:00
2020-09-28 17:39:41 +00:00
logger = logging.getLogger(__name__)
2019-09-10 07:42:45 +00:00
2018-05-31 19:10:15 +00:00
_DECL_BASE: Any = declarative_base()
2018-06-23 13:27:29 +00:00
_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls'
2017-05-12 17:11:56 +00:00
2020-10-16 05:39:12 +00:00
def init_db(db_url: str, clean_open_orders: bool = False) -> None:
"""
Initializes this module with the given config,
registers all known command handlers
and starts polling for message updates
:param db_url: Database to use
:param clean_open_orders: Remove open orders from the database.
Useful for dry-run or if all orders have been reset on the exchange.
:return: None
"""
kwargs = {}
if db_url == 'sqlite://':
kwargs.update({
'poolclass': StaticPool,
2021-05-23 06:56:41 +00:00
})
# Take care of thread ownership
if db_url.startswith('sqlite://'):
kwargs.update({
'connect_args': {'check_same_thread': False},
})
2018-06-07 19:35:57 +00:00
try:
2021-04-13 15:34:20 +00:00
engine = create_engine(db_url, future=True, **kwargs)
2018-06-07 19:35:57 +00:00
except NoSuchModuleError:
raise OperationalException(f"Given value for db_url: '{db_url}' "
f"is no valid database URL! (See {_SQL_DOCS_URL})")
2018-06-07 19:35:57 +00:00
2019-10-29 13:26:03 +00:00
# https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
# Scoped sessions proxy requests to the appropriate thread-local session.
# We should use the scoped_session object - not a seperately initialized version
2021-04-13 15:34:20 +00:00
Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
2021-04-05 06:46:12 +00:00
Trade.query = Trade._session.query_property()
Order.query = Trade._session.query_property()
PairLock.query = Trade._session.query_property()
2020-10-17 09:28:34 +00:00
previous_tables = inspect(engine).get_table_names()
2017-11-07 19:13:36 +00:00
_DECL_BASE.metadata.create_all(engine)
check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
# Clean dry_run DB if the db is not in-memory
if clean_open_orders and db_url != 'sqlite://':
clean_dry_run_db()
2020-10-16 05:39:12 +00:00
def cleanup_db() -> None:
"""
Flushes all pending operations to disk.
:return: None
"""
2021-04-15 05:57:52 +00:00
Trade.commit()
def clean_dry_run_db() -> None:
"""
Remove open_order_id from a Dry_run DB
:return: None
"""
for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all():
# Check we are updating only a dry_run order not a prod one
if 'dry_run' in trade.open_order_id:
trade.open_order_id = None
2021-04-15 05:57:52 +00:00
Trade.commit()
2020-08-13 07:34:53 +00:00
class Order(_DECL_BASE):
"""
Order database model
Keeps a record of all orders placed on the exchange
One to many relationship with Trades:
- One trade can have many orders
- One Order can only be associated with one Trade
Mirrors CCXT Order structure
"""
__tablename__ = 'orders'
2020-08-13 14:14:28 +00:00
# Uniqueness should be ensured over pair, order_id
# its likely that order_id is unique per Pair on some exchanges.
2020-08-13 15:17:52 +00:00
__table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
2020-08-13 07:34:53 +00:00
id = Column(Integer, primary_key=True)
2020-08-13 15:17:52 +00:00
ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
2020-08-13 07:34:53 +00:00
2020-08-13 17:37:41 +00:00
trade = relationship("Trade", back_populates="orders")
ft_order_side = Column(String(25), nullable=False)
ft_pair = Column(String(25), nullable=False)
2020-08-13 15:17:52 +00:00
ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
order_id = Column(String(255), nullable=False, index=True)
status = Column(String(255), nullable=True)
symbol = Column(String(25), nullable=True)
order_type = Column(String(50), nullable=True)
side = Column(String(25), nullable=True)
price = Column(Float, nullable=True)
2021-05-21 18:35:39 +00:00
average = Column(Float, nullable=True)
amount = Column(Float, nullable=True)
2020-08-13 07:34:53 +00:00
filled = Column(Float, nullable=True)
remaining = Column(Float, nullable=True)
cost = Column(Float, nullable=True)
order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
2020-08-13 07:34:53 +00:00
order_filled_date = Column(DateTime, nullable=True)
2020-08-13 12:13:58 +00:00
order_update_date = Column(DateTime, nullable=True)
def __repr__(self):
2020-08-13 15:17:52 +00:00
return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
2020-08-24 04:50:43 +00:00
f'side={self.side}, order_type={self.order_type}, status={self.status})')
2020-08-13 07:34:53 +00:00
def update_from_ccxt_object(self, order):
"""
Update Order from ccxt response
Only updates if fields are available from ccxt -
"""
if self.order_id != str(order['id']):
raise DependencyException("Order-id's don't match")
self.status = order.get('status', self.status)
self.symbol = order.get('symbol', self.symbol)
self.order_type = order.get('type', self.order_type)
self.side = order.get('side', self.side)
self.price = order.get('price', self.price)
self.amount = order.get('amount', self.amount)
self.filled = order.get('filled', self.filled)
2021-05-21 18:35:39 +00:00
self.average = order.get('average', self.average)
self.remaining = order.get('remaining', self.remaining)
self.cost = order.get('cost', self.cost)
if 'timestamp' in order and order['timestamp'] is not None:
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
2020-09-06 12:17:45 +00:00
self.ft_is_open = True
if self.status in NON_OPEN_EXCHANGE_STATES:
2020-08-13 15:17:52 +00:00
self.ft_is_open = False
2021-08-12 05:02:36 +00:00
if (order.get('filled', 0.0) or 0.0) > 0:
2021-04-13 09:55:03 +00:00
self.order_filled_date = datetime.now(timezone.utc)
self.order_update_date = datetime.now(timezone.utc)
2020-08-13 15:17:52 +00:00
2020-08-13 12:13:58 +00:00
@staticmethod
2020-08-13 13:39:29 +00:00
def update_orders(orders: List['Order'], order: Dict[str, Any]):
2020-08-13 12:13:58 +00:00
"""
2020-08-13 15:18:56 +00:00
Get all non-closed orders - useful when trying to batch-update orders
2020-08-13 12:13:58 +00:00
"""
if not isinstance(order, dict):
logger.warning(f"{order} is not a valid response object.")
return
filtered_orders = [o for o in orders if o.order_id == order.get('id')]
2020-08-13 13:54:36 +00:00
if filtered_orders:
oobj = filtered_orders[0]
oobj.update_from_ccxt_object(order)
2021-04-13 17:52:33 +00:00
Order.query.session.commit()
2020-08-13 13:54:36 +00:00
else:
logger.warning(f"Did not find order for {order}.")
2020-08-13 12:13:58 +00:00
2020-08-13 07:34:53 +00:00
@staticmethod
2020-08-13 14:14:28 +00:00
def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order':
2020-08-13 07:34:53 +00:00
"""
Parse an order from a ccxt object and return a new order Object.
"""
2020-08-13 14:14:28 +00:00
o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair)
o.update_from_ccxt_object(order)
2020-08-13 07:34:53 +00:00
return o
2020-08-13 15:18:56 +00:00
@staticmethod
2020-08-13 17:37:41 +00:00
def get_open_orders() -> List['Order']:
2020-08-13 15:18:56 +00:00
"""
2021-11-09 07:48:25 +00:00
Retrieve open orders from the database
:return: List of open orders
2020-08-13 15:18:56 +00:00
"""
return Order.query.filter(Order.ft_is_open.is_(True)).all()
2020-08-13 07:34:53 +00:00
class LocalTrade():
2018-03-02 15:22:00 +00:00
"""
2020-08-13 07:34:53 +00:00
Trade database model.
Used in backtesting - must be aligned to Trade model!
2017-05-12 17:11:56 +00:00
"""
use_db: bool = False
2020-11-16 19:09:34 +00:00
# Trades container for backtesting
trades: List['LocalTrade'] = []
2021-03-13 09:16:32 +00:00
trades_open: List['LocalTrade'] = []
total_profit: float = 0
id: int = 0
orders: List[Order] = []
exchange: str = ''
pair: str = ''
is_open: bool = True
fee_open: float = 0.0
fee_open_cost: Optional[float] = None
fee_open_currency: str = ''
fee_close: float = 0.0
fee_close_cost: Optional[float] = None
fee_close_currency: str = ''
2021-02-20 19:07:00 +00:00
open_rate: float = 0.0
open_rate_requested: Optional[float] = None
# open_trade_value - calculated via _calc_open_trade_value
2021-02-20 19:07:00 +00:00
open_trade_value: float = 0.0
close_rate: Optional[float] = None
close_rate_requested: Optional[float] = None
close_profit: Optional[float] = None
close_profit_abs: Optional[float] = None
2021-02-20 19:07:00 +00:00
stake_amount: float = 0.0
amount: float = 0.0
amount_requested: Optional[float] = None
open_date: datetime
close_date: Optional[datetime] = None
open_order_id: Optional[str] = None
2018-06-26 18:49:07 +00:00
# absolute value of the stop loss
stop_loss: float = 0.0
2019-03-28 20:18:26 +00:00
# percentage value of the stop loss
stop_loss_pct: float = 0.0
2018-06-26 18:49:07 +00:00
# absolute value of the initial stop loss
initial_stop_loss: float = 0.0
2019-03-28 20:18:26 +00:00
# percentage value of the initial stop loss
initial_stop_loss_pct: float = 0.0
# stoploss order id which is on exchange
stoploss_order_id: Optional[str] = None
# last update time of the stoploss order on exchange
stoploss_last_update: Optional[datetime] = None
# absolute value of the highest reached price
max_rate: float = 0.0
# Lowest price reached
min_rate: float = 0.0
sell_reason: str = ''
sell_order_status: str = ''
strategy: str = ''
2021-07-21 19:34:20 +00:00
buy_tag: Optional[str] = None
timeframe: Optional[int] = None
2017-05-12 17:11:56 +00:00
2019-12-17 06:02:02 +00:00
def __init__(self, **kwargs):
for key in kwargs:
setattr(self, key, kwargs[key])
self.recalc_open_trade_value()
2019-12-17 06:02:02 +00:00
2017-05-12 17:11:56 +00:00
def __repr__(self):
2020-10-17 09:25:42 +00:00
open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
2018-06-23 13:27:29 +00:00
return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
f'open_rate={self.open_rate:.8f}, open_since={open_since})')
@property
def open_date_utc(self):
return self.open_date.replace(tzinfo=timezone.utc)
@property
def close_date_utc(self):
return self.close_date.replace(tzinfo=timezone.utc)
def to_json(self) -> Dict[str, Any]:
return {
'trade_id': self.id,
'pair': self.pair,
2020-04-06 09:00:31 +00:00
'is_open': self.is_open,
'exchange': self.exchange,
'amount': round(self.amount, 8),
'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
'stake_amount': round(self.stake_amount, 8),
'strategy': self.strategy,
2021-07-21 13:05:35 +00:00
'buy_tag': self.buy_tag,
2020-06-02 12:56:34 +00:00
'timeframe': self.timeframe,
2020-04-06 09:00:31 +00:00
'fee_open': self.fee_open,
2020-04-30 04:51:42 +00:00
'fee_open_cost': self.fee_open_cost,
'fee_open_currency': self.fee_open_currency,
2020-04-06 09:00:31 +00:00
'fee_close': self.fee_close,
2020-04-30 04:51:42 +00:00
'fee_close_cost': self.fee_close_cost,
'fee_close_currency': self.fee_close_currency,
2020-10-17 18:32:23 +00:00
'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
'open_rate': self.open_rate,
'open_rate_requested': self.open_rate_requested,
'open_trade_value': round(self.open_trade_value, 8),
2020-10-17 18:32:23 +00:00
'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
2019-05-06 04:55:12 +00:00
if self.close_date else None),
'close_timestamp': int(self.close_date.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
'close_rate': self.close_rate,
2020-04-06 09:00:31 +00:00
'close_rate_requested': self.close_rate_requested,
'close_profit': self.close_profit, # Deprecated
'close_profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None,
'close_profit_abs': self.close_profit_abs, # Deprecated
'trade_duration_s': (int((self.close_date_utc - self.open_date_utc).total_seconds())
if self.close_date else None),
'trade_duration': (int((self.close_date_utc - self.open_date_utc).total_seconds() // 60)
2021-01-23 11:43:27 +00:00
if self.close_date else None),
'profit_ratio': self.close_profit,
'profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None,
'profit_abs': self.close_profit_abs,
2021-10-20 17:13:34 +00:00
'sell_reason': self.sell_reason,
'sell_order_status': self.sell_order_status,
2020-06-01 09:05:37 +00:00
'stop_loss_abs': self.stop_loss,
'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None,
'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
'stoploss_order_id': self.stoploss_order_id,
2020-10-17 18:32:23 +00:00
'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT)
if self.stoploss_last_update else None),
'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None,
2020-06-01 09:05:37 +00:00
'initial_stop_loss_abs': self.initial_stop_loss,
'initial_stop_loss_ratio': (self.initial_stop_loss_pct
if self.initial_stop_loss_pct else None),
2019-05-06 04:55:12 +00:00
'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100
if self.initial_stop_loss_pct else None),
2020-04-06 09:00:31 +00:00
'min_rate': self.min_rate,
'max_rate': self.max_rate,
2020-04-06 09:00:31 +00:00
'open_order_id': self.open_order_id,
}
@staticmethod
def reset_trades() -> None:
"""
Resets all trades. Only active for backtesting mode.
"""
LocalTrade.trades = []
2021-03-13 09:16:32 +00:00
LocalTrade.trades_open = []
LocalTrade.total_profit = 0
def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
"""
Adjust the max_rate and min_rate.
"""
self.max_rate = max(current_price, self.max_rate or self.open_rate)
self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
2020-12-11 06:41:06 +00:00
def _set_new_stoploss(self, new_loss: float, stoploss: float):
"""Assign new stop value"""
self.stop_loss = new_loss
self.stop_loss_pct = -1 * abs(stoploss)
self.stoploss_last_update = datetime.utcnow()
2020-02-02 04:00:40 +00:00
def adjust_stop_loss(self, current_price: float, stoploss: float,
initial: bool = False) -> None:
2019-03-17 12:18:29 +00:00
"""
This adjusts the stop loss to it's most recently observed setting
:param current_price: Current rate the asset is traded
:param stoploss: Stoploss as factor (sample -0.05 -> -5% below current price).
:param initial: Called to initiate stop_loss.
Skips everything if self.stop_loss is already set.
"""
2018-06-27 04:38:49 +00:00
if initial and not (self.stop_loss is None or self.stop_loss == 0):
# Don't modify if called with initial and nothing to do
return
2018-06-26 20:41:28 +00:00
new_loss = float(current_price * (1 - abs(stoploss)))
2018-06-26 18:49:07 +00:00
# no stop loss assigned yet
2018-07-01 17:54:26 +00:00
if not self.stop_loss:
2019-09-11 20:32:08 +00:00
logger.debug(f"{self.pair} - Assigning new stoploss...")
2020-12-11 06:41:06 +00:00
self._set_new_stoploss(new_loss, stoploss)
2018-06-26 18:49:07 +00:00
self.initial_stop_loss = new_loss
2019-03-31 11:15:35 +00:00
self.initial_stop_loss_pct = -1 * abs(stoploss)
2018-06-26 18:49:07 +00:00
# evaluate if the stop loss needs to be updated
else:
if new_loss > self.stop_loss: # stop losses only walk up, never down!
2019-09-11 20:32:08 +00:00
logger.debug(f"{self.pair} - Adjusting stoploss...")
2020-12-11 06:41:06 +00:00
self._set_new_stoploss(new_loss, stoploss)
2018-06-26 18:49:07 +00:00
else:
2019-09-11 20:32:08 +00:00
logger.debug(f"{self.pair} - Keeping current stoploss...")
2018-06-26 18:49:07 +00:00
logger.debug(
2019-09-11 23:29:47 +00:00
f"{self.pair} - Stoploss adjusted. current_price={current_price:.8f}, "
f"open_rate={self.open_rate:.8f}, max_rate={self.max_rate:.8f}, "
f"initial_stop_loss={self.initial_stop_loss:.8f}, "
f"stop_loss={self.stop_loss:.8f}. "
2019-09-10 07:42:45 +00:00
f"Trailing stoploss saved us: "
2019-09-11 23:29:47 +00:00
f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f}.")
2018-06-26 18:49:07 +00:00
def update(self, order: Dict) -> None:
2017-06-08 18:01:01 +00:00
"""
Updates this entity with amount and actual open/close rates.
:param order: order retrieved by exchange.fetch_order()
:return: None
2017-06-08 18:01:01 +00:00
"""
2018-06-23 13:27:29 +00:00
order_type = order['type']
# Ignore open and cancelled orders
if order['status'] == 'open' or safe_value_fallback(order, 'average', 'price') is None:
return
logger.info('Updating trade (id=%s) ...', self.id)
2017-12-17 21:07:56 +00:00
if order_type in ('market', 'limit') and order['side'] == 'buy':
# Update open rate and actual amount
self.open_rate = float(safe_value_fallback(order, 'average', 'price'))
self.amount = float(safe_value_fallback(order, 'filled', 'amount'))
if self.is_open:
logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.')
self.open_order_id = None
self.recalc_trade_from_orders()
elif order_type in ('market', 'limit') and order['side'] == 'sell':
if self.is_open:
logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.')
self.close(safe_value_fallback(order, 'average', 'price'))
2020-11-25 20:39:12 +00:00
elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'):
2018-11-23 14:17:36 +00:00
self.stoploss_order_id = None
self.close_rate_requested = self.stop_loss
self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
if self.is_open:
logger.info(f'{order_type.upper()} is hit for {self}.')
self.close(safe_value_fallback(order, 'average', 'price'))
else:
2018-06-23 13:27:29 +00:00
raise ValueError(f'Unknown order type: {order_type}')
2021-06-13 09:17:44 +00:00
Trade.commit()
2017-06-08 18:01:01 +00:00
2020-12-07 15:07:00 +00:00
def close(self, rate: float, *, show_msg: bool = True) -> None:
"""
Sets close_rate to the given rate, calculates total profit
and marks trade as closed
"""
self.close_rate = rate
self.close_profit = self.calc_profit_ratio()
2020-03-22 10:16:09 +00:00
self.close_profit_abs = self.calc_profit()
self.close_date = self.close_date or datetime.utcnow()
self.is_open = False
self.sell_order_status = 'closed'
self.open_order_id = None
2020-11-16 19:21:32 +00:00
if show_msg:
logger.info(
'Marking %s as closed as the trade is fulfilled and found no open orders for it.',
self
)
2020-05-01 14:00:42 +00:00
def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float],
2020-05-01 18:34:58 +00:00
side: str) -> None:
2020-05-01 14:00:42 +00:00
"""
Update Fee parameters. Only acts once per side
"""
if side == 'buy' and self.fee_open_currency is None:
self.fee_open_cost = fee_cost
self.fee_open_currency = fee_currency
if fee_rate is not None:
self.fee_open = fee_rate
# Assume close-fee will fall into the same fee category and take an educated guess
self.fee_close = fee_rate
elif side == 'sell' and self.fee_close_currency is None:
self.fee_close_cost = fee_cost
self.fee_close_currency = fee_currency
if fee_rate is not None:
self.fee_close = fee_rate
2020-05-01 18:34:58 +00:00
def fee_updated(self, side: str) -> bool:
"""
Verify if this side (buy / sell) has already been updated
"""
if side == 'buy':
return self.fee_open_currency is not None
elif side == 'sell':
return self.fee_close_currency is not None
2020-05-01 18:02:38 +00:00
else:
return False
2020-08-13 13:39:29 +00:00
def update_order(self, order: Dict) -> None:
Order.update_orders(self.orders, order)
def get_exit_order_count(self) -> int:
"""
Get amount of failed exiting orders
assumes full exits.
"""
2021-11-06 12:10:41 +00:00
return len([o for o in self.orders if o.ft_order_side == 'sell'])
def _calc_open_trade_value(self) -> float:
2017-12-17 21:07:56 +00:00
"""
2019-12-17 06:09:56 +00:00
Calculate the open_rate including open_fee.
2018-11-01 12:05:57 +00:00
:return: Price in of the open trade incl. Fees
2017-12-17 21:07:56 +00:00
"""
2019-12-17 18:30:04 +00:00
buy_trade = Decimal(self.amount) * Decimal(self.open_rate)
2019-12-17 06:02:02 +00:00
fees = buy_trade * Decimal(self.fee_open)
2017-12-17 21:07:56 +00:00
return float(buy_trade + fees)
def recalc_open_trade_value(self) -> None:
"""
Recalculate open_trade_value.
Must be called whenever open_rate or fee_open is changed.
"""
self.open_trade_value = self._calc_open_trade_value()
def calc_close_trade_value(self, rate: Optional[float] = None,
2019-09-10 07:42:45 +00:00
fee: Optional[float] = None) -> float:
2017-12-17 21:07:56 +00:00
"""
2018-11-01 12:05:57 +00:00
Calculate the close_rate including fee
2017-12-17 21:07:56 +00:00
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
2017-12-17 21:07:56 +00:00
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
2017-12-17 21:07:56 +00:00
:return: Price in BTC of the open trade
"""
if rate is None and not self.close_rate:
return 0.0
sell_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) # type: ignore
fees = sell_trade * Decimal(fee or self.fee_close)
2017-12-17 21:07:56 +00:00
return float(sell_trade - fees)
2019-09-10 07:42:45 +00:00
def calc_profit(self, rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
2017-12-17 21:07:56 +00:00
"""
2018-11-01 12:05:57 +00:00
Calculate the absolute profit in stake currency between Close and Open trade
2017-12-17 21:07:56 +00:00
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
2017-12-17 21:07:56 +00:00
:param rate: close rate to compare with (optional).
If rate is not set self.close_rate will be used
2018-11-01 12:05:57 +00:00
:return: profit in stake currency as float
2017-12-17 21:07:56 +00:00
"""
close_trade_value = self.calc_close_trade_value(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
2017-12-17 21:07:56 +00:00
)
profit = close_trade_value - self.open_trade_value
2018-06-23 13:27:29 +00:00
return float(f"{profit:.8f}")
2017-12-17 21:07:56 +00:00
def calc_profit_ratio(self, rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculates the profit as ratio (including fee).
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
2018-03-17 21:12:21 +00:00
:param fee: fee to use on the close rate (optional).
:return: profit ratio as float
"""
close_trade_value = self.calc_close_trade_value(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
2017-12-17 21:07:56 +00:00
)
if self.open_trade_value == 0.0:
return 0.0
profit_ratio = (close_trade_value / self.open_trade_value) - 1
2020-02-28 09:36:39 +00:00
return float(f"{profit_ratio:.8f}")
def recalc_trade_from_orders(self):
# We need at least 2 orders for averaging amounts and rates.
if len(self.orders) < 2:
# Just in case, still recalc open trade value
self.recalc_open_trade_value()
return
total_amount = 0.0
total_stake = 0.0
for temp_order in self.orders:
2021-12-18 16:55:47 +00:00
if (temp_order.ft_is_open or
(temp_order.ft_order_side != 'buy') or
(temp_order.status not in NON_OPEN_EXCHANGE_STATES)):
continue
tmp_amount = temp_order.amount
if temp_order.filled is not None:
tmp_amount = temp_order.filled
if tmp_amount > 0.0 and temp_order.average is not None:
total_amount += tmp_amount
total_stake += temp_order.average * tmp_amount
if total_amount > 0:
self.open_rate = total_stake / total_amount
self.stake_amount = total_stake
self.amount = total_amount
self.fee_open_cost = self.fee_open * self.stake_amount
self.recalc_open_trade_value()
2021-12-10 21:17:12 +00:00
if self.stop_loss_pct is not None and self.open_rate is not None:
self.adjust_stop_loss(self.open_rate, self.stop_loss_pct)
2021-12-09 18:03:41 +00:00
def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]:
"""
2020-09-11 04:59:07 +00:00
Finds latest order for this orderside and status
:param order_side: Side of the order (either 'buy' or 'sell')
:param is_open: Only search for open orders?
2020-09-11 04:59:07 +00:00
:return: latest Order object if it exists, else None
"""
2020-08-22 13:48:00 +00:00
orders = [o for o in self.orders if o.side == order_side]
if is_open is not None:
orders = [o for o in orders if o.ft_is_open == is_open]
if len(orders) > 0:
return orders[-1]
else:
return None
def select_filled_orders(self, order_side: str) -> List['Order']:
"""
Finds filled orders for this orderside.
:param order_side: Side of the order (either 'buy' or 'sell')
:return: array of Order objects
"""
return [o for o in self.orders if o.ft_order_side == order_side and
2022-01-14 18:25:29 +00:00
o.ft_is_open is False and
(o.filled or 0) > 0 and
o.status in NON_OPEN_EXCHANGE_STATES]
2022-01-13 18:31:03 +00:00
@property
def nr_of_successful_buys(self) -> int:
"""
Helper function to count the number of buy orders that have been filled.
:return: int count of buy orders that have been filled for this trade.
"""
return len(self.select_filled_orders('buy'))
2022-01-13 18:31:03 +00:00
@property
def nr_of_successful_sells(self) -> int:
"""
Helper function to count the number of sell orders that have been filled.
:return: int count of sell orders that have been filled for this trade.
"""
return len(self.select_filled_orders('sell'))
2020-11-16 19:09:34 +00:00
@staticmethod
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
open_date: datetime = None, close_date: datetime = None,
) -> List['LocalTrade']:
2020-11-16 19:09:34 +00:00
"""
Helper function to query Trades.
Returns a List of trades, filtered on the parameters given.
In live mode, converts the filter to a database query and returns all rows
In Backtest mode, uses filters on Trade.trades to get the result.
:return: unsorted List[Trade]
"""
# Offline mode - without database
2021-03-13 09:16:32 +00:00
if is_open is not None:
if is_open:
sel_trades = LocalTrade.trades_open
else:
sel_trades = LocalTrade.trades
else:
# Not used during backtesting, but might be used by a strategy
sel_trades = list(LocalTrade.trades + LocalTrade.trades_open)
2021-03-13 09:16:32 +00:00
if pair:
sel_trades = [trade for trade in sel_trades if trade.pair == pair]
if open_date:
sel_trades = [trade for trade in sel_trades if trade.open_date > open_date]
if close_date:
sel_trades = [trade for trade in sel_trades if trade.close_date
and trade.close_date > close_date]
2021-03-13 09:16:32 +00:00
return sel_trades
2020-11-16 19:09:34 +00:00
2021-03-13 09:16:32 +00:00
@staticmethod
def close_bt_trade(trade):
LocalTrade.trades_open.remove(trade)
LocalTrade.trades.append(trade)
LocalTrade.total_profit += trade.close_profit_abs
@staticmethod
def add_bt_trade(trade):
if trade.is_open:
LocalTrade.trades_open.append(trade)
else:
LocalTrade.trades.append(trade)
2019-10-29 14:01:10 +00:00
@staticmethod
def get_open_trades() -> List[Any]:
"""
Query trades from persistence layer
"""
return Trade.get_trades_proxy(is_open=True)
2019-10-29 10:15:33 +00:00
@staticmethod
def stoploss_reinitialization(desired_stoploss):
"""
Adjust initial Stoploss to desired stoploss for all open trades.
"""
for trade in Trade.get_open_trades():
logger.info("Found open trade: %s", trade)
# skip case if trailing-stop changed the stoploss already.
if (trade.stop_loss == trade.initial_stop_loss
2021-07-21 13:05:35 +00:00
and trade.initial_stop_loss_pct != desired_stoploss):
# Stoploss value got changed
2019-09-10 07:42:45 +00:00
logger.info(f"Stoploss for {trade} needs adjustment...")
# Force reset of stoploss
trade.stop_loss = None
trade.adjust_stop_loss(trade.open_rate, desired_stoploss)
2019-09-10 07:42:45 +00:00
logger.info(f"New stoploss: {trade.stop_loss}.")
2020-10-17 09:28:34 +00:00
class Trade(_DECL_BASE, LocalTrade):
"""
Trade database model.
Also handles updating and querying trades
2021-02-27 18:57:42 +00:00
Note: Fields must be aligned with LocalTrade class
"""
__tablename__ = 'trades'
use_db: bool = True
id = Column(Integer, primary_key=True)
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
exchange = Column(String(25), nullable=False)
pair = Column(String(25), nullable=False, index=True)
is_open = Column(Boolean, nullable=False, default=True, index=True)
fee_open = Column(Float, nullable=False, default=0.0)
fee_open_cost = Column(Float, nullable=True)
fee_open_currency = Column(String(25), nullable=True)
fee_close = Column(Float, nullable=False, default=0.0)
fee_close_cost = Column(Float, nullable=True)
fee_close_currency = Column(String(25), nullable=True)
open_rate = Column(Float)
open_rate_requested = Column(Float)
# open_trade_value - calculated via _calc_open_trade_value
open_trade_value = Column(Float)
close_rate = Column(Float)
close_rate_requested = Column(Float)
close_profit = Column(Float)
close_profit_abs = Column(Float)
stake_amount = Column(Float, nullable=False)
amount = Column(Float)
amount_requested = Column(Float)
open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
close_date = Column(DateTime)
open_order_id = Column(String(255))
# absolute value of the stop loss
stop_loss = Column(Float, nullable=True, default=0.0)
# percentage value of the stop loss
stop_loss_pct = Column(Float, nullable=True)
# absolute value of the initial stop loss
initial_stop_loss = Column(Float, nullable=True, default=0.0)
# percentage value of the initial stop loss
initial_stop_loss_pct = Column(Float, nullable=True)
# stoploss order id which is on exchange
stoploss_order_id = Column(String(255), nullable=True, index=True)
# last update time of the stoploss order on exchange
stoploss_last_update = Column(DateTime, nullable=True)
# absolute value of the highest reached price
max_rate = Column(Float, nullable=True, default=0.0)
# Lowest price reached
min_rate = Column(Float, nullable=True)
sell_reason = Column(String(100), nullable=True)
sell_order_status = Column(String(100), nullable=True)
strategy = Column(String(100), nullable=True)
2021-07-21 13:05:35 +00:00
buy_tag = Column(String(100), nullable=True)
timeframe = Column(Integer, nullable=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.recalc_open_trade_value()
def delete(self) -> None:
for order in self.orders:
Order.query.session.delete(order)
Trade.query.session.delete(self)
2021-04-15 05:57:52 +00:00
Trade.commit()
@staticmethod
def commit():
2021-04-13 17:52:33 +00:00
Trade.query.session.commit()
@staticmethod
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
open_date: datetime = None, close_date: datetime = None,
) -> List['LocalTrade']:
"""
Helper function to query Trades.j
Returns a List of trades, filtered on the parameters given.
In live mode, converts the filter to a database query and returns all rows
In Backtest mode, uses filters on Trade.trades to get the result.
:return: unsorted List[Trade]
"""
if Trade.use_db:
trade_filter = []
if pair:
trade_filter.append(Trade.pair == pair)
if open_date:
trade_filter.append(Trade.open_date > open_date)
if close_date:
trade_filter.append(Trade.close_date > close_date)
if is_open is not None:
trade_filter.append(Trade.is_open.is_(is_open))
return Trade.get_trades(trade_filter).all()
else:
return LocalTrade.get_trades_proxy(
pair=pair, is_open=is_open,
open_date=open_date,
close_date=close_date
)
@staticmethod
def get_trades(trade_filter=None) -> Query:
"""
Helper function to query Trades using filters.
NOTE: Not supported in Backtesting.
:param trade_filter: Optional filter to apply to trades
Can be either a Filter object, or a List of filters
e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
e.g. `(trade_filter=Trade.id == trade_id)`
:return: unsorted query object
"""
if not Trade.use_db:
raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.')
if trade_filter is not None:
if not isinstance(trade_filter, list):
trade_filter = [trade_filter]
return Trade.query.filter(*trade_filter)
else:
return Trade.query
@staticmethod
def get_open_order_trades() -> List['Trade']:
"""
Returns all open trades
NOTE: Not supported in Backtesting.
"""
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def get_open_trades_without_assigned_fees():
"""
Returns all open trades which don't have open fees set correctly
NOTE: Not supported in Backtesting.
"""
return Trade.get_trades([Trade.fee_open_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(True),
]).all()
@staticmethod
def get_sold_trades_without_assigned_fees():
"""
Returns all closed trades which don't have fees set correctly
NOTE: Not supported in Backtesting.
"""
return Trade.get_trades([Trade.fee_close_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(False),
]).all()
@staticmethod
def get_total_closed_profit() -> float:
"""
Retrieves total realized profit
"""
if Trade.use_db:
total_profit = Trade.query.with_entities(
func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar()
else:
total_profit = sum(
t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False))
return total_profit or 0
@staticmethod
def total_open_trades_stakes() -> float:
"""
Calculates total invested amount in open trades
in stake currency
"""
if Trade.use_db:
total_open_stake_amount = Trade.query.with_entities(
func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
else:
total_open_stake_amount = sum(
t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True))
return total_open_stake_amount or 0
@staticmethod
def get_overall_performance(minutes=None) -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, including profit and trade count
NOTE: Not supported in Backtesting.
"""
filters = [Trade.is_open.is_(False)]
if minutes:
start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes)
filters.append(Trade.close_date >= start_date)
pair_rates = Trade.query.with_entities(
Trade.pair,
func.sum(Trade.close_profit).label('profit_sum'),
2021-05-15 17:39:46 +00:00
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
func.count(Trade.pair).label('count')
).filter(*filters)\
.group_by(Trade.pair) \
2021-05-15 17:39:46 +00:00
.order_by(desc('profit_sum_abs')) \
.all()
2021-10-31 09:42:42 +00:00
return [
{
'pair': pair,
2021-10-31 09:42:42 +00:00
'profit_ratio': profit,
'profit': round(profit * 100, 2), # Compatibility mode
'profit_pct': round(profit * 100, 2),
2021-05-15 17:39:46 +00:00
'profit_abs': profit_abs,
'count': count
}
2021-05-15 17:39:46 +00:00
for pair, profit, profit_abs, count in pair_rates
]
@staticmethod
2021-10-24 13:18:29 +00:00
def get_buy_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, based on buy tag performance
Can either be average for all pairs or a specific pair provided
NOTE: Not supported in Backtesting.
"""
filters = [Trade.is_open.is_(False)]
if(pair is not None):
filters.append(Trade.pair == pair)
buy_tag_perf = Trade.query.with_entities(
Trade.buy_tag,
func.sum(Trade.close_profit).label('profit_sum'),
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
func.count(Trade.pair).label('count')
).filter(*filters)\
.group_by(Trade.buy_tag) \
.order_by(desc('profit_sum_abs')) \
.all()
2021-10-31 09:42:42 +00:00
return [
{
'buy_tag': buy_tag if buy_tag is not None else "Other",
2021-10-31 09:42:42 +00:00
'profit_ratio': profit,
'profit_pct': round(profit * 100, 2),
'profit_abs': profit_abs,
'count': count
}
for buy_tag, profit, profit_abs, count in buy_tag_perf
]
@staticmethod
2021-10-24 13:18:29 +00:00
def get_sell_reason_performance(pair: Optional[str]) -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, based on sell reason performance
Can either be average for all pairs or a specific pair provided
NOTE: Not supported in Backtesting.
"""
filters = [Trade.is_open.is_(False)]
if(pair is not None):
filters.append(Trade.pair == pair)
sell_tag_perf = Trade.query.with_entities(
Trade.sell_reason,
func.sum(Trade.close_profit).label('profit_sum'),
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
func.count(Trade.pair).label('count')
).filter(*filters)\
.group_by(Trade.sell_reason) \
.order_by(desc('profit_sum_abs')) \
.all()
2021-10-31 09:42:42 +00:00
return [
{
'sell_reason': sell_reason if sell_reason is not None else "Other",
2021-10-31 09:42:42 +00:00
'profit_ratio': profit,
'profit_pct': round(profit * 100, 2),
'profit_abs': profit_abs,
'count': count
}
for sell_reason, profit, profit_abs, count in sell_tag_perf
]
@staticmethod
2021-10-24 13:18:29 +00:00
def get_mix_tag_performance(pair: Optional[str]) -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, based on buy_tag + sell_reason performance
Can either be average for all pairs or a specific pair provided
NOTE: Not supported in Backtesting.
"""
filters = [Trade.is_open.is_(False)]
if(pair is not None):
filters.append(Trade.pair == pair)
mix_tag_perf = Trade.query.with_entities(
Trade.id,
Trade.buy_tag,
Trade.sell_reason,
func.sum(Trade.close_profit).label('profit_sum'),
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
func.count(Trade.pair).label('count')
).filter(*filters)\
.group_by(Trade.id) \
.order_by(desc('profit_sum_abs')) \
.all()
2021-10-20 17:13:34 +00:00
return_list: List[Dict] = []
for id, buy_tag, sell_reason, profit, profit_abs, count in mix_tag_perf:
buy_tag = buy_tag if buy_tag is not None else "Other"
sell_reason = sell_reason if sell_reason is not None else "Other"
if(sell_reason is not None and buy_tag is not None):
mix_tag = buy_tag + " " + sell_reason
i = 0
if not any(item["mix_tag"] == mix_tag for item in return_list):
return_list.append({'mix_tag': mix_tag,
'profit': profit,
2021-11-11 12:55:55 +00:00
'profit_pct': round(profit * 100, 2),
'profit_abs': profit_abs,
'count': count})
else:
while i < len(return_list):
if return_list[i]["mix_tag"] == mix_tag:
return_list[i] = {
'mix_tag': mix_tag,
'profit': profit + return_list[i]["profit"],
2021-11-11 12:55:55 +00:00
'profit_pct': round(profit + return_list[i]["profit"] * 100, 2),
'profit_abs': profit_abs + return_list[i]["profit_abs"],
'count': 1 + return_list[i]["count"]}
i += 1
return return_list
@staticmethod
def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)):
"""
Get best pair with closed trade.
NOTE: Not supported in Backtesting.
:returns: Tuple containing (pair, profit_sum)
"""
best_pair = Trade.query.with_entities(
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date)) \
.group_by(Trade.pair) \
.order_by(desc('profit_sum')).first()
return best_pair
2020-10-17 09:28:34 +00:00
class PairLock(_DECL_BASE):
"""
Pair Locks database model.
"""
__tablename__ = 'pairlocks'
2020-10-17 09:28:34 +00:00
id = Column(Integer, primary_key=True)
pair = Column(String(25), nullable=False, index=True)
reason = Column(String(255), nullable=True)
2020-10-17 09:28:34 +00:00
# Time the pair was locked (start time)
lock_time = Column(DateTime, nullable=False)
# Time until the pair is locked (end time)
lock_end_time = Column(DateTime, nullable=False, index=True)
2020-10-17 09:28:34 +00:00
2020-10-17 13:15:35 +00:00
active = Column(Boolean, nullable=False, default=True, index=True)
2020-10-17 09:28:34 +00:00
def __repr__(self):
2020-10-17 09:40:01 +00:00
lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
2020-10-17 09:28:34 +00:00
return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, '
f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
2020-10-17 09:28:34 +00:00
2020-10-17 09:40:01 +00:00
@staticmethod
2020-10-27 09:08:24 +00:00
def query_pair_locks(pair: Optional[str], now: datetime) -> Query:
2020-10-17 09:28:34 +00:00
"""
2020-10-24 11:55:54 +00:00
Get all currently active locks for this pair
2020-10-17 13:15:35 +00:00
:param pair: Pair to check for. Returns all current locks if pair is empty
:param now: Datetime object (generated via datetime.now(timezone.utc)).
2020-10-17 09:28:34 +00:00
"""
filters = [PairLock.lock_end_time > now,
2020-10-17 13:15:35 +00:00
# Only active locks
PairLock.active.is_(True), ]
if pair:
filters.append(PairLock.pair == pair)
2020-10-17 09:28:34 +00:00
return PairLock.query.filter(
2020-10-17 13:15:35 +00:00
*filters
2020-10-25 09:54:30 +00:00
)
2020-10-17 13:15:35 +00:00
def to_json(self) -> Dict[str, Any]:
return {
2021-03-01 06:51:33 +00:00
'id': self.id,
2020-10-17 13:15:35 +00:00
'pair': self.pair,
2020-10-17 18:32:23 +00:00
'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT),
2020-10-17 15:58:07 +00:00
'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000),
2020-10-17 18:32:23 +00:00
'lock_end_time': self.lock_end_time.strftime(DATETIME_PRINT_FORMAT),
2020-10-17 15:58:07 +00:00
'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc
).timestamp() * 1000),
2020-10-17 13:15:35 +00:00
'reason': self.reason,
'active': self.active,
}