stable/freqtrade/persistence.py

444 lines
18 KiB
Python
Raw Normal View History

2018-03-02 15:22:00 +00:00
"""
This module contains the class to persist trades into SQLite
"""
import logging
2017-05-12 17:11:56 +00:00
from datetime import datetime
from decimal import Decimal
from typing import Any, Dict, List, Optional
2017-05-12 17:11:56 +00:00
import arrow
2018-01-10 07:51:36 +00:00
from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
2018-07-04 07:31:35 +00:00
create_engine, inspect)
2018-06-07 19:35:57 +00:00
from sqlalchemy.exc import NoSuchModuleError
2017-05-12 17:11:56 +00:00
from sqlalchemy.ext.declarative import declarative_base
2017-09-03 06:50:48 +00:00
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from sqlalchemy import func
2017-11-09 22:45:22 +00:00
from sqlalchemy.pool import StaticPool
2017-05-12 17:11:56 +00:00
2018-06-07 19:35:57 +00:00
from freqtrade import OperationalException
logger = logging.getLogger(__name__)
2018-05-31 19:10:15 +00:00
_DECL_BASE: Any = declarative_base()
2018-06-23 13:27:29 +00:00
_SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls'
2017-05-12 17:11:56 +00:00
def init(db_url: str, clean_open_orders: bool = False) -> None:
"""
Initializes this module with the given config,
registers all known command handlers
and starts polling for message updates
:param db_url: Database to use
:param clean_open_orders: Remove open orders from the database.
Useful for dry-run or if all orders have been reset on the exchange.
:return: None
"""
kwargs = {}
# Take care of thread ownership if in-memory db
if db_url == 'sqlite://':
kwargs.update({
'connect_args': {'check_same_thread': False},
'poolclass': StaticPool,
'echo': False,
})
2018-06-07 19:35:57 +00:00
try:
engine = create_engine(db_url, **kwargs)
except NoSuchModuleError:
2018-06-23 13:27:29 +00:00
raise OperationalException(f'Given value for db_url: \'{db_url}\' '
f'is no valid database URL! (See {_SQL_DOCS_URL})')
2018-06-07 19:35:57 +00:00
session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.session = session()
Trade.query = session.query_property()
2017-11-07 19:13:36 +00:00
_DECL_BASE.metadata.create_all(engine)
2018-05-06 07:09:53 +00:00
check_migrate(engine)
# Clean dry_run DB if the db is not in-memory
if clean_open_orders and db_url != 'sqlite://':
clean_dry_run_db()
2018-05-06 07:09:53 +00:00
def has_column(columns, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
2018-05-06 07:09:53 +00:00
def check_migrate(engine) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = inspector.get_table_names()
table_back_name = 'trades_bak'
2018-07-01 18:03:07 +00:00
for i, table_back_name in enumerate(tabs):
table_back_name = f'trades_bak{i}'
logger.debug(f'trying {table_back_name}')
2018-05-06 07:09:53 +00:00
# Check for latest column
2019-03-28 20:18:26 +00:00
if not has_column(cols, 'stop_loss_pct'):
logger.info(f'Running database migration - backup available as {table_back_name}')
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_close = get_column_def(cols, 'fee_close', 'fee')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
2019-03-29 07:08:29 +00:00
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
2019-03-29 07:08:29 +00:00
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
2019-03-16 19:04:39 +00:00
min_rate = get_column_def(cols, 'min_rate', 'null')
2018-07-11 17:57:01 +00:00
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
ticker_interval = get_column_def(cols, 'ticker_interval', 'null')
2018-05-06 07:09:53 +00:00
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
2018-05-06 07:09:53 +00:00
# let SQLAlchemy create the schema as required
_DECL_BASE.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
2018-05-06 07:09:53 +00:00
(id, exchange, pair, is_open, fee_open, fee_close, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, open_date, close_date, open_order_id,
2019-03-28 20:18:26 +00:00
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, strategy,
ticker_interval
)
2018-05-12 11:39:16 +00:00
select id, lower(exchange),
2018-05-12 11:37:42 +00:00
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_close} fee_close,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, open_date, close_date, open_order_id,
2019-03-28 20:18:26 +00:00
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{strategy} strategy, {ticker_interval} ticker_interval
from {table_back_name}
2018-05-06 07:09:53 +00:00
""")
2018-05-12 08:04:41 +00:00
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
2018-05-06 07:09:53 +00:00
def cleanup() -> None:
"""
Flushes all pending operations to disk.
:return: None
"""
Trade.session.flush()
def clean_dry_run_db() -> None:
"""
Remove open_order_id from a Dry_run DB
:return: None
"""
for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all():
# Check we are updating only a dry_run order not a prod one
if 'dry_run' in trade.open_order_id:
trade.open_order_id = None
2017-11-07 19:13:36 +00:00
class Trade(_DECL_BASE):
2018-03-02 15:22:00 +00:00
"""
Class used to define a trade structure
"""
2017-05-12 17:11:56 +00:00
__tablename__ = 'trades'
id = Column(Integer, primary_key=True)
2017-10-06 10:22:04 +00:00
exchange = Column(String, nullable=False)
2018-08-12 07:30:12 +00:00
pair = Column(String, nullable=False, index=True)
2018-08-02 23:39:13 +00:00
is_open = Column(Boolean, nullable=False, default=True, index=True)
fee_open = Column(Float, nullable=False, default=0.0)
fee_close = Column(Float, nullable=False, default=0.0)
open_rate = Column(Float)
open_rate_requested = Column(Float)
2017-05-12 17:11:56 +00:00
close_rate = Column(Float)
close_rate_requested = Column(Float)
2017-05-12 17:11:56 +00:00
close_profit = Column(Float)
stake_amount = Column(Float, nullable=False)
amount = Column(Float)
2017-05-12 17:11:56 +00:00
open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
close_date = Column(DateTime)
2017-05-14 12:14:16 +00:00
open_order_id = Column(String)
2018-06-26 18:49:07 +00:00
# absolute value of the stop loss
stop_loss = Column(Float, nullable=True, default=0.0)
2019-03-28 20:18:26 +00:00
# percentage value of the stop loss
2019-03-29 07:08:29 +00:00
stop_loss_pct = Column(Float, nullable=True)
2018-06-26 18:49:07 +00:00
# absolute value of the initial stop loss
initial_stop_loss = Column(Float, nullable=True, default=0.0)
2019-03-28 20:18:26 +00:00
# percentage value of the initial stop loss
2019-03-29 07:08:29 +00:00
initial_stop_loss_pct = Column(Float, nullable=True)
# stoploss order id which is on exchange
2018-11-23 14:17:36 +00:00
stoploss_order_id = Column(String, nullable=True, index=True)
# last update time of the stoploss order on exchange
stoploss_last_update = Column(DateTime, nullable=True)
# absolute value of the highest reached price
2018-06-26 18:49:07 +00:00
max_rate = Column(Float, nullable=True, default=0.0)
# Lowest price reached
2019-03-16 19:04:39 +00:00
min_rate = Column(Float, nullable=True)
2018-07-11 17:57:01 +00:00
sell_reason = Column(String, nullable=True)
strategy = Column(String, nullable=True)
ticker_interval = Column(Integer, nullable=True)
2017-05-12 17:11:56 +00:00
def __repr__(self):
2018-06-23 13:27:29 +00:00
open_since = arrow.get(self.open_date).humanize() if self.is_open else 'closed'
return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
f'open_rate={self.open_rate:.8f}, open_since={open_since})')
def to_json(self) -> Dict[str, Any]:
return {
'trade_id': self.id,
'pair': self.pair,
2019-05-06 04:55:12 +00:00
'open_date_hum': arrow.get(self.open_date).humanize(),
'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"),
'close_date_hum': (arrow.get(self.close_date).humanize()
if self.close_date else None),
'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S")
if self.close_date else None),
'open_rate': self.open_rate,
'close_rate': self.close_rate,
'amount': round(self.amount, 8),
'stake_amount': round(self.stake_amount, 8),
'stop_loss': self.stop_loss,
'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None,
'initial_stop_loss': self.initial_stop_loss,
2019-05-06 04:55:12 +00:00
'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100
if self.initial_stop_loss_pct else None),
}
2019-03-16 19:06:15 +00:00
def adjust_min_max_rates(self, current_price: float):
"""
Adjust the max_rate and min_rate.
"""
self.max_rate = max(current_price, self.max_rate or self.open_rate)
self.min_rate = min(current_price, self.min_rate or self.open_rate)
2018-07-01 17:54:26 +00:00
def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False):
2019-03-17 12:18:29 +00:00
"""
This adjusts the stop loss to it's most recently observed setting
:param current_price: Current rate the asset is traded
:param stoploss: Stoploss as factor (sample -0.05 -> -5% below current price).
:param initial: Called to initiate stop_loss.
Skips everything if self.stop_loss is already set.
"""
2018-06-26 18:49:07 +00:00
2018-06-27 04:38:49 +00:00
if initial and not (self.stop_loss is None or self.stop_loss == 0):
# Don't modify if called with initial and nothing to do
return
2018-06-26 20:41:28 +00:00
new_loss = float(current_price * (1 - abs(stoploss)))
2018-06-26 18:49:07 +00:00
# no stop loss assigned yet
2018-07-01 17:54:26 +00:00
if not self.stop_loss:
2018-06-26 18:49:07 +00:00
logger.debug("assigning new stop loss")
self.stop_loss = new_loss
2019-03-31 11:15:35 +00:00
self.stop_loss_pct = -1 * abs(stoploss)
2018-06-26 18:49:07 +00:00
self.initial_stop_loss = new_loss
2019-03-31 11:15:35 +00:00
self.initial_stop_loss_pct = -1 * abs(stoploss)
2019-01-15 10:04:32 +00:00
self.stoploss_last_update = datetime.utcnow()
2018-06-26 18:49:07 +00:00
# evaluate if the stop loss needs to be updated
else:
if new_loss > self.stop_loss: # stop losses only walk up, never down!
self.stop_loss = new_loss
2019-03-31 11:15:35 +00:00
self.stop_loss_pct = -1 * abs(stoploss)
2019-01-15 10:04:32 +00:00
self.stoploss_last_update = datetime.utcnow()
2018-06-26 18:49:07 +00:00
logger.debug("adjusted stop loss")
else:
logger.debug("keeping current stop loss")
logger.debug(
2018-06-26 20:41:28 +00:00
f"{self.pair} - current price {current_price:.8f}, "
f"bought at {self.open_rate:.8f} and calculated "
f"stop loss is at: {self.initial_stop_loss:.8f} initial "
f"stop at {self.stop_loss:.8f}. "
f"trailing stop loss saved us: "
f"{float(self.stop_loss) - float(self.initial_stop_loss):.8f} "
f"and max observed rate was {self.max_rate:.8f}")
2018-06-26 18:49:07 +00:00
def update(self, order: Dict) -> None:
2017-06-08 18:01:01 +00:00
"""
Updates this entity with amount and actual open/close rates.
:param order: order retrieved by exchange.get_order()
:return: None
2017-06-08 18:01:01 +00:00
"""
2018-06-23 13:27:29 +00:00
order_type = order['type']
# Ignore open and cancelled orders
if order['status'] == 'open' or order['price'] is None:
return
logger.info('Updating trade (id=%s) ...', self.id)
2017-12-17 21:07:56 +00:00
if order_type in ('market', 'limit') and order['side'] == 'buy':
# Update open rate and actual amount
self.open_rate = Decimal(order['price'])
2017-12-17 21:07:56 +00:00
self.amount = Decimal(order['amount'])
logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
self.open_order_id = None
elif order_type in ('market', 'limit') and order['side'] == 'sell':
self.close(order['price'])
logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self)
2018-11-23 14:17:36 +00:00
elif order_type == 'stop_loss_limit':
self.stoploss_order_id = None
self.close_rate_requested = self.stop_loss
logger.info('STOP_LOSS_LIMIT is hit for %s.', self)
self.close(order['average'])
else:
2018-06-23 13:27:29 +00:00
raise ValueError(f'Unknown order type: {order_type}')
2017-12-27 10:41:11 +00:00
cleanup()
2017-06-08 18:01:01 +00:00
def close(self, rate: float) -> None:
"""
Sets close_rate to the given rate, calculates total profit
and marks trade as closed
"""
2017-12-17 21:07:56 +00:00
self.close_rate = Decimal(rate)
self.close_profit = self.calc_profit_percent()
self.close_date = datetime.utcnow()
self.is_open = False
self.open_order_id = None
logger.info(
'Marking %s as closed as the trade is fulfilled and found no open orders for it.',
self
)
def calc_open_trade_price(
self,
fee: Optional[float] = None) -> float:
2017-12-17 21:07:56 +00:00
"""
2018-11-01 12:05:57 +00:00
Calculate the open_rate including fee.
2017-12-17 21:07:56 +00:00
:param fee: fee to use on the open rate (optional).
If rate is not set self.fee will be used
2018-11-01 12:05:57 +00:00
:return: Price in of the open trade incl. Fees
2017-12-17 21:07:56 +00:00
"""
buy_trade = (Decimal(self.amount) * Decimal(self.open_rate))
fees = buy_trade * Decimal(fee or self.fee_open)
2017-12-17 21:07:56 +00:00
return float(buy_trade + fees)
def calc_close_trade_price(
self,
rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
2017-12-17 21:07:56 +00:00
"""
2018-11-01 12:05:57 +00:00
Calculate the close_rate including fee
2017-12-17 21:07:56 +00:00
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
:return: Price in BTC of the open trade
"""
if rate is None and not self.close_rate:
return 0.0
sell_trade = (Decimal(self.amount) * Decimal(rate or self.close_rate))
fees = sell_trade * Decimal(fee or self.fee_close)
2017-12-17 21:07:56 +00:00
return float(sell_trade - fees)
def calc_profit(
self,
rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
2017-12-17 21:07:56 +00:00
"""
2018-11-01 12:05:57 +00:00
Calculate the absolute profit in stake currency between Close and Open trade
2017-12-17 21:07:56 +00:00
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
:param rate: close rate to compare with (optional).
If rate is not set self.close_rate will be used
2018-11-01 12:05:57 +00:00
:return: profit in stake currency as float
2017-12-17 21:07:56 +00:00
"""
open_trade_price = self.calc_open_trade_price()
close_trade_price = self.calc_close_trade_price(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
2017-12-17 21:07:56 +00:00
)
2018-06-23 13:27:29 +00:00
profit = close_trade_price - open_trade_price
return float(f"{profit:.8f}")
2017-12-17 21:07:56 +00:00
def calc_profit_percent(
self,
rate: Optional[float] = None,
fee: Optional[float] = None) -> float:
"""
Calculates the profit in percentage (including fee).
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
2018-03-17 21:12:21 +00:00
:param fee: fee to use on the close rate (optional).
:return: profit in percentage as float
"""
2017-12-17 21:07:56 +00:00
open_trade_price = self.calc_open_trade_price()
close_trade_price = self.calc_close_trade_price(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
2017-12-17 21:07:56 +00:00
)
2018-06-23 13:27:29 +00:00
profit_percent = (close_trade_price / open_trade_price) - 1
return float(f"{profit_percent:.8f}")
2018-12-03 18:46:22 +00:00
@staticmethod
2018-12-03 18:55:37 +00:00
def total_open_trades_stakes() -> float:
"""
Calculates total invested amount in open trades
in stake currency
"""
total_open_stake_amount = Trade.session.query(func.sum(Trade.stake_amount))\
.filter(Trade.is_open.is_(True))\
.scalar()
return total_open_stake_amount or 0
@staticmethod
def get_open_trades() -> List[Any]:
"""
Query trades from persistence layer
"""
return Trade.query.filter(Trade.is_open.is_(True)).all()
@staticmethod
def stoploss_reinitialization(desired_stoploss):
"""
Adjust initial Stoploss to desired stoploss for all open trades.
"""
for trade in Trade.get_open_trades():
logger.info("Found open trade: %s", trade)
# skip case if trailing-stop changed the stoploss already.
if (trade.stop_loss == trade.initial_stop_loss
and trade.initial_stop_loss_pct != desired_stoploss):
# Stoploss value got changed
logger.info(f"Stoploss for {trade} needs adjustment.")
# Force reset of stoploss
trade.stop_loss = None
trade.adjust_stop_loss(trade.open_rate, desired_stoploss)
logger.info(f"new stoploss: {trade.stop_loss}, ")