adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3682 from freqtrade/db_keep_orders

Browse Source 
Keep order history in the database
This commit is contained in:
Matthias
2020-09-19 17:12:14 +02:00
committed by GitHub
parent 5daaed1449 1f086e1466
commit a559611c15
25 changed files with 1504 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/data/history/history_utils.py
View File

@@ -136,7 +136,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
start = None
if timerange:
if timerange.starttype == 'date':
# TODO: convert to date for conversion
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
# Intentionally don't pass timerange in - since we need to load the full dataset.

7
freqtrade/exceptions.py
View File

@@ -51,6 +51,13 @@ class RetryableOrderError(InvalidOrderException):
"""
class InsufficientFundsError(InvalidOrderException):
"""
This error is used when there are not enough funds available on the exchange
to create an order.
"""
class TemporaryError(ExchangeError):
"""
Temporary network or exchange related error.

4
freqtrade/exchange/binance.py
View File

@@ -4,7 +4,7 @@ from typing import Dict
import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError,
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
InvalidOrderException, OperationalException,
TemporaryError)
from freqtrade.exchange import Exchange
@@ -80,7 +80,7 @@ class Binance(Exchange):
'stop price: %s. limit: %s', pair, stop_price, rate)
return order
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to sell amount {amount} at rate {rate}. '
f'Message: {e}') from e

4
freqtrade/exchange/common.py
View File

@@ -9,7 +9,11 @@ from freqtrade.exceptions import (DDosProtection, RetryableOrderError,
logger = logging.getLogger(__name__)
# Maximum default retry count.
# Functions are always called RETRY_COUNT + 1 times (for the original call)
API_RETRY_COUNT = 4
API_FETCH_ORDER_RETRY_COUNT = 5
BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "

27
freqtrade/exchange/exchange.py
View File

@@ -8,7 +8,6 @@ import logging
from copy import deepcopy
from datetime import datetime, timezone
from math import ceil
from random import randint
from typing import Any, Dict, List, Optional, Tuple
import arrow
@@ -21,9 +20,11 @@ from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.exceptions import (DDosProtection, ExchangeError,
InsufficientFundsError,
InvalidOrderException, OperationalException,
RetryableOrderError, TemporaryError)
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT,
BAD_EXCHANGES, retrier, retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
CcxtModuleType = Any
@@ -487,11 +488,11 @@ class Exchange:
def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{randint(0, 10**6)}'
order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount)
dry_order = {
"id": order_id,
'pair': pair,
'id': order_id,
'symbol': pair,
'price': rate,
'average': rate,
'amount': _amount,
@@ -500,6 +501,7 @@ class Exchange:
'side': side,
'remaining': _amount,
'datetime': arrow.utcnow().isoformat(),
'timestamp': int(arrow.utcnow().timestamp * 1000),
'status': "closed" if ordertype == "market" else "open",
'fee': None,
'info': {}
@@ -538,7 +540,7 @@ class Exchange:
amount, rate_for_order, params)
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
f'Tried to {side} amount {amount} at rate {rate}.'
f'Message: {e}') from e
@@ -1027,7 +1029,7 @@ class Exchange:
return order
@retrier(retries=5)
@retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:
try:
@@ -1056,6 +1058,17 @@ class Exchange:
# Assign method to fetch_stoploss_order to allow easy overriding in other classes
fetch_stoploss_order = fetch_order
def fetch_order_or_stoploss_order(self, order_id: str, pair: str,
stoploss_order: bool = False) -> Dict:
"""
Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
the stoploss_order parameter
:param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
"""
if stoploss_order:
return self.fetch_stoploss_order(order_id, pair)
return self.fetch_order(order_id, pair)
@retrier
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
"""

8
freqtrade/exchange/ftx.py
View File

@@ -4,11 +4,11 @@ from typing import Any, Dict
import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError,
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
InvalidOrderException, OperationalException,
TemporaryError)
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
logger = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ class Ftx(Exchange):
'stop price: %s.', pair, stop_price)
return order
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e
@@ -88,7 +88,7 @@ class Ftx(Exchange):
except ccxt.BaseError as e:
raise OperationalException(e) from e
@retrier(retries=5)
@retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:
try:

4
freqtrade/exchange/kraken.py
View File

@@ -4,7 +4,7 @@ from typing import Any, Dict
import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError,
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
InvalidOrderException, OperationalException,
TemporaryError)
from freqtrade.exchange import Exchange
@@ -98,7 +98,7 @@ class Kraken(Exchange):
'stop price: %s.', pair, stop_price)
return order
except ccxt.InsufficientFunds as e:
raise ExchangeError(
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e

196
freqtrade/freqtradebot.py
View File

@@ -17,12 +17,12 @@ from freqtrade.configuration import validate_config_consistency
from freqtrade.data.converter import order_book_to_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.exceptions import (DependencyException, ExchangeError,
from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
InvalidOrderException, PricingError)
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.misc import safe_value_fallback, safe_value_fallback2
from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Trade
from freqtrade.persistence import Order, Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State
@@ -134,6 +134,10 @@ class FreqtradeBot:
# Adjust stoploss if it was changed
Trade.stoploss_reinitialization(self.strategy.stoploss)
# Only update open orders on startup
# This will update the database after the initial migration
self.update_open_orders()
def process(self) -> None:
"""
Queries the persistence layer for open trades and handles them,
@@ -144,6 +148,8 @@ class FreqtradeBot:
# Check whether markets have to be reloaded and reload them when it's needed
self.exchange.reload_markets()
self.update_closed_trades_without_assigned_fees()
# Query trades from persistence layer
trades = Trade.get_open_trades()
@@ -227,6 +233,104 @@ class FreqtradeBot:
open_trades = len(Trade.get_open_trades())
return max(0, self.config['max_open_trades'] - open_trades)
def update_open_orders(self):
"""
Updates open orders based on order list kept in the database.
Mainly updates the state of orders - but may also close trades
"""
orders = Order.get_open_orders()
logger.info(f"Updating {len(orders)} open orders.")
for order in orders:
try:
fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
order.ft_order_side == 'stoploss')
self.update_trade_state(order.trade, order.order_id, fo)
except ExchangeError as e:
logger.warning(f"Error updating Order {order.order_id} due to {e}")
def update_closed_trades_without_assigned_fees(self):
"""
Update closed trades without close fees assigned.
Only acts when Orders are in the database, otherwise the last orderid is unknown.
"""
trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees()
for trade in trades:
if not trade.is_open and not trade.fee_updated('sell'):
# Get sell fee
order = trade.select_order('sell', False)
if order:
logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id,
stoploss_order=order.ft_order_side == 'stoploss')
trades: List[Trade] = Trade.get_open_trades_without_assigned_fees()
for trade in trades:
if trade.is_open and not trade.fee_updated('buy'):
order = trade.select_order('buy', False)
if order:
logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id)
def handle_insufficient_funds(self, trade: Trade):
"""
Determine if we ever opened a sell order for this trade.
If not, try update buy fees - otherwise "refind" the open order we obviously lost.
"""
sell_order = trade.select_order('sell', None)
if sell_order:
self.refind_lost_order(trade)
else:
self.reupdate_buy_order_fees(trade)
def reupdate_buy_order_fees(self, trade: Trade):
"""
Get buy order from database, and try to reupdate.
Handles trades where the initial fee-update did not work.
"""
logger.info(f"Trying to reupdate buy fees for {trade}")
order = trade.select_order('buy', False)
if order:
logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id)
def refind_lost_order(self, trade):
"""
Try refinding a lost trade.
Only used when InsufficientFunds appears on sell orders (stoploss or sell).
Tries to walk the stored orders and sell them off eventually.
"""
logger.info(f"Trying to refind lost order for {trade}")
for order in trade.orders:
logger.info(f"Trying to refind {order}")
fo = None
if not order.ft_is_open:
logger.debug(f"Order {order} is no longer open.")
continue
if order.ft_order_side == 'buy':
# Skip buy side - this is handled by reupdate_buy_order_fees
continue
try:
fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
order.ft_order_side == 'stoploss')
if order.ft_order_side == 'stoploss':
if fo and fo['status'] == 'open':
# Assume this as the open stoploss order
trade.stoploss_order_id = order.order_id
elif order.ft_order_side == 'sell':
if fo and fo['status'] == 'open':
# Assume this as the open order
trade.open_order_id = order.order_id
if fo:
logger.info(f"Found {order} for trade {trade}.jj")
self.update_trade_state(trade, order.order_id, fo,
stoploss_order=order.ft_order_side == 'stoploss')
except ExchangeError:
logger.warning(f"Error updating {order.order_id}.")
#
# BUY / enter positions / open trades logic and methods
#
@@ -528,6 +632,7 @@ class FreqtradeBot:
order = self.exchange.buy(pair=pair, ordertype=order_type,
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
order_id = order['id']
order_status = order.get('status', None)
@@ -556,7 +661,6 @@ class FreqtradeBot:
stake_amount = order['cost']
amount = safe_value_fallback(order, 'filled', 'amount')
buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
order_id = None
# in case of FOK the order may be filled immediately and fully
elif order_status == 'closed':
@@ -581,10 +685,11 @@ class FreqtradeBot:
strategy=self.strategy.get_strategy_name(),
timeframe=timeframe_to_minutes(self.config['timeframe'])
)
trade.orders.append(order_obj)
# Update fees if order is closed
if order_status == 'closed':
self.update_trade_state(trade, order)
self.update_trade_state(trade, order_id, order)
Trade.session.add(trade)
Trade.session.flush()
@@ -783,8 +888,16 @@ class FreqtradeBot:
stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount,
stop_price=stop_price,
order_types=self.strategy.order_types)
order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss')
trade.orders.append(order_obj)
trade.stoploss_order_id = str(stoploss_order['id'])
return True
except InsufficientFundsError as e:
logger.warning(f"Unable to place stoploss order {e}.")
# Try to figure out what went wrong
self.handle_insufficient_funds(trade)
except InvalidOrderException as e:
trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}')
@@ -814,10 +927,14 @@ class FreqtradeBot:
except InvalidOrderException as exception:
logger.warning('Unable to fetch stoploss order: %s', exception)
if stoploss_order:
trade.update_order(stoploss_order)
# We check if stoploss order is fulfilled
if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
self.update_trade_state(trade, stoploss_order, sl_order=True)
self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
stoploss_order=True)
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair,
timeframe_to_next_date(self.config['timeframe']))
@@ -869,10 +986,11 @@ class FreqtradeBot:
update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
# cancelling the current stoploss on exchange first
logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) '
'in order to add another one ...', order['id'])
logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
f"(orderid:{order['id']}) in order to add another one ...")
try:
self.exchange.cancel_stoploss_order(order['id'], trade.pair)
co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
trade.update_order(co)
except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {order['id']} "
f"for pair {trade.pair}")
@@ -927,7 +1045,7 @@ class FreqtradeBot:
logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
continue
fully_cancelled = self.update_trade_state(trade, order)
fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order)
if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and (
fully_cancelled
@@ -995,8 +1113,7 @@ class FreqtradeBot:
if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
logger.info('Buy order fully cancelled. Removing %s from database.', trade)
# if trade is not partially completed, just delete the trade
Trade.session.delete(trade)
Trade.session.flush()
trade.delete()
was_trade_fully_canceled = True
reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
else:
@@ -1007,7 +1124,7 @@ class FreqtradeBot:
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = filled_amount
trade.stake_amount = trade.amount * trade.open_rate
self.update_trade_state(trade, corder, trade.amount)
self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None
logger.info('Partial buy order timeout for %s.', trade)
@@ -1121,19 +1238,28 @@ class FreqtradeBot:
logger.info(f"User requested abortion of selling {trade.pair}")
return False
# Execute sell and update trade record
order = self.exchange.sell(pair=str(trade.pair),
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=time_in_force
)
try:
# Execute sell and update trade record
order = self.exchange.sell(pair=trade.pair,
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=time_in_force
)
except InsufficientFundsError as e:
logger.warning(f"Unable to place order {e}.")
# Try to figure out what went wrong
self.handle_insufficient_funds(trade)
return False
order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell')
trade.orders.append(order_obj)
trade.open_order_id = order['id']
trade.close_rate_requested = limit
trade.sell_reason = sell_reason.value
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') == 'closed':
self.update_trade_state(trade, order)
self.update_trade_state(trade, trade.open_order_id, order)
Trade.session.flush()
# Lock pair for one candle to prevent immediate rebuys
@@ -1230,30 +1356,35 @@ class FreqtradeBot:
# Common update trade state methods
#
def update_trade_state(self, trade: Trade, action_order: dict = None,
order_amount: float = None, sl_order: bool = False) -> bool:
def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None,
stoploss_order: bool = False) -> bool:
"""
Checks trades with open orders and updates the amount if necessary
Handles closing both buy and sell orders.
:param trade: Trade object of the trade we're analyzing
:param order_id: Order-id of the order we're analyzing
:param action_order: Already aquired order object
:return: True if order has been cancelled without being filled partially, False otherwise
"""
# Get order details for actual price per unit
if trade.open_order_id:
order_id = trade.open_order_id
elif trade.stoploss_order_id and sl_order:
order_id = trade.stoploss_order_id
else:
if not order_id:
logger.warning(f'Orderid for trade {trade} is empty.')
return False
# Update trade with order values
logger.info('Found open order for %s', trade)
try:
order = action_order or self.exchange.fetch_order(order_id, trade.pair)
order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id,
trade.pair,
stoploss_order)
except InvalidOrderException as exception:
logger.warning('Unable to fetch order %s: %s', order_id, exception)
return False
trade.update_order(order)
# Try update amount (binance-fix)
try:
new_amount = self.get_real_amount(trade, order, order_amount)
new_amount = self.get_real_amount(trade, order)
if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount,
abs_tol=constants.MATH_CLOSE_PREC):
order['amount'] = new_amount
@@ -1291,7 +1422,7 @@ class FreqtradeBot:
return real_amount
return amount
def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float:
def get_real_amount(self, trade: Trade, order: Dict) -> float:
"""
Detect and update trade fee.
Calls trade.update_fee() uppon correct detection.
@@ -1300,8 +1431,7 @@ class FreqtradeBot:
:return: identical (or new) amount for the trade
"""
# Init variables
if order_amount is None:
order_amount = safe_value_fallback(order, 'filled', 'amount')
order_amount = safe_value_fallback(order, 'filled', 'amount')
# Only run for closed orders
if trade.fee_updated(order.get('side', '')) or order['status'] == 'open':
return order_amount
@@ -1325,7 +1455,7 @@ class FreqtradeBot:
"""
fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
"""
trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair,
trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
trade.open_date)
if len(trades) == 0:

4
freqtrade/persistence/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# flake8: noqa: F401
from freqtrade.persistence.models import (Order, Trade, clean_dry_run_db,
cleanup, init)

149
freqtrade/persistence/migrations.py Normal file
View File

@@ -0,0 +1,149 @@
import logging
from typing import List
from sqlalchemy import inspect
logger = logging.getLogger(__name__)
def get_table_names_for_table(inspector, tabletype):
return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
def has_column(columns: List, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def get_backup_name(tabs, backup_prefix: str):
table_back_name = backup_prefix
for i, table_back_name in enumerate(tabs):
table_back_name = f'{backup_prefix}{i}'
logger.debug(f'trying {table_back_name}')
return table_back_name
def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
fee_close = get_column_def(cols, 'fee_close', 'fee')
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
else:
timeframe = get_column_def(cols, 'timeframe', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
timeframe, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
def migrate_open_orders_to_trades(engine):
engine.execute("""
insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
select id ft_trade_id, pair ft_pair, open_order_id,
case when close_rate_requested is null then 'buy'
else 'sell' end ft_order_side, 1 ft_is_open
from trades
where open_order_id is not null
union all
select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
'stoploss' ft_order_side, 1 ft_is_open
from trades
where stoploss_order_id is not null
""")
def check_migrate(engine, decl_base, previous_tables) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = get_table_names_for_table(inspector, 'trades')
table_back_name = get_backup_name(tabs, 'trades_bak')
# Check for latest column
if not has_column(cols, 'amount_requested'):
logger.info(f'Running database migration for trades - backup: {table_back_name}')
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
if 'orders' not in previous_tables:
logger.info('Moving open orders to Orders table.')
migrate_open_orders_to_trades(engine)
else:
pass
# Empty for now - as there is only one iteration of the orders table so far.
# table_back_name = get_backup_name(tabs, 'orders_bak')

286
freqtrade/persistence.py → freqtrade/persistence/models.py
View File

@@ -7,17 +7,19 @@ from decimal import Decimal
from typing import Any, Dict, List, Optional
import arrow
from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String,
create_engine, desc, func, inspect)
from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
String, create_engine, desc, func, inspect)
from sqlalchemy.exc import NoSuchModuleError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Query
from sqlalchemy.orm import Query, relationship
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.schema import UniqueConstraint
from freqtrade.exceptions import OperationalException
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.misc import safe_value_fallback
from freqtrade.persistence.migrations import check_migrate
logger = logging.getLogger(__name__)
@@ -57,121 +59,18 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
# We should use the scoped_session object - not a seperately initialized version
Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.query = Trade.session.query_property()
# Copy session attributes to order object too
Order.session = Trade.session
Order.query = Order.session.query_property()
previous_tables = inspect(engine).get_table_names()
_DECL_BASE.metadata.create_all(engine)
check_migrate(engine)
check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
# Clean dry_run DB if the db is not in-memory
if clean_open_orders and db_url != 'sqlite://':
clean_dry_run_db()
def has_column(columns: List, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def check_migrate(engine) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = inspector.get_table_names()
table_back_name = 'trades_bak'
for i, table_back_name in enumerate(tabs):
table_back_name = f'trades_bak{i}'
logger.debug(f'trying {table_back_name}')
# Check for latest column
if not has_column(cols, 'amount_requested'):
logger.info(f'Running database migration - backup available as {table_back_name}')
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
fee_close = get_column_def(cols, 'fee_close', 'fee')
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
else:
timeframe = get_column_def(cols, 'timeframe', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
_DECL_BASE.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
timeframe, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
def cleanup() -> None:
"""
Flushes all pending operations to disk.
@@ -191,13 +90,117 @@ def clean_dry_run_db() -> None:
trade.open_order_id = None
class Order(_DECL_BASE):
"""
Order database model
Keeps a record of all orders placed on the exchange
One to many relationship with Trades:
- One trade can have many orders
- One Order can only be associated with one Trade
Mirrors CCXT Order structure
"""
__tablename__ = 'orders'
# Uniqueness should be ensured over pair, order_id
# its likely that order_id is unique per Pair on some exchanges.
__table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
id = Column(Integer, primary_key=True)
ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
trade = relationship("Trade", back_populates="orders")
ft_order_side = Column(String, nullable=False)
ft_pair = Column(String, nullable=False)
ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
order_id = Column(String, nullable=False, index=True)
status = Column(String, nullable=True)
symbol = Column(String, nullable=True)
order_type = Column(String, nullable=True)
side = Column(String, nullable=True)
price = Column(Float, nullable=True)
amount = Column(Float, nullable=True)
filled = Column(Float, nullable=True)
remaining = Column(Float, nullable=True)
cost = Column(Float, nullable=True)
order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
order_filled_date = Column(DateTime, nullable=True)
order_update_date = Column(DateTime, nullable=True)
def __repr__(self):
return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
f'side={self.side}, order_type={self.order_type}, status={self.status})')
def update_from_ccxt_object(self, order):
"""
Update Order from ccxt response
Only updates if fields are available from ccxt -
"""
if self.order_id != str(order['id']):
raise DependencyException("Order-id's don't match")
self.status = order.get('status', self.status)
self.symbol = order.get('symbol', self.symbol)
self.order_type = order.get('type', self.order_type)
self.side = order.get('side', self.side)
self.price = order.get('price', self.price)
self.amount = order.get('amount', self.amount)
self.filled = order.get('filled', self.filled)
self.remaining = order.get('remaining', self.remaining)
self.cost = order.get('cost', self.cost)
if 'timestamp' in order and order['timestamp'] is not None:
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
self.ft_is_open = True
if self.status in ('closed', 'canceled', 'cancelled'):
self.ft_is_open = False
if order.get('filled', 0) > 0:
self.order_filled_date = arrow.utcnow().datetime
self.order_update_date = arrow.utcnow().datetime
@staticmethod
def update_orders(orders: List['Order'], order: Dict[str, Any]):
"""
Get all non-closed orders - useful when trying to batch-update orders
"""
filtered_orders = [o for o in orders if o.order_id == order['id']]
if filtered_orders:
oobj = filtered_orders[0]
oobj.update_from_ccxt_object(order)
else:
logger.warning(f"Did not find order for {order['id']}.")
@staticmethod
def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order':
"""
Parse an order from a ccxt object and return a new order Object.
"""
o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair)
o.update_from_ccxt_object(order)
return o
@staticmethod
def get_open_orders() -> List['Order']:
"""
"""
return Order.query.filter(Order.ft_is_open.is_(True)).all()
class Trade(_DECL_BASE):
"""
Class used to define a trade structure
Trade database model.
Also handles updating and querying trades
"""
__tablename__ = 'trades'
id = Column(Integer, primary_key=True)
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
exchange = Column(String, nullable=False)
pair = Column(String, nullable=False, index=True)
is_open = Column(Boolean, nullable=False, default=True, index=True)
@@ -380,15 +383,18 @@ class Trade(_DECL_BASE):
self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price'))
self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount'))
self.recalc_open_trade_price()
logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self)
if self.is_open:
logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.')
self.open_order_id = None
elif order_type in ('market', 'limit') and order['side'] == 'sell':
if self.is_open:
logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.')
self.close(safe_value_fallback(order, 'average', 'price'))
logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self)
elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'):
self.stoploss_order_id = None
self.close_rate_requested = self.stop_loss
logger.info('%s is hit for %s.', order_type.upper(), self)
if self.is_open:
logger.info(f'{order_type.upper()} is hit for {self}.')
self.close(order['average'])
else:
raise ValueError(f'Unknown order type: {order_type}')
@@ -402,7 +408,7 @@ class Trade(_DECL_BASE):
self.close_rate = Decimal(rate)
self.close_profit = self.calc_profit_ratio()
self.close_profit_abs = self.calc_profit()
self.close_date = datetime.utcnow()
self.close_date = self.close_date or datetime.utcnow()
self.is_open = False
self.sell_order_status = 'closed'
self.open_order_id = None
@@ -440,6 +446,17 @@ class Trade(_DECL_BASE):
else:
return False
def update_order(self, order: Dict) -> None:
Order.update_orders(self.orders, order)
def delete(self) -> None:
for order in self.orders:
Order.session.delete(order)
Trade.session.delete(self)
Trade.session.flush()
def _calc_open_trade_price(self) -> float:
"""
Calculate the open_rate including open_fee.
@@ -506,6 +523,21 @@ class Trade(_DECL_BASE):
profit_ratio = (close_trade_price / self.open_trade_price) - 1
return float(f"{profit_ratio:.8f}")
def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]:
"""
Finds latest order for this orderside and status
:param order_side: Side of the order (either 'buy' or 'sell')
:param is_open: Only search for open orders?
:return: latest Order object if it exists, else None
"""
orders = [o for o in self.orders if o.side == order_side]
if is_open is not None:
orders = [o for o in orders if o.ft_is_open == is_open]
if len(orders) > 0:
return orders[-1]
else:
return None
@staticmethod
def get_trades(trade_filter=None) -> Query:
"""
@@ -537,6 +569,26 @@ class Trade(_DECL_BASE):
"""
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def get_open_trades_without_assigned_fees():
"""
Returns all open trades which don't have open fees set correctly
"""
return Trade.get_trades([Trade.fee_open_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(True),
]).all()
@staticmethod
def get_sold_trades_without_assigned_fees():
"""
Returns all closed trades which don't have fees set correctly
"""
return Trade.get_trades([Trade.fee_close_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(False),
]).all()
@staticmethod
def total_open_trades_stakes() -> float:
"""

3
freqtrade/rpc/rpc.py
View File

@@ -562,8 +562,7 @@ class RPC:
except (ExchangeError):
pass
Trade.session.delete(trade)
Trade.session.flush()
trade.delete()
self._freqtrade.wallets.update()
return {
'result': 'success',