adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3682 from freqtrade/db_keep_orders

Browse Source 
Keep order history in the database
This commit is contained in:
Matthias 2020-09-19 17:12:14 +02:00 committed by GitHub
commit a559611c15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1504 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/ci.yml vendored
View File

@ -5,7 +5,6 @@ on:
branches: branches:
- master - master
- develop - develop
- github_actions_tests
tags: tags:
release: release:
types: [published] types: [published]

2
docs/developer.md
View File

@ -120,6 +120,8 @@ Below is an outline of exception inheritance hierarchy:
| +---+ InvalidOrderException | +---+ InvalidOrderException
| | | |
| +---+ RetryableOrderError | +---+ RetryableOrderError
| |
| +---+ InsufficientFundsError
| |
+---+ StrategyError +---+ StrategyError
``` ```

1
freqtrade/data/history/history_utils.py
View File

@ -136,7 +136,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
start = None start = None
if timerange: if timerange:
if timerange.starttype == 'date': if timerange.starttype == 'date':
# TODO: convert to date for conversion
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
# Intentionally don't pass timerange in - since we need to load the full dataset. # Intentionally don't pass timerange in - since we need to load the full dataset.

7
freqtrade/exceptions.py
View File

@ -51,6 +51,13 @@ class RetryableOrderError(InvalidOrderException):
""" """
class InsufficientFundsError(InvalidOrderException):
"""
This error is used when there are not enough funds available on the exchange
to create an order.
"""
class TemporaryError(ExchangeError): class TemporaryError(ExchangeError):
""" """
Temporary network or exchange related error. Temporary network or exchange related error.

4
freqtrade/exchange/binance.py
View File

@ -4,7 +4,7 @@ from typing import Dict
import ccxt import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError, from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
InvalidOrderException, OperationalException, InvalidOrderException, OperationalException,
TemporaryError) TemporaryError)
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
@ -80,7 +80,7 @@ class Binance(Exchange):
'stop price: %s. limit: %s', pair, stop_price, rate) 'stop price: %s. limit: %s', pair, stop_price, rate)
return order return order
except ccxt.InsufficientFunds as e: except ccxt.InsufficientFunds as e:
raise ExchangeError( raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to sell amount {amount} at rate {rate}. ' f'Tried to sell amount {amount} at rate {rate}. '
f'Message: {e}') from e f'Message: {e}') from e

4
freqtrade/exchange/common.py
View File

@ -9,7 +9,11 @@ from freqtrade.exceptions import (DDosProtection, RetryableOrderError,
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Maximum default retry count.
# Functions are always called RETRY_COUNT + 1 times (for the original call)
API_RETRY_COUNT = 4 API_RETRY_COUNT = 4
API_FETCH_ORDER_RETRY_COUNT = 5
BAD_EXCHANGES = { BAD_EXCHANGES = {
"bitmex": "Various reasons.", "bitmex": "Various reasons.",
"bitstamp": "Does not provide history. " "bitstamp": "Does not provide history. "

27
freqtrade/exchange/exchange.py
View File

@ -8,7 +8,6 @@ import logging
from copy import deepcopy from copy import deepcopy
from datetime import datetime, timezone from datetime import datetime, timezone
from math import ceil from math import ceil
from random import randint
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
import arrow import arrow
@ -21,9 +20,11 @@ from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.exceptions import (DDosProtection, ExchangeError, from freqtrade.exceptions import (DDosProtection, ExchangeError,
InsufficientFundsError,
InvalidOrderException, OperationalException, InvalidOrderException, OperationalException,
RetryableOrderError, TemporaryError) RetryableOrderError, TemporaryError)
from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT,
BAD_EXCHANGES, retrier, retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
CcxtModuleType = Any CcxtModuleType = Any
@ -487,11 +488,11 @@ class Exchange:
def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]: rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{randint(0, 10**6)}' order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount) _amount = self.amount_to_precision(pair, amount)
dry_order = { dry_order = {
"id": order_id, 'id': order_id,
'pair': pair, 'symbol': pair,
'price': rate, 'price': rate,
'average': rate, 'average': rate,
'amount': _amount, 'amount': _amount,
@ -500,6 +501,7 @@ class Exchange:
'side': side, 'side': side,
'remaining': _amount, 'remaining': _amount,
'datetime': arrow.utcnow().isoformat(), 'datetime': arrow.utcnow().isoformat(),
'timestamp': int(arrow.utcnow().timestamp * 1000),
'status': "closed" if ordertype == "market" else "open", 'status': "closed" if ordertype == "market" else "open",
'fee': None, 'fee': None,
'info': {} 'info': {}
@ -538,7 +540,7 @@ class Exchange:
amount, rate_for_order, params) amount, rate_for_order, params)
except ccxt.InsufficientFunds as e: except ccxt.InsufficientFunds as e:
raise ExchangeError( raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
f'Tried to {side} amount {amount} at rate {rate}.' f'Tried to {side} amount {amount} at rate {rate}.'
f'Message: {e}') from e f'Message: {e}') from e
@ -1027,7 +1029,7 @@ class Exchange:
return order return order
@retrier(retries=5) @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_order(self, order_id: str, pair: str) -> Dict: def fetch_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']: if self._config['dry_run']:
try: try:
@ -1056,6 +1058,17 @@ class Exchange:
# Assign method to fetch_stoploss_order to allow easy overriding in other classes # Assign method to fetch_stoploss_order to allow easy overriding in other classes
fetch_stoploss_order = fetch_order fetch_stoploss_order = fetch_order
def fetch_order_or_stoploss_order(self, order_id: str, pair: str,
stoploss_order: bool = False) -> Dict:
"""
Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
the stoploss_order parameter
:param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
"""
if stoploss_order:
return self.fetch_stoploss_order(order_id, pair)
return self.fetch_order(order_id, pair)
@retrier @retrier
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
""" """

8
freqtrade/exchange/ftx.py
View File

@ -4,11 +4,11 @@ from typing import Any, Dict
import ccxt import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError, from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
InvalidOrderException, OperationalException, InvalidOrderException, OperationalException,
TemporaryError) TemporaryError)
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -71,7 +71,7 @@ class Ftx(Exchange):
'stop price: %s.', pair, stop_price) 'stop price: %s.', pair, stop_price)
return order return order
except ccxt.InsufficientFunds as e: except ccxt.InsufficientFunds as e:
raise ExchangeError( raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e f'Message: {e}') from e
@ -88,7 +88,7 @@ class Ftx(Exchange):
except ccxt.BaseError as e: except ccxt.BaseError as e:
raise OperationalException(e) from e raise OperationalException(e) from e
@retrier(retries=5) @retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']: if self._config['dry_run']:
try: try:

4
freqtrade/exchange/kraken.py
View File

@ -4,7 +4,7 @@ from typing import Any, Dict
import ccxt import ccxt
from freqtrade.exceptions import (DDosProtection, ExchangeError, from freqtrade.exceptions import (DDosProtection, InsufficientFundsError,
InvalidOrderException, OperationalException, InvalidOrderException, OperationalException,
TemporaryError) TemporaryError)
from freqtrade.exchange import Exchange from freqtrade.exchange import Exchange
@ -98,7 +98,7 @@ class Kraken(Exchange):
'stop price: %s.', pair, stop_price) 'stop price: %s.', pair, stop_price)
return order return order
except ccxt.InsufficientFunds as e: except ccxt.InsufficientFunds as e:
raise ExchangeError( raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e f'Message: {e}') from e

196
freqtrade/freqtradebot.py
View File

@ -17,12 +17,12 @@ from freqtrade.configuration import validate_config_consistency
from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.converter import order_book_to_dataframe
from freqtrade.data.dataprovider import DataProvider from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge from freqtrade.edge import Edge
from freqtrade.exceptions import (DependencyException, ExchangeError, from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
InvalidOrderException, PricingError) InvalidOrderException, PricingError)
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date
from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.misc import safe_value_fallback, safe_value_fallback2
from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.pairlist.pairlistmanager import PairListManager
from freqtrade.persistence import Trade from freqtrade.persistence import Order, Trade
from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State from freqtrade.state import State
@ -134,6 +134,10 @@ class FreqtradeBot:
# Adjust stoploss if it was changed # Adjust stoploss if it was changed
Trade.stoploss_reinitialization(self.strategy.stoploss) Trade.stoploss_reinitialization(self.strategy.stoploss)
# Only update open orders on startup
# This will update the database after the initial migration
self.update_open_orders()
def process(self) -> None: def process(self) -> None:
""" """
Queries the persistence layer for open trades and handles them, Queries the persistence layer for open trades and handles them,
@ -144,6 +148,8 @@ class FreqtradeBot:
# Check whether markets have to be reloaded and reload them when it's needed # Check whether markets have to be reloaded and reload them when it's needed
self.exchange.reload_markets() self.exchange.reload_markets()
self.update_closed_trades_without_assigned_fees()
# Query trades from persistence layer # Query trades from persistence layer
trades = Trade.get_open_trades() trades = Trade.get_open_trades()
@ -227,6 +233,104 @@ class FreqtradeBot:
open_trades = len(Trade.get_open_trades()) open_trades = len(Trade.get_open_trades())
return max(0, self.config['max_open_trades'] - open_trades) return max(0, self.config['max_open_trades'] - open_trades)
def update_open_orders(self):
"""
Updates open orders based on order list kept in the database.
Mainly updates the state of orders - but may also close trades
"""
orders = Order.get_open_orders()
logger.info(f"Updating {len(orders)} open orders.")
for order in orders:
try:
fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
order.ft_order_side == 'stoploss')
self.update_trade_state(order.trade, order.order_id, fo)
except ExchangeError as e:
logger.warning(f"Error updating Order {order.order_id} due to {e}")
def update_closed_trades_without_assigned_fees(self):
"""
Update closed trades without close fees assigned.
Only acts when Orders are in the database, otherwise the last orderid is unknown.
"""
trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees()
for trade in trades:
if not trade.is_open and not trade.fee_updated('sell'):
# Get sell fee
order = trade.select_order('sell', False)
if order:
logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id,
stoploss_order=order.ft_order_side == 'stoploss')
trades: List[Trade] = Trade.get_open_trades_without_assigned_fees()
for trade in trades:
if trade.is_open and not trade.fee_updated('buy'):
order = trade.select_order('buy', False)
if order:
logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id)
def handle_insufficient_funds(self, trade: Trade):
"""
Determine if we ever opened a sell order for this trade.
If not, try update buy fees - otherwise "refind" the open order we obviously lost.
"""
sell_order = trade.select_order('sell', None)
if sell_order:
self.refind_lost_order(trade)
else:
self.reupdate_buy_order_fees(trade)
def reupdate_buy_order_fees(self, trade: Trade):
"""
Get buy order from database, and try to reupdate.
Handles trades where the initial fee-update did not work.
"""
logger.info(f"Trying to reupdate buy fees for {trade}")
order = trade.select_order('buy', False)
if order:
logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.")
self.update_trade_state(trade, order.order_id)
def refind_lost_order(self, trade):
"""
Try refinding a lost trade.
Only used when InsufficientFunds appears on sell orders (stoploss or sell).
Tries to walk the stored orders and sell them off eventually.
"""
logger.info(f"Trying to refind lost order for {trade}")
for order in trade.orders:
logger.info(f"Trying to refind {order}")
fo = None
if not order.ft_is_open:
logger.debug(f"Order {order} is no longer open.")
continue
if order.ft_order_side == 'buy':
# Skip buy side - this is handled by reupdate_buy_order_fees
continue
try:
fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair,
order.ft_order_side == 'stoploss')
if order.ft_order_side == 'stoploss':
if fo and fo['status'] == 'open':
# Assume this as the open stoploss order
trade.stoploss_order_id = order.order_id
elif order.ft_order_side == 'sell':
if fo and fo['status'] == 'open':
# Assume this as the open order
trade.open_order_id = order.order_id
if fo:
logger.info(f"Found {order} for trade {trade}.jj")
self.update_trade_state(trade, order.order_id, fo,
stoploss_order=order.ft_order_side == 'stoploss')
except ExchangeError:
logger.warning(f"Error updating {order.order_id}.")
# #
# BUY / enter positions / open trades logic and methods # BUY / enter positions / open trades logic and methods
# #
@ -528,6 +632,7 @@ class FreqtradeBot:
order = self.exchange.buy(pair=pair, ordertype=order_type, order = self.exchange.buy(pair=pair, ordertype=order_type,
amount=amount, rate=buy_limit_requested, amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force) time_in_force=time_in_force)
order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
order_id = order['id'] order_id = order['id']
order_status = order.get('status', None) order_status = order.get('status', None)
@ -556,7 +661,6 @@ class FreqtradeBot:
stake_amount = order['cost'] stake_amount = order['cost']
amount = safe_value_fallback(order, 'filled', 'amount') amount = safe_value_fallback(order, 'filled', 'amount')
buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
order_id = None
# in case of FOK the order may be filled immediately and fully # in case of FOK the order may be filled immediately and fully
elif order_status == 'closed': elif order_status == 'closed':
@ -581,10 +685,11 @@ class FreqtradeBot:
strategy=self.strategy.get_strategy_name(), strategy=self.strategy.get_strategy_name(),
timeframe=timeframe_to_minutes(self.config['timeframe']) timeframe=timeframe_to_minutes(self.config['timeframe'])
) )
trade.orders.append(order_obj)
# Update fees if order is closed # Update fees if order is closed
if order_status == 'closed': if order_status == 'closed':
self.update_trade_state(trade, order) self.update_trade_state(trade, order_id, order)
Trade.session.add(trade) Trade.session.add(trade)
Trade.session.flush() Trade.session.flush()
@ -783,8 +888,16 @@ class FreqtradeBot:
stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount, stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount,
stop_price=stop_price, stop_price=stop_price,
order_types=self.strategy.order_types) order_types=self.strategy.order_types)
order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss')
trade.orders.append(order_obj)
trade.stoploss_order_id = str(stoploss_order['id']) trade.stoploss_order_id = str(stoploss_order['id'])
return True return True
except InsufficientFundsError as e:
logger.warning(f"Unable to place stoploss order {e}.")
# Try to figure out what went wrong
self.handle_insufficient_funds(trade)
except InvalidOrderException as e: except InvalidOrderException as e:
trade.stoploss_order_id = None trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}') logger.error(f'Unable to place a stoploss order on exchange. {e}')
@ -814,10 +927,14 @@ class FreqtradeBot:
except InvalidOrderException as exception: except InvalidOrderException as exception:
logger.warning('Unable to fetch stoploss order: %s', exception) logger.warning('Unable to fetch stoploss order: %s', exception)
if stoploss_order:
trade.update_order(stoploss_order)
# We check if stoploss order is fulfilled # We check if stoploss order is fulfilled
if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'): if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'):
trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
self.update_trade_state(trade, stoploss_order, sl_order=True) self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
stoploss_order=True)
# Lock pair for one candle to prevent immediate rebuys # Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, self.strategy.lock_pair(trade.pair,
timeframe_to_next_date(self.config['timeframe'])) timeframe_to_next_date(self.config['timeframe']))
@ -869,10 +986,11 @@ class FreqtradeBot:
update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
# cancelling the current stoploss on exchange first # cancelling the current stoploss on exchange first
logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) ' logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
'in order to add another one ...', order['id']) f"(orderid:{order['id']}) in order to add another one ...")
try: try:
self.exchange.cancel_stoploss_order(order['id'], trade.pair) co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
trade.update_order(co)
except InvalidOrderException: except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {order['id']} " logger.exception(f"Could not cancel stoploss order {order['id']} "
f"for pair {trade.pair}") f"for pair {trade.pair}")
@ -927,7 +1045,7 @@ class FreqtradeBot:
logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc())
continue continue
fully_cancelled = self.update_trade_state(trade, order) fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order)
if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and ( if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and (
fully_cancelled fully_cancelled
@ -995,8 +1113,7 @@ class FreqtradeBot:
if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC):
logger.info('Buy order fully cancelled. Removing %s from database.', trade) logger.info('Buy order fully cancelled. Removing %s from database.', trade)
# if trade is not partially completed, just delete the trade # if trade is not partially completed, just delete the trade
Trade.session.delete(trade) trade.delete()
Trade.session.flush()
was_trade_fully_canceled = True was_trade_fully_canceled = True
reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
else: else:
@ -1007,7 +1124,7 @@ class FreqtradeBot:
# we need to fall back to the values from order if corder does not contain these keys. # we need to fall back to the values from order if corder does not contain these keys.
trade.amount = filled_amount trade.amount = filled_amount
trade.stake_amount = trade.amount * trade.open_rate trade.stake_amount = trade.amount * trade.open_rate
self.update_trade_state(trade, corder, trade.amount) self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None trade.open_order_id = None
logger.info('Partial buy order timeout for %s.', trade) logger.info('Partial buy order timeout for %s.', trade)
@ -1121,19 +1238,28 @@ class FreqtradeBot:
logger.info(f"User requested abortion of selling {trade.pair}") logger.info(f"User requested abortion of selling {trade.pair}")
return False return False
# Execute sell and update trade record try:
order = self.exchange.sell(pair=str(trade.pair), # Execute sell and update trade record
ordertype=order_type, order = self.exchange.sell(pair=trade.pair,
amount=amount, rate=limit, ordertype=order_type,
time_in_force=time_in_force amount=amount, rate=limit,
) time_in_force=time_in_force
)
except InsufficientFundsError as e:
logger.warning(f"Unable to place order {e}.")
# Try to figure out what went wrong
self.handle_insufficient_funds(trade)
return False
order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell')
trade.orders.append(order_obj)
trade.open_order_id = order['id'] trade.open_order_id = order['id']
trade.close_rate_requested = limit trade.close_rate_requested = limit
trade.sell_reason = sell_reason.value trade.sell_reason = sell_reason.value
# In case of market sell orders the order can be closed immediately # In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') == 'closed': if order.get('status', 'unknown') == 'closed':
self.update_trade_state(trade, order) self.update_trade_state(trade, trade.open_order_id, order)
Trade.session.flush() Trade.session.flush()
# Lock pair for one candle to prevent immediate rebuys # Lock pair for one candle to prevent immediate rebuys
@ -1230,30 +1356,35 @@ class FreqtradeBot:
# Common update trade state methods # Common update trade state methods
# #
def update_trade_state(self, trade: Trade, action_order: dict = None, def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None,
order_amount: float = None, sl_order: bool = False) -> bool: stoploss_order: bool = False) -> bool:
""" """
Checks trades with open orders and updates the amount if necessary Checks trades with open orders and updates the amount if necessary
Handles closing both buy and sell orders. Handles closing both buy and sell orders.
:param trade: Trade object of the trade we're analyzing
:param order_id: Order-id of the order we're analyzing
:param action_order: Already aquired order object
:return: True if order has been cancelled without being filled partially, False otherwise :return: True if order has been cancelled without being filled partially, False otherwise
""" """
# Get order details for actual price per unit if not order_id:
if trade.open_order_id: logger.warning(f'Orderid for trade {trade} is empty.')
order_id = trade.open_order_id
elif trade.stoploss_order_id and sl_order:
order_id = trade.stoploss_order_id
else:
return False return False
# Update trade with order values # Update trade with order values
logger.info('Found open order for %s', trade) logger.info('Found open order for %s', trade)
try: try:
order = action_order or self.exchange.fetch_order(order_id, trade.pair) order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id,
trade.pair,
stoploss_order)
except InvalidOrderException as exception: except InvalidOrderException as exception:
logger.warning('Unable to fetch order %s: %s', order_id, exception) logger.warning('Unable to fetch order %s: %s', order_id, exception)
return False return False
trade.update_order(order)
# Try update amount (binance-fix) # Try update amount (binance-fix)
try: try:
new_amount = self.get_real_amount(trade, order, order_amount) new_amount = self.get_real_amount(trade, order)
if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount, if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount,
abs_tol=constants.MATH_CLOSE_PREC): abs_tol=constants.MATH_CLOSE_PREC):
order['amount'] = new_amount order['amount'] = new_amount
@ -1291,7 +1422,7 @@ class FreqtradeBot:
return real_amount return real_amount
return amount return amount
def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: def get_real_amount(self, trade: Trade, order: Dict) -> float:
""" """
Detect and update trade fee. Detect and update trade fee.
Calls trade.update_fee() uppon correct detection. Calls trade.update_fee() uppon correct detection.
@ -1300,8 +1431,7 @@ class FreqtradeBot:
:return: identical (or new) amount for the trade :return: identical (or new) amount for the trade
""" """
# Init variables # Init variables
if order_amount is None: order_amount = safe_value_fallback(order, 'filled', 'amount')
order_amount = safe_value_fallback(order, 'filled', 'amount')
# Only run for closed orders # Only run for closed orders
if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': if trade.fee_updated(order.get('side', '')) or order['status'] == 'open':
return order_amount return order_amount
@ -1325,7 +1455,7 @@ class FreqtradeBot:
""" """
fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
""" """
trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
trade.open_date) trade.open_date)
if len(trades) == 0: if len(trades) == 0:

4
freqtrade/persistence/__init__.py Normal file
View File

@ -0,0 +1,4 @@
# flake8: noqa: F401
from freqtrade.persistence.models import (Order, Trade, clean_dry_run_db,
cleanup, init)

149
freqtrade/persistence/migrations.py Normal file
View File

@ -0,0 +1,149 @@
import logging
from typing import List
from sqlalchemy import inspect
logger = logging.getLogger(__name__)
def get_table_names_for_table(inspector, tabletype):
return [t for t in inspector.get_table_names() if t.startswith(tabletype)]
def has_column(columns: List, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def get_backup_name(tabs, backup_prefix: str):
table_back_name = backup_prefix
for i, table_back_name in enumerate(tabs):
table_back_name = f'{backup_prefix}{i}'
logger.debug(f'trying {table_back_name}')
return table_back_name
def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
fee_close = get_column_def(cols, 'fee_close', 'fee')
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
else:
timeframe = get_column_def(cols, 'timeframe', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
timeframe, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
def migrate_open_orders_to_trades(engine):
engine.execute("""
insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
select id ft_trade_id, pair ft_pair, open_order_id,
case when close_rate_requested is null then 'buy'
else 'sell' end ft_order_side, 1 ft_is_open
from trades
where open_order_id is not null
union all
select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
'stoploss' ft_order_side, 1 ft_is_open
from trades
where stoploss_order_id is not null
""")
def check_migrate(engine, decl_base, previous_tables) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = get_table_names_for_table(inspector, 'trades')
table_back_name = get_backup_name(tabs, 'trades_bak')
# Check for latest column
if not has_column(cols, 'amount_requested'):
logger.info(f'Running database migration for trades - backup: {table_back_name}')
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
if 'orders' not in previous_tables:
logger.info('Moving open orders to Orders table.')
migrate_open_orders_to_trades(engine)
else:
pass
# Empty for now - as there is only one iteration of the orders table so far.
# table_back_name = get_backup_name(tabs, 'orders_bak')

286
freqtrade/persistence.py → freqtrade/persistence/models.py
View File

@ -7,17 +7,19 @@ from decimal import Decimal
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
import arrow import arrow
from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
create_engine, desc, func, inspect) String, create_engine, desc, func, inspect)
from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.exc import NoSuchModuleError
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Query from sqlalchemy.orm import Query, relationship
from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker from sqlalchemy.orm.session import sessionmaker
from sqlalchemy.pool import StaticPool from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.schema import UniqueConstraint
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.misc import safe_value_fallback from freqtrade.misc import safe_value_fallback
from freqtrade.persistence.migrations import check_migrate
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -57,121 +59,18 @@ def init(db_url: str, clean_open_orders: bool = False) -> None:
# We should use the scoped_session object - not a seperately initialized version # We should use the scoped_session object - not a seperately initialized version
Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.query = Trade.session.query_property() Trade.query = Trade.session.query_property()
# Copy session attributes to order object too
Order.session = Trade.session
Order.query = Order.session.query_property()
previous_tables = inspect(engine).get_table_names()
_DECL_BASE.metadata.create_all(engine) _DECL_BASE.metadata.create_all(engine)
check_migrate(engine) check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
# Clean dry_run DB if the db is not in-memory # Clean dry_run DB if the db is not in-memory
if clean_open_orders and db_url != 'sqlite://': if clean_open_orders and db_url != 'sqlite://':
clean_dry_run_db() clean_dry_run_db()
def has_column(columns: List, searchname: str) -> bool:
return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1
def get_column_def(columns: List, column: str, default: str) -> str:
return default if not has_column(columns, column) else column
def check_migrate(engine) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
tabs = inspector.get_table_names()
table_back_name = 'trades_bak'
for i, table_back_name in enumerate(tabs):
table_back_name = f'trades_bak{i}'
logger.debug(f'trying {table_back_name}')
# Check for latest column
if not has_column(cols, 'amount_requested'):
logger.info(f'Running database migration - backup available as {table_back_name}')
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
fee_close = get_column_def(cols, 'fee_close', 'fee')
fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null')
fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null')
open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null')
close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null')
stop_loss = get_column_def(cols, 'stop_loss', '0.0')
stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null')
initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0')
initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null')
stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null')
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
else:
timeframe = get_column_def(cols, 'timeframe', 'null')
open_trade_price = get_column_def(cols, 'open_trade_price',
f'amount * open_rate * (1 + {fee_open})')
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
engine.execute(f"alter table trades rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
_DECL_BASE.metadata.create_all(engine)
# Copy data back - following the correct schema
engine.execute(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
timeframe, open_trade_price, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
open_rate, {open_rate_requested} open_rate_requested, close_rate,
{close_rate_requested} close_rate_requested, close_profit,
stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id,
{stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct,
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs
from {table_back_name}
""")
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
def cleanup() -> None: def cleanup() -> None:
""" """
Flushes all pending operations to disk. Flushes all pending operations to disk.
@ -191,13 +90,117 @@ def clean_dry_run_db() -> None:
trade.open_order_id = None trade.open_order_id = None
class Order(_DECL_BASE):
"""
Order database model
Keeps a record of all orders placed on the exchange
One to many relationship with Trades:
- One trade can have many orders
- One Order can only be associated with one Trade
Mirrors CCXT Order structure
"""
__tablename__ = 'orders'
# Uniqueness should be ensured over pair, order_id
# its likely that order_id is unique per Pair on some exchanges.
__table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),)
id = Column(Integer, primary_key=True)
ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True)
trade = relationship("Trade", back_populates="orders")
ft_order_side = Column(String, nullable=False)
ft_pair = Column(String, nullable=False)
ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
order_id = Column(String, nullable=False, index=True)
status = Column(String, nullable=True)
symbol = Column(String, nullable=True)
order_type = Column(String, nullable=True)
side = Column(String, nullable=True)
price = Column(Float, nullable=True)
amount = Column(Float, nullable=True)
filled = Column(Float, nullable=True)
remaining = Column(Float, nullable=True)
cost = Column(Float, nullable=True)
order_date = Column(DateTime, nullable=True, default=datetime.utcnow)
order_filled_date = Column(DateTime, nullable=True)
order_update_date = Column(DateTime, nullable=True)
def __repr__(self):
return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, '
f'side={self.side}, order_type={self.order_type}, status={self.status})')
def update_from_ccxt_object(self, order):
"""
Update Order from ccxt response
Only updates if fields are available from ccxt -
"""
if self.order_id != str(order['id']):
raise DependencyException("Order-id's don't match")
self.status = order.get('status', self.status)
self.symbol = order.get('symbol', self.symbol)
self.order_type = order.get('type', self.order_type)
self.side = order.get('side', self.side)
self.price = order.get('price', self.price)
self.amount = order.get('amount', self.amount)
self.filled = order.get('filled', self.filled)
self.remaining = order.get('remaining', self.remaining)
self.cost = order.get('cost', self.cost)
if 'timestamp' in order and order['timestamp'] is not None:
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
self.ft_is_open = True
if self.status in ('closed', 'canceled', 'cancelled'):
self.ft_is_open = False
if order.get('filled', 0) > 0:
self.order_filled_date = arrow.utcnow().datetime
self.order_update_date = arrow.utcnow().datetime
@staticmethod
def update_orders(orders: List['Order'], order: Dict[str, Any]):
"""
Get all non-closed orders - useful when trying to batch-update orders
"""
filtered_orders = [o for o in orders if o.order_id == order['id']]
if filtered_orders:
oobj = filtered_orders[0]
oobj.update_from_ccxt_object(order)
else:
logger.warning(f"Did not find order for {order['id']}.")
@staticmethod
def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order':
"""
Parse an order from a ccxt object and return a new order Object.
"""
o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair)
o.update_from_ccxt_object(order)
return o
@staticmethod
def get_open_orders() -> List['Order']:
"""
"""
return Order.query.filter(Order.ft_is_open.is_(True)).all()
class Trade(_DECL_BASE): class Trade(_DECL_BASE):
""" """
Class used to define a trade structure Trade database model.
Also handles updating and querying trades
""" """
__tablename__ = 'trades' __tablename__ = 'trades'
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
exchange = Column(String, nullable=False) exchange = Column(String, nullable=False)
pair = Column(String, nullable=False, index=True) pair = Column(String, nullable=False, index=True)
is_open = Column(Boolean, nullable=False, default=True, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True)
@ -380,15 +383,18 @@ class Trade(_DECL_BASE):
self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price'))
self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount'))
self.recalc_open_trade_price() self.recalc_open_trade_price()
logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) if self.is_open:
logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.')
self.open_order_id = None self.open_order_id = None
elif order_type in ('market', 'limit') and order['side'] == 'sell': elif order_type in ('market', 'limit') and order['side'] == 'sell':
if self.is_open:
logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.')
self.close(safe_value_fallback(order, 'average', 'price')) self.close(safe_value_fallback(order, 'average', 'price'))
logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self)
elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'): elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'):
self.stoploss_order_id = None self.stoploss_order_id = None
self.close_rate_requested = self.stop_loss self.close_rate_requested = self.stop_loss
logger.info('%s is hit for %s.', order_type.upper(), self) if self.is_open:
logger.info(f'{order_type.upper()} is hit for {self}.')
self.close(order['average']) self.close(order['average'])
else: else:
raise ValueError(f'Unknown order type: {order_type}') raise ValueError(f'Unknown order type: {order_type}')
@ -402,7 +408,7 @@ class Trade(_DECL_BASE):
self.close_rate = Decimal(rate) self.close_rate = Decimal(rate)
self.close_profit = self.calc_profit_ratio() self.close_profit = self.calc_profit_ratio()
self.close_profit_abs = self.calc_profit() self.close_profit_abs = self.calc_profit()
self.close_date = datetime.utcnow() self.close_date = self.close_date or datetime.utcnow()
self.is_open = False self.is_open = False
self.sell_order_status = 'closed' self.sell_order_status = 'closed'
self.open_order_id = None self.open_order_id = None
@ -440,6 +446,17 @@ class Trade(_DECL_BASE):
else: else:
return False return False
def update_order(self, order: Dict) -> None:
Order.update_orders(self.orders, order)
def delete(self) -> None:
for order in self.orders:
Order.session.delete(order)
Trade.session.delete(self)
Trade.session.flush()
def _calc_open_trade_price(self) -> float: def _calc_open_trade_price(self) -> float:
""" """
Calculate the open_rate including open_fee. Calculate the open_rate including open_fee.
@ -506,6 +523,21 @@ class Trade(_DECL_BASE):
profit_ratio = (close_trade_price / self.open_trade_price) - 1 profit_ratio = (close_trade_price / self.open_trade_price) - 1
return float(f"{profit_ratio:.8f}") return float(f"{profit_ratio:.8f}")
def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]:
"""
Finds latest order for this orderside and status
:param order_side: Side of the order (either 'buy' or 'sell')
:param is_open: Only search for open orders?
:return: latest Order object if it exists, else None
"""
orders = [o for o in self.orders if o.side == order_side]
if is_open is not None:
orders = [o for o in orders if o.ft_is_open == is_open]
if len(orders) > 0:
return orders[-1]
else:
return None
@staticmethod @staticmethod
def get_trades(trade_filter=None) -> Query: def get_trades(trade_filter=None) -> Query:
""" """
@ -537,6 +569,26 @@ class Trade(_DECL_BASE):
""" """
return Trade.get_trades(Trade.open_order_id.isnot(None)).all() return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def get_open_trades_without_assigned_fees():
"""
Returns all open trades which don't have open fees set correctly
"""
return Trade.get_trades([Trade.fee_open_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(True),
]).all()
@staticmethod
def get_sold_trades_without_assigned_fees():
"""
Returns all closed trades which don't have fees set correctly
"""
return Trade.get_trades([Trade.fee_close_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(False),
]).all()
@staticmethod @staticmethod
def total_open_trades_stakes() -> float: def total_open_trades_stakes() -> float:
""" """

3
freqtrade/rpc/rpc.py
View File

@ -562,8 +562,7 @@ class RPC:
except (ExchangeError): except (ExchangeError):
pass pass
Trade.session.delete(trade) trade.delete()
Trade.session.flush()
self._freqtrade.wallets.update() self._freqtrade.wallets.update()
return { return {
'result': 'success', 'result': 'success',

3
tests/commands/test_commands.py
View File

@ -18,6 +18,7 @@ from freqtrade.state import RunMode
from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re,
patch_exchange, patch_exchange,
patched_configuration_load_config_file) patched_configuration_load_config_file)
from tests.conftest_trades import MOCK_TRADE_COUNT
def test_setup_utils_configuration(): def test_setup_utils_configuration():
@ -1116,7 +1117,7 @@ def test_show_trades(mocker, fee, capsys, caplog):
pargs = get_args(args) pargs = get_args(args)
pargs['config'] = None pargs['config'] = None
start_show_trades(pargs) start_show_trades(pargs)
assert log_has("Printing 4 Trades: ", caplog) assert log_has(f"Printing {MOCK_TRADE_COUNT} Trades: ", caplog)
captured = capsys.readouterr() captured = capsys.readouterr()
assert "Trade(id=1" in captured.out assert "Trade(id=1" in captured.out
assert "Trade(id=2" in captured.out assert "Trade(id=2" in captured.out

100
tests/conftest.py
View File

@ -22,6 +22,8 @@ from freqtrade.freqtradebot import FreqtradeBot
from freqtrade.persistence import Trade from freqtrade.persistence import Trade
from freqtrade.resolvers import ExchangeResolver from freqtrade.resolvers import ExchangeResolver
from freqtrade.worker import Worker from freqtrade.worker import Worker
from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3,
mock_trade_4, mock_trade_5, mock_trade_6)
logging.getLogger('').setLevel(logging.INFO) logging.getLogger('').setLevel(logging.INFO)
@ -172,64 +174,22 @@ def create_mock_trades(fee):
Create some fake trades ... Create some fake trades ...
""" """
# Simulate dry_run entries # Simulate dry_run entries
trade = Trade( trade = mock_trade_1(fee)
pair='ETH/BTC',
stake_amount=0.001,
amount=123.0,
amount_requested=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='dry_run_buy_12345',
strategy='DefaultStrategy',
)
Trade.session.add(trade) Trade.session.add(trade)
trade = Trade( trade = mock_trade_2(fee)
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
amount_requested=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
close_rate=0.128,
close_profit=0.005,
exchange='bittrex',
is_open=False,
open_order_id='dry_run_sell_12345',
strategy='DefaultStrategy',
)
Trade.session.add(trade) Trade.session.add(trade)
trade = Trade( trade = mock_trade_3(fee)
pair='XRP/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.05,
close_rate=0.06,
close_profit=0.01,
exchange='bittrex',
is_open=False,
)
Trade.session.add(trade) Trade.session.add(trade)
# Simulate prod entry trade = mock_trade_4(fee)
trade = Trade( Trade.session.add(trade)
pair='ETC/BTC',
stake_amount=0.001, trade = mock_trade_5(fee)
amount=123.0, Trade.session.add(trade)
amount_requested=124.0,
fee_open=fee.return_value, trade = mock_trade_6(fee)
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='prod_buy_12345',
strategy='DefaultStrategy',
)
Trade.session.add(trade) Trade.session.add(trade)
@ -823,22 +783,32 @@ def markets_empty():
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def limit_buy_order(): def limit_buy_order_open():
return { return {
'id': 'mocked_limit_buy', 'id': 'mocked_limit_buy',
'type': 'limit', 'type': 'limit',
'side': 'buy', 'side': 'buy',
'symbol': 'mocked', 'symbol': 'mocked',
'datetime': arrow.utcnow().isoformat(), 'datetime': arrow.utcnow().isoformat(),
'timestamp': arrow.utcnow().timestamp,
'price': 0.00001099, 'price': 0.00001099,
'amount': 90.99181073, 'amount': 90.99181073,
'filled': 90.99181073, 'filled': 0.0,
'cost': 0.0009999, 'cost': 0.0009999,
'remaining': 0.0, 'remaining': 90.99181073,
'status': 'closed' 'status': 'open'
} }
@pytest.fixture(scope='function')
def limit_buy_order(limit_buy_order_open):
order = deepcopy(limit_buy_order_open)
order['status'] = 'closed'
order['filled'] = order['amount']
order['remaining'] = 0.0
return order
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def market_buy_order(): def market_buy_order():
return { return {
@ -1021,21 +991,31 @@ def limit_buy_order_canceled_empty(request):
@pytest.fixture @pytest.fixture
def limit_sell_order(): def limit_sell_order_open():
return { return {
'id': 'mocked_limit_sell', 'id': 'mocked_limit_sell',
'type': 'limit', 'type': 'limit',
'side': 'sell', 'side': 'sell',
'pair': 'mocked', 'pair': 'mocked',
'datetime': arrow.utcnow().isoformat(), 'datetime': arrow.utcnow().isoformat(),
'timestamp': arrow.utcnow().timestamp,
'price': 0.00001173, 'price': 0.00001173,
'amount': 90.99181073, 'amount': 90.99181073,
'filled': 90.99181073, 'filled': 0.0,
'remaining': 0.0, 'remaining': 90.99181073,
'status': 'closed' 'status': 'open'
} }
@pytest.fixture
def limit_sell_order(limit_sell_order_open):
order = deepcopy(limit_sell_order_open)
order['remaining'] = 0.0
order['filled'] = order['amount']
order['status'] = 'closed'
return order
@pytest.fixture @pytest.fixture
def order_book_l2(): def order_book_l2():
return MagicMock(return_value={ return MagicMock(return_value={

279
tests/conftest_trades.py Normal file
View File

@ -0,0 +1,279 @@
from freqtrade.persistence.models import Order, Trade
MOCK_TRADE_COUNT = 6
def mock_order_1():
return {
'id': '1234',
'symbol': 'ETH/BTC',
'status': 'closed',
'side': 'buy',
'type': 'limit',
'price': 0.123,
'amount': 123.0,
'filled': 123.0,
'remaining': 0.0,
}
def mock_trade_1(fee):
trade = Trade(
pair='ETH/BTC',
stake_amount=0.001,
amount=123.0,
amount_requested=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='dry_run_buy_12345',
strategy='DefaultStrategy',
)
o = Order.parse_from_ccxt_object(mock_order_1(), 'ETH/BTC', 'buy')
trade.orders.append(o)
return trade
def mock_order_2():
return {
'id': '1235',
'symbol': 'ETC/BTC',
'status': 'closed',
'side': 'buy',
'type': 'limit',
'price': 0.123,
'amount': 123.0,
'filled': 123.0,
'remaining': 0.0,
}
def mock_order_2_sell():
return {
'id': '12366',
'symbol': 'ETC/BTC',
'status': 'closed',
'side': 'sell',
'type': 'limit',
'price': 0.128,
'amount': 123.0,
'filled': 123.0,
'remaining': 0.0,
}
def mock_trade_2(fee):
"""
Closed trade...
"""
trade = Trade(
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
amount_requested=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
close_rate=0.128,
close_profit=0.005,
exchange='bittrex',
is_open=False,
open_order_id='dry_run_sell_12345',
strategy='DefaultStrategy',
)
o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy')
trade.orders.append(o)
o = Order.parse_from_ccxt_object(mock_order_2_sell(), 'ETC/BTC', 'sell')
trade.orders.append(o)
return trade
def mock_order_3():
return {
'id': '41231a12a',
'symbol': 'XRP/BTC',
'status': 'closed',
'side': 'buy',
'type': 'limit',
'price': 0.05,
'amount': 123.0,
'filled': 123.0,
'remaining': 0.0,
}
def mock_order_3_sell():
return {
'id': '41231a666a',
'symbol': 'XRP/BTC',
'status': 'closed',
'side': 'sell',
'type': 'stop_loss_limit',
'price': 0.06,
'average': 0.06,
'amount': 123.0,
'filled': 123.0,
'remaining': 0.0,
}
def mock_trade_3(fee):
"""
Closed trade
"""
trade = Trade(
pair='XRP/BTC',
stake_amount=0.001,
amount=123.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.05,
close_rate=0.06,
close_profit=0.01,
exchange='bittrex',
is_open=False,
)
o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy')
trade.orders.append(o)
o = Order.parse_from_ccxt_object(mock_order_3_sell(), 'XRP/BTC', 'sell')
trade.orders.append(o)
return trade
def mock_order_4():
return {
'id': 'prod_buy_12345',
'symbol': 'ETC/BTC',
'status': 'open',
'side': 'buy',
'type': 'limit',
'price': 0.123,
'amount': 123.0,
'filled': 0.0,
'remaining': 123.0,
}
def mock_trade_4(fee):
"""
Simulate prod entry
"""
trade = Trade(
pair='ETC/BTC',
stake_amount=0.001,
amount=123.0,
amount_requested=124.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
open_order_id='prod_buy_12345',
strategy='DefaultStrategy',
)
o = Order.parse_from_ccxt_object(mock_order_4(), 'ETC/BTC', 'buy')
trade.orders.append(o)
return trade
def mock_order_5():
return {
'id': 'prod_buy_3455',
'symbol': 'XRP/BTC',
'status': 'closed',
'side': 'buy',
'type': 'limit',
'price': 0.123,
'amount': 123.0,
'filled': 123.0,
'remaining': 0.0,
}
def mock_order_5_stoploss():
return {
'id': 'prod_stoploss_3455',
'symbol': 'XRP/BTC',
'status': 'open',
'side': 'sell',
'type': 'stop_loss_limit',
'price': 0.123,
'amount': 123.0,
'filled': 0.0,
'remaining': 123.0,
}
def mock_trade_5(fee):
"""
Simulate prod entry with stoploss
"""
trade = Trade(
pair='XRP/BTC',
stake_amount=0.001,
amount=123.0,
amount_requested=124.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.123,
exchange='bittrex',
strategy='SampleStrategy',
stoploss_order_id='prod_stoploss_3455'
)
o = Order.parse_from_ccxt_object(mock_order_5(), 'XRP/BTC', 'buy')
trade.orders.append(o)
o = Order.parse_from_ccxt_object(mock_order_5_stoploss(), 'XRP/BTC', 'stoploss')
trade.orders.append(o)
return trade
def mock_order_6():
return {
'id': 'prod_buy_6',
'symbol': 'LTC/BTC',
'status': 'closed',
'side': 'buy',
'type': 'limit',
'price': 0.15,
'amount': 2.0,
'filled': 2.0,
'remaining': 0.0,
}
def mock_order_6_sell():
return {
'id': 'prod_sell_6',
'symbol': 'LTC/BTC',
'status': 'open',
'side': 'sell',
'type': 'limit',
'price': 0.20,
'amount': 2.0,
'filled': 0.0,
'remaining': 2.0,
}
def mock_trade_6(fee):
"""
Simulate prod entry with open sell order
"""
trade = Trade(
pair='LTC/BTC',
stake_amount=0.001,
amount=2.0,
amount_requested=2.0,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_rate=0.15,
exchange='bittrex',
strategy='SampleStrategy',
open_order_id="prod_sell_6",
)
o = Order.parse_from_ccxt_object(mock_order_6(), 'LTC/BTC', 'buy')
trade.orders.append(o)
o = Order.parse_from_ccxt_object(mock_order_6_sell(), 'LTC/BTC', 'sell')
trade.orders.append(o)
return trade

3
tests/data/test_btanalysis.py
View File

@ -20,6 +20,7 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS,
from freqtrade.data.history import load_data, load_pair_history from freqtrade.data.history import load_data, load_pair_history
from freqtrade.optimize.backtesting import BacktestResult from freqtrade.optimize.backtesting import BacktestResult
from tests.conftest import create_mock_trades from tests.conftest import create_mock_trades
from tests.conftest_trades import MOCK_TRADE_COUNT
def test_get_latest_backtest_filename(testdatadir, mocker): def test_get_latest_backtest_filename(testdatadir, mocker):
@ -110,7 +111,7 @@ def test_load_trades_from_db(default_conf, fee, mocker):
trades = load_trades_from_db(db_url=default_conf['db_url']) trades = load_trades_from_db(db_url=default_conf['db_url'])
assert init_mock.call_count == 1 assert init_mock.call_count == 1
assert len(trades) == 4 assert len(trades) == MOCK_TRADE_COUNT
assert isinstance(trades, DataFrame) assert isinstance(trades, DataFrame)
assert "pair" in trades.columns assert "pair" in trades.columns
assert "open_date" in trades.columns assert "open_date" in trades.columns

46
tests/exchange/test_exchange.py
View File

@ -1,5 +1,3 @@
# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement
# pragma pylint: disable=protected-access
import copy import copy
import logging import logging
from datetime import datetime, timezone from datetime import datetime, timezone
@ -15,7 +13,8 @@ from freqtrade.exceptions import (DDosProtection, DependencyException,
InvalidOrderException, OperationalException, InvalidOrderException, OperationalException,
TemporaryError) TemporaryError)
from freqtrade.exchange import Binance, Exchange, Kraken from freqtrade.exchange import Binance, Exchange, Kraken
from freqtrade.exchange.common import API_RETRY_COUNT, calculate_backoff from freqtrade.exchange.common import (API_RETRY_COUNT, API_FETCH_ORDER_RETRY_COUNT,
calculate_backoff)
from freqtrade.exchange.exchange import (market_is_active, from freqtrade.exchange.exchange import (market_is_active,
timeframe_to_minutes, timeframe_to_minutes,
timeframe_to_msecs, timeframe_to_msecs,
@ -808,7 +807,7 @@ def test_dry_run_order(default_conf, mocker, side, exchange_name):
assert f'dry_run_{side}_' in order["id"] assert f'dry_run_{side}_' in order["id"]
assert order["side"] == side assert order["side"] == side
assert order["type"] == "limit" assert order["type"] == "limit"
assert order["pair"] == "ETH/BTC" assert order["symbol"] == "ETH/BTC"
@pytest.mark.parametrize("side", [ @pytest.mark.parametrize("side", [
@ -1766,7 +1765,7 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name):
cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC') cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC')
assert order['id'] == cancel_order['id'] assert order['id'] == cancel_order['id']
assert order['amount'] == cancel_order['amount'] assert order['amount'] == cancel_order['amount']
assert order['pair'] == cancel_order['pair'] assert order['symbol'] == cancel_order['symbol']
assert cancel_order['status'] == 'canceled' assert cancel_order['status'] == 'canceled'
@ -1903,12 +1902,14 @@ def test_fetch_order(default_conf, mocker, exchange_name):
# Ensure backoff is called # Ensure backoff is called
assert tm.call_args_list[0][0][0] == 1 assert tm.call_args_list[0][0][0] == 1
assert tm.call_args_list[1][0][0] == 2 assert tm.call_args_list[1][0][0] == 2
assert tm.call_args_list[2][0][0] == 5 if API_FETCH_ORDER_RETRY_COUNT > 2:
assert tm.call_args_list[3][0][0] == 10 assert tm.call_args_list[2][0][0] == 5
assert api_mock.fetch_order.call_count == 6 if API_FETCH_ORDER_RETRY_COUNT > 3:
assert tm.call_args_list[3][0][0] == 10
assert api_mock.fetch_order.call_count == API_FETCH_ORDER_RETRY_COUNT + 1
ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
'fetch_order', 'fetch_order', retries=6, 'fetch_order', 'fetch_order', retries=API_FETCH_ORDER_RETRY_COUNT + 1,
order_id='_', pair='TKN/BTC') order_id='_', pair='TKN/BTC')
@ -1941,10 +1942,35 @@ def test_fetch_stoploss_order(default_conf, mocker, exchange_name):
ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name,
'fetch_stoploss_order', 'fetch_order', 'fetch_stoploss_order', 'fetch_order',
retries=6, retries=API_FETCH_ORDER_RETRY_COUNT + 1,
order_id='_', pair='TKN/BTC') order_id='_', pair='TKN/BTC')
def test_fetch_order_or_stoploss_order(default_conf, mocker):
exchange = get_patched_exchange(mocker, default_conf, id='binance')
fetch_order_mock = MagicMock()
fetch_stoploss_order_mock = MagicMock()
mocker.patch.multiple('freqtrade.exchange.Exchange',
fetch_order=fetch_order_mock,
fetch_stoploss_order=fetch_stoploss_order_mock,
)
exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', False)
assert fetch_order_mock.call_count == 1
assert fetch_order_mock.call_args_list[0][0][0] == '1234'
assert fetch_order_mock.call_args_list[0][0][1] == 'ETH/BTC'
assert fetch_stoploss_order_mock.call_count == 0
fetch_order_mock.reset_mock()
fetch_stoploss_order_mock.reset_mock()
exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', True)
assert fetch_order_mock.call_count == 0
assert fetch_stoploss_order_mock.call_count == 1
assert fetch_stoploss_order_mock.call_args_list[0][0][0] == '1234'
assert fetch_stoploss_order_mock.call_args_list[0][0][1] == 'ETH/BTC'
@pytest.mark.parametrize("exchange_name", EXCHANGES) @pytest.mark.parametrize("exchange_name", EXCHANGES)
def test_name(default_conf, mocker, exchange_name): def test_name(default_conf, mocker, exchange_name):
exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)

5
tests/exchange/test_ftx.py
View File

@ -1,5 +1,3 @@
# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement
# pragma pylint: disable=protected-access
from random import randint from random import randint
from unittest.mock import MagicMock from unittest.mock import MagicMock
@ -7,6 +5,7 @@ import ccxt
import pytest import pytest
from freqtrade.exceptions import DependencyException, InvalidOrderException from freqtrade.exceptions import DependencyException, InvalidOrderException
from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT
from tests.conftest import get_patched_exchange from tests.conftest import get_patched_exchange
from .test_exchange import ccxt_exceptionhandlers from .test_exchange import ccxt_exceptionhandlers
@ -154,5 +153,5 @@ def test_fetch_stoploss_order(default_conf, mocker):
ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'ftx', ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'ftx',
'fetch_stoploss_order', 'fetch_orders', 'fetch_stoploss_order', 'fetch_orders',
retries=6, retries=API_FETCH_ORDER_RETRY_COUNT + 1,
order_id='_', pair='TKN/BTC') order_id='_', pair='TKN/BTC')

2
tests/exchange/test_kraken.py
View File

@ -1,5 +1,3 @@
# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement
# pragma pylint: disable=protected-access
from random import randint from random import randint
from unittest.mock import MagicMock from unittest.mock import MagicMock

7
tests/rpc/test_rpc.py
View File

@ -313,7 +313,6 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog):
with pytest.raises(RPCException, match='invalid argument'): with pytest.raises(RPCException, match='invalid argument'):
rpc._rpc_delete('200') rpc._rpc_delete('200')
create_mock_trades(fee)
trades = Trade.query.all() trades = Trade.query.all()
trades[1].stoploss_order_id = '1234' trades[1].stoploss_order_id = '1234'
trades[2].stoploss_order_id = '1234' trades[2].stoploss_order_id = '1234'
@ -717,11 +716,13 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None:
mocker.patch( mocker.patch(
'freqtrade.exchange.Exchange.fetch_order', 'freqtrade.exchange.Exchange.fetch_order',
side_effect=[{ side_effect=[{
'id': '1234',
'status': 'open', 'status': 'open',
'type': 'limit', 'type': 'limit',
'side': 'buy', 'side': 'buy',
'filled': filled_amount 'filled': filled_amount
}, { }, {
'id': '1234',
'status': 'closed', 'status': 'closed',
'type': 'limit', 'type': 'limit',
'side': 'buy', 'side': 'buy',
@ -837,10 +838,10 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None:
assert counts["current"] == 1 assert counts["current"] == 1
def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None: def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order_open) -> None:
default_conf['forcebuy_enable'] = True default_conf['forcebuy_enable'] = True
mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) buy_mm = MagicMock(return_value=limit_buy_order_open)
mocker.patch.multiple( mocker.patch.multiple(
'freqtrade.exchange.Exchange', 'freqtrade.exchange.Exchange',
get_balances=MagicMock(return_value=ticker), get_balances=MagicMock(return_value=ticker),

2
tests/rpc/test_rpc_telegram.py
View File

@ -253,7 +253,6 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None:
mocker.patch.multiple( mocker.patch.multiple(
'freqtrade.exchange.Exchange', 'freqtrade.exchange.Exchange',
fetch_ticker=ticker, fetch_ticker=ticker,
buy=MagicMock(return_value={'id': 'mocked_order_id'}),
get_fee=fee, get_fee=fee,
) )
msg_mock = MagicMock() msg_mock = MagicMock()
@ -1007,7 +1006,6 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None:
mocker.patch.multiple( mocker.patch.multiple(
'freqtrade.exchange.Exchange', 'freqtrade.exchange.Exchange',
fetch_ticker=ticker, fetch_ticker=ticker,
buy=MagicMock(return_value={'id': 'mocked_order_id'}),
get_fee=fee, get_fee=fee,
) )
freqtradebot = get_patched_freqtradebot(mocker, default_conf) freqtradebot = get_patched_freqtradebot(mocker, default_conf)

531
tests/test_freqtradebot.py
View File

File diff suppressed because it is too large Load Diff

208
tests/test_persistence.py
View File

@ -7,9 +7,9 @@ import pytest
from sqlalchemy import create_engine from sqlalchemy import create_engine
from freqtrade import constants from freqtrade import constants
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.persistence import Trade, clean_dry_run_db, init from freqtrade.persistence import Order, Trade, clean_dry_run_db, init
from tests.conftest import log_has, create_mock_trades from tests.conftest import create_mock_trades, log_has, log_has_re
def test_init_create_session(default_conf): def test_init_create_session(default_conf):
@ -22,7 +22,7 @@ def test_init_create_session(default_conf):
def test_init_custom_db_url(default_conf, mocker): def test_init_custom_db_url(default_conf, mocker):
# Update path to a value other than default, but still in-memory # Update path to a value other than default, but still in-memory
default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'})
create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
init(default_conf['db_url'], default_conf['dry_run']) init(default_conf['db_url'], default_conf['dry_run'])
assert create_engine_mock.call_count == 1 assert create_engine_mock.call_count == 1
@ -40,7 +40,7 @@ def test_init_prod_db(default_conf, mocker):
default_conf.update({'dry_run': False}) default_conf.update({'dry_run': False})
default_conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) default_conf.update({'db_url': constants.DEFAULT_DB_PROD_URL})
create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
init(default_conf['db_url'], default_conf['dry_run']) init(default_conf['db_url'], default_conf['dry_run'])
assert create_engine_mock.call_count == 1 assert create_engine_mock.call_count == 1
@ -51,7 +51,7 @@ def test_init_dryrun_db(default_conf, mocker):
default_conf.update({'dry_run': True}) default_conf.update({'dry_run': True})
default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL})
create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
init(default_conf['db_url'], default_conf['dry_run']) init(default_conf['db_url'], default_conf['dry_run'])
assert create_engine_mock.call_count == 1 assert create_engine_mock.call_count == 1
@ -93,6 +93,8 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
stake_amount=0.001, stake_amount=0.001,
open_rate=0.01, open_rate=0.01,
amount=5, amount=5,
is_open=True,
open_date=arrow.utcnow().datetime,
fee_open=fee.return_value, fee_open=fee.return_value,
fee_close=fee.return_value, fee_close=fee.return_value,
exchange='bittrex', exchange='bittrex',
@ -107,9 +109,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
assert trade.open_rate == 0.00001099 assert trade.open_rate == 0.00001099
assert trade.close_profit is None assert trade.close_profit is None
assert trade.close_date is None assert trade.close_date is None
assert log_has("LIMIT_BUY has been fulfilled for Trade(id=2, " assert log_has_re(r"LIMIT_BUY has been fulfilled for Trade\(id=2, "
"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).",
caplog) caplog)
caplog.clear() caplog.clear()
trade.open_order_id = 'something' trade.open_order_id = 'something'
@ -118,9 +120,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
assert trade.close_rate == 0.00001173 assert trade.close_rate == 0.00001173
assert trade.close_profit == 0.06201058 assert trade.close_profit == 0.06201058
assert trade.close_date is not None assert trade.close_date is not None
assert log_has("LIMIT_SELL has been fulfilled for Trade(id=2, " assert log_has_re(r"LIMIT_SELL has been fulfilled for Trade\(id=2, "
"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).",
caplog) caplog)
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
@ -131,8 +133,10 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog):
stake_amount=0.001, stake_amount=0.001,
amount=5, amount=5,
open_rate=0.01, open_rate=0.01,
is_open=True,
fee_open=fee.return_value, fee_open=fee.return_value,
fee_close=fee.return_value, fee_close=fee.return_value,
open_date=arrow.utcnow().datetime,
exchange='bittrex', exchange='bittrex',
) )
@ -142,20 +146,21 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog):
assert trade.open_rate == 0.00004099 assert trade.open_rate == 0.00004099
assert trade.close_profit is None assert trade.close_profit is None
assert trade.close_date is None assert trade.close_date is None
assert log_has("MARKET_BUY has been fulfilled for Trade(id=1, " assert log_has_re(r"MARKET_BUY has been fulfilled for Trade\(id=1, "
"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).",
caplog) caplog)
caplog.clear() caplog.clear()
trade.is_open = True
trade.open_order_id = 'something' trade.open_order_id = 'something'
trade.update(market_sell_order) trade.update(market_sell_order)
assert trade.open_order_id is None assert trade.open_order_id is None
assert trade.close_rate == 0.00004173 assert trade.close_rate == 0.00004173
assert trade.close_profit == 0.01297561 assert trade.close_profit == 0.01297561
assert trade.close_date is not None assert trade.close_date is not None
assert log_has("MARKET_SELL has been fulfilled for Trade(id=1, " assert log_has_re(r"MARKET_SELL has been fulfilled for Trade\(id=1, "
"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).",
caplog) caplog)
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
@ -184,6 +189,36 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee):
assert trade.calc_profit_ratio() == 0.06201058 assert trade.calc_profit_ratio() == 0.06201058
@pytest.mark.usefixtures("init_persistence")
def test_trade_close(limit_buy_order, limit_sell_order, fee):
trade = Trade(
pair='ETH/BTC',
stake_amount=0.001,
open_rate=0.01,
amount=5,
is_open=True,
fee_open=fee.return_value,
fee_close=fee.return_value,
open_date=arrow.Arrow(2020, 2, 1, 15, 5, 1).datetime,
exchange='bittrex',
)
assert trade.close_profit is None
assert trade.close_date is None
assert trade.is_open is True
trade.close(0.02)
assert trade.is_open is False
assert trade.close_profit == 0.99002494
assert trade.close_date is not None
new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime,
assert trade.close_date != new_date
# Close should NOT update close_date if the trade has been closed already
assert trade.is_open is False
trade.close_date = new_date
trade.close(0.02)
assert trade.close_date == new_date
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
def test_calc_close_trade_price_exception(limit_buy_order, fee): def test_calc_close_trade_price_exception(limit_buy_order, fee):
trade = Trade( trade = Trade(
@ -421,9 +456,9 @@ def test_migrate_old(mocker, default_conf, fee):
PRIMARY KEY (id), PRIMARY KEY (id),
CHECK (is_open IN (0, 1)) CHECK (is_open IN (0, 1))
);""" );"""
insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee,
open_rate, stake_amount, amount, open_date) open_rate, stake_amount, amount, open_date)
VALUES ('BITTREX', 'BTC_ETC', 1, {fee}, VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee},
0.00258580, {stake}, {amount}, 0.00258580, {stake}, {amount},
'2017-11-28 12:44:24.000000') '2017-11-28 12:44:24.000000')
""".format(fee=fee.return_value, """.format(fee=fee.return_value,
@ -440,7 +475,7 @@ def test_migrate_old(mocker, default_conf, fee):
amount=amount amount=amount
) )
engine = create_engine('sqlite://') engine = create_engine('sqlite://')
mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
# Create table using the old format # Create table using the old format
engine.execute(create_table_old) engine.execute(create_table_old)
@ -481,6 +516,12 @@ def test_migrate_old(mocker, default_conf, fee):
assert pytest.approx(trade.close_profit_abs) == trade.calc_profit() assert pytest.approx(trade.close_profit_abs) == trade.calc_profit()
assert trade.sell_order_status is None assert trade.sell_order_status is None
# Should've created one order
assert len(Order.query.all()) == 1
order = Order.query.first()
assert order.order_id == '123123'
assert order.ft_order_side == 'buy'
def test_migrate_new(mocker, default_conf, fee, caplog): def test_migrate_new(mocker, default_conf, fee, caplog):
""" """
@ -509,22 +550,25 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
sell_reason VARCHAR, sell_reason VARCHAR,
strategy VARCHAR, strategy VARCHAR,
ticker_interval INTEGER, ticker_interval INTEGER,
stoploss_order_id VARCHAR,
PRIMARY KEY (id), PRIMARY KEY (id),
CHECK (is_open IN (0, 1)) CHECK (is_open IN (0, 1))
);""" );"""
insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
open_rate, stake_amount, amount, open_date, open_rate, stake_amount, amount, open_date,
stop_loss, initial_stop_loss, max_rate, ticker_interval) stop_loss, initial_stop_loss, max_rate, ticker_interval,
open_order_id, stoploss_order_id)
VALUES ('binance', 'ETC/BTC', 1, {fee}, VALUES ('binance', 'ETC/BTC', 1, {fee},
0.00258580, {stake}, {amount}, 0.00258580, {stake}, {amount},
'2019-11-28 12:44:24.000000', '2019-11-28 12:44:24.000000',
0.0, 0.0, 0.0, '5m') 0.0, 0.0, 0.0, '5m',
'buy_order', 'stop_order_id222')
""".format(fee=fee.return_value, """.format(fee=fee.return_value,
stake=default_conf.get("stake_amount"), stake=default_conf.get("stake_amount"),
amount=amount amount=amount
) )
engine = create_engine('sqlite://') engine = create_engine('sqlite://')
mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
# Create table using the old format # Create table using the old format
engine.execute(create_table_old) engine.execute(create_table_old)
@ -558,14 +602,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
assert trade.sell_reason is None assert trade.sell_reason is None
assert trade.strategy is None assert trade.strategy is None
assert trade.timeframe == '5m' assert trade.timeframe == '5m'
assert trade.stoploss_order_id is None assert trade.stoploss_order_id == 'stop_order_id222'
assert trade.stoploss_last_update is None assert trade.stoploss_last_update is None
assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak1", caplog)
assert log_has("trying trades_bak2", caplog) assert log_has("trying trades_bak2", caplog)
assert log_has("Running database migration - backup available as trades_bak2", caplog) assert log_has("Running database migration for trades - backup: trades_bak2", caplog)
assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.open_trade_price == trade._calc_open_trade_price()
assert trade.close_profit_abs is None assert trade.close_profit_abs is None
assert log_has("Moving open orders to Orders table.", caplog)
orders = Order.query.all()
assert len(orders) == 2
assert orders[0].order_id == 'buy_order'
assert orders[0].ft_order_side == 'buy'
assert orders[1].order_id == 'stop_order_id222'
assert orders[1].ft_order_side == 'stoploss'
def test_migrate_mid_state(mocker, default_conf, fee, caplog): def test_migrate_mid_state(mocker, default_conf, fee, caplog):
""" """
@ -601,7 +654,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
amount=amount amount=amount
) )
engine = create_engine('sqlite://') engine = create_engine('sqlite://')
mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
# Create table using the old format # Create table using the old format
engine.execute(create_table_old) engine.execute(create_table_old)
@ -626,7 +679,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
assert trade.initial_stop_loss == 0.0 assert trade.initial_stop_loss == 0.0
assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.open_trade_price == trade._calc_open_trade_price()
assert log_has("trying trades_bak0", caplog) assert log_has("trying trades_bak0", caplog)
assert log_has("Running database migration - backup available as trades_bak0", caplog) assert log_has("Running database migration for trades - backup: trades_bak0", caplog)
def test_adjust_stop_loss(fee): def test_adjust_stop_loss(fee):
@ -713,10 +766,10 @@ def test_adjust_min_max_rates(fee):
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
def test_get_open(default_conf, fee): def test_get_open(fee):
create_mock_trades(fee) create_mock_trades(fee)
assert len(Trade.get_open_trades()) == 2 assert len(Trade.get_open_trades()) == 4
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
@ -986,7 +1039,7 @@ def test_total_open_trades_stakes(fee):
assert res == 0 assert res == 0
create_mock_trades(fee) create_mock_trades(fee)
res = Trade.total_open_trades_stakes() res = Trade.total_open_trades_stakes()
assert res == 0.002 assert res == 0.004
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
@ -1012,3 +1065,96 @@ def test_get_best_pair(fee):
assert len(res) == 2 assert len(res) == 2
assert res[0] == 'XRP/BTC' assert res[0] == 'XRP/BTC'
assert res[1] == 0.01 assert res[1] == 0.01
@pytest.mark.usefixtures("init_persistence")
def test_update_order_from_ccxt():
# Most basic order return (only has orderid)
o = Order.parse_from_ccxt_object({'id': '1234'}, 'ETH/BTC', 'buy')
assert isinstance(o, Order)
assert o.ft_pair == 'ETH/BTC'
assert o.ft_order_side == 'buy'
assert o.order_id == '1234'
assert o.ft_is_open
ccxt_order = {
'id': '1234',
'side': 'buy',
'symbol': 'ETH/BTC',
'type': 'limit',
'price': 1234.5,
'amount': 20.0,
'filled': 9,
'remaining': 11,
'status': 'open',
'timestamp': 1599394315123
}
o = Order.parse_from_ccxt_object(ccxt_order, 'ETH/BTC', 'buy')
assert isinstance(o, Order)
assert o.ft_pair == 'ETH/BTC'
assert o.ft_order_side == 'buy'
assert o.order_id == '1234'
assert o.order_type == 'limit'
assert o.price == 1234.5
assert o.filled == 9
assert o.remaining == 11
assert o.order_date is not None
assert o.ft_is_open
assert o.order_filled_date is None
# Order has been closed
ccxt_order.update({'filled': 20.0, 'remaining': 0.0, 'status': 'closed'})
o.update_from_ccxt_object(ccxt_order)
assert o.filled == 20.0
assert o.remaining == 0.0
assert not o.ft_is_open
assert o.order_filled_date is not None
ccxt_order.update({'id': 'somethingelse'})
with pytest.raises(DependencyException, match=r"Order-id's don't match"):
o.update_from_ccxt_object(ccxt_order)
@pytest.mark.usefixtures("init_persistence")
def test_select_order(fee):
create_mock_trades(fee)
trades = Trade.get_trades().all()
# Open buy order, no sell order
order = trades[0].select_order('buy', True)
assert order is None
order = trades[0].select_order('buy', False)
assert order is not None
order = trades[0].select_order('sell', None)
assert order is None
# closed buy order, and open sell order
order = trades[1].select_order('buy', True)
assert order is None
order = trades[1].select_order('buy', False)
assert order is not None
order = trades[1].select_order('buy', None)
assert order is not None
order = trades[1].select_order('sell', True)
assert order is None
order = trades[1].select_order('sell', False)
assert order is not None
# Has open buy order
order = trades[3].select_order('buy', True)
assert order is not None
order = trades[3].select_order('buy', False)
assert order is None
# Open sell order
order = trades[4].select_order('buy', True)
assert order is None
order = trades[4].select_order('buy', False)
assert order is not None
order = trades[4].select_order('sell', True)
assert order is not None
assert order.ft_order_side == 'stoploss'
order = trades[4].select_order('sell', False)
assert order is None