diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16bf1959d..e8bc01fa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ on: branches: - master - develop - - github_actions_tests tags: release: types: [published] diff --git a/docs/developer.md b/docs/developer.md index 111c7a96f..9d47258b7 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -120,6 +120,8 @@ Below is an outline of exception inheritance hierarchy: | +---+ InvalidOrderException | | | +---+ RetryableOrderError +| | +| +---+ InsufficientFundsError | +---+ StrategyError ``` diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index dd09c4c05..ac234a72e 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -136,7 +136,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona start = None if timerange: if timerange.starttype == 'date': - # TODO: convert to date for conversion start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) # Intentionally don't pass timerange in - since we need to load the full dataset. diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py index e2bc969a9..caf970606 100644 --- a/freqtrade/exceptions.py +++ b/freqtrade/exceptions.py @@ -51,6 +51,13 @@ class RetryableOrderError(InvalidOrderException): """ +class InsufficientFundsError(InvalidOrderException): + """ + This error is used when there are not enough funds available on the exchange + to create an order. + """ + + class TemporaryError(ExchangeError): """ Temporary network or exchange related error. diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index f2fe1d6ad..d7da34482 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -4,7 +4,7 @@ from typing import Dict import ccxt -from freqtrade.exceptions import (DDosProtection, ExchangeError, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange @@ -80,7 +80,7 @@ class Binance(Exchange): 'stop price: %s. limit: %s', pair, stop_price, rate) return order except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to sell amount {amount} at rate {rate}. ' f'Message: {e}') from e diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 7f6dfe0eb..9abd42aa7 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -9,7 +9,11 @@ from freqtrade.exceptions import (DDosProtection, RetryableOrderError, logger = logging.getLogger(__name__) +# Maximum default retry count. +# Functions are always called RETRY_COUNT + 1 times (for the original call) API_RETRY_COUNT = 4 +API_FETCH_ORDER_RETRY_COUNT = 5 + BAD_EXCHANGES = { "bitmex": "Various reasons.", "bitstamp": "Does not provide history. " diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b89da14eb..aac45967d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -8,7 +8,6 @@ import logging from copy import deepcopy from datetime import datetime, timezone from math import ceil -from random import randint from typing import Any, Dict, List, Optional, Tuple import arrow @@ -21,9 +20,11 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DDosProtection, ExchangeError, + InsufficientFundsError, InvalidOrderException, OperationalException, RetryableOrderError, TemporaryError) -from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async +from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, + BAD_EXCHANGES, retrier, retrier_async) from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 CcxtModuleType = Any @@ -487,11 +488,11 @@ class Exchange: def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: - order_id = f'dry_run_{side}_{randint(0, 10**6)}' + order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) dry_order = { - "id": order_id, - 'pair': pair, + 'id': order_id, + 'symbol': pair, 'price': rate, 'average': rate, 'amount': _amount, @@ -500,6 +501,7 @@ class Exchange: 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), + 'timestamp': int(arrow.utcnow().timestamp * 1000), 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} @@ -538,7 +540,7 @@ class Exchange: amount, rate_for_order, params) except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e @@ -1027,7 +1029,7 @@ class Exchange: return order - @retrier(retries=5) + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: try: @@ -1056,6 +1058,17 @@ class Exchange: # Assign method to fetch_stoploss_order to allow easy overriding in other classes fetch_stoploss_order = fetch_order + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + @retrier def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: """ diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 441d97215..a5ee0c408 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -4,11 +4,11 @@ from typing import Any, Dict import ccxt -from freqtrade.exceptions import (DDosProtection, ExchangeError, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange -from freqtrade.exchange.common import retrier +from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier logger = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class Ftx(Exchange): 'stop price: %s.', pair, stop_price) return order except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Message: {e}') from e @@ -88,7 +88,7 @@ class Ftx(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e - @retrier(retries=5) + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: try: diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 52b992dcc..e6b5da88e 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -4,7 +4,7 @@ from typing import Any, Dict import ccxt -from freqtrade.exceptions import (DDosProtection, ExchangeError, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange @@ -98,7 +98,7 @@ class Kraken(Exchange): 'stop price: %s.', pair, stop_price) return order except ccxt.InsufficientFunds as e: - raise ExchangeError( + raise InsufficientFundsError( f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Message: {e}') from e diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3c6e6b726..eec09a17c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -17,12 +17,12 @@ from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.exceptions import (DependencyException, ExchangeError, +from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.persistence import Trade +from freqtrade.persistence import Order, Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -134,6 +134,10 @@ class FreqtradeBot: # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) + # Only update open orders on startup + # This will update the database after the initial migration + self.update_open_orders() + def process(self) -> None: """ Queries the persistence layer for open trades and handles them, @@ -144,6 +148,8 @@ class FreqtradeBot: # Check whether markets have to be reloaded and reload them when it's needed self.exchange.reload_markets() + self.update_closed_trades_without_assigned_fees() + # Query trades from persistence layer trades = Trade.get_open_trades() @@ -227,6 +233,104 @@ class FreqtradeBot: open_trades = len(Trade.get_open_trades()) return max(0, self.config['max_open_trades'] - open_trades) + def update_open_orders(self): + """ + Updates open orders based on order list kept in the database. + Mainly updates the state of orders - but may also close trades + """ + orders = Order.get_open_orders() + logger.info(f"Updating {len(orders)} open orders.") + for order in orders: + try: + fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, + order.ft_order_side == 'stoploss') + + self.update_trade_state(order.trade, order.order_id, fo) + + except ExchangeError as e: + logger.warning(f"Error updating Order {order.order_id} due to {e}") + + def update_closed_trades_without_assigned_fees(self): + """ + Update closed trades without close fees assigned. + Only acts when Orders are in the database, otherwise the last orderid is unknown. + """ + trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees() + for trade in trades: + + if not trade.is_open and not trade.fee_updated('sell'): + # Get sell fee + order = trade.select_order('sell', False) + if order: + logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id, + stoploss_order=order.ft_order_side == 'stoploss') + + trades: List[Trade] = Trade.get_open_trades_without_assigned_fees() + for trade in trades: + if trade.is_open and not trade.fee_updated('buy'): + order = trade.select_order('buy', False) + if order: + logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id) + + def handle_insufficient_funds(self, trade: Trade): + """ + Determine if we ever opened a sell order for this trade. + If not, try update buy fees - otherwise "refind" the open order we obviously lost. + """ + sell_order = trade.select_order('sell', None) + if sell_order: + self.refind_lost_order(trade) + else: + self.reupdate_buy_order_fees(trade) + + def reupdate_buy_order_fees(self, trade: Trade): + """ + Get buy order from database, and try to reupdate. + Handles trades where the initial fee-update did not work. + """ + logger.info(f"Trying to reupdate buy fees for {trade}") + order = trade.select_order('buy', False) + if order: + logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id) + + def refind_lost_order(self, trade): + """ + Try refinding a lost trade. + Only used when InsufficientFunds appears on sell orders (stoploss or sell). + Tries to walk the stored orders and sell them off eventually. + """ + logger.info(f"Trying to refind lost order for {trade}") + for order in trade.orders: + logger.info(f"Trying to refind {order}") + fo = None + if not order.ft_is_open: + logger.debug(f"Order {order} is no longer open.") + continue + if order.ft_order_side == 'buy': + # Skip buy side - this is handled by reupdate_buy_order_fees + continue + try: + fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, + order.ft_order_side == 'stoploss') + if order.ft_order_side == 'stoploss': + if fo and fo['status'] == 'open': + # Assume this as the open stoploss order + trade.stoploss_order_id = order.order_id + elif order.ft_order_side == 'sell': + if fo and fo['status'] == 'open': + # Assume this as the open order + trade.open_order_id = order.order_id + if fo: + logger.info(f"Found {order} for trade {trade}.jj") + self.update_trade_state(trade, order.order_id, fo, + stoploss_order=order.ft_order_side == 'stoploss') + + except ExchangeError: + logger.warning(f"Error updating {order.order_id}.") + # # BUY / enter positions / open trades logic and methods # @@ -528,6 +632,7 @@ class FreqtradeBot: order = self.exchange.buy(pair=pair, ordertype=order_type, amount=amount, rate=buy_limit_requested, time_in_force=time_in_force) + order_obj = Order.parse_from_ccxt_object(order, pair, 'buy') order_id = order['id'] order_status = order.get('status', None) @@ -556,7 +661,6 @@ class FreqtradeBot: stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') - order_id = None # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': @@ -581,10 +685,11 @@ class FreqtradeBot: strategy=self.strategy.get_strategy_name(), timeframe=timeframe_to_minutes(self.config['timeframe']) ) + trade.orders.append(order_obj) # Update fees if order is closed if order_status == 'closed': - self.update_trade_state(trade, order) + self.update_trade_state(trade, order_id, order) Trade.session.add(trade) Trade.session.flush() @@ -783,8 +888,16 @@ class FreqtradeBot: stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount, stop_price=stop_price, order_types=self.strategy.order_types) + + order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss') + trade.orders.append(order_obj) trade.stoploss_order_id = str(stoploss_order['id']) return True + except InsufficientFundsError as e: + logger.warning(f"Unable to place stoploss order {e}.") + # Try to figure out what went wrong + self.handle_insufficient_funds(trade) + except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') @@ -814,10 +927,14 @@ class FreqtradeBot: except InvalidOrderException as exception: logger.warning('Unable to fetch stoploss order: %s', exception) + if stoploss_order: + trade.update_order(stoploss_order) + # We check if stoploss order is fulfilled if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'): trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - self.update_trade_state(trade, stoploss_order, sl_order=True) + self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, + stoploss_order=True) # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['timeframe'])) @@ -869,10 +986,11 @@ class FreqtradeBot: update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: # cancelling the current stoploss on exchange first - logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) ' - 'in order to add another one ...', order['id']) + logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} " + f"(orderid:{order['id']}) in order to add another one ...") try: - self.exchange.cancel_stoploss_order(order['id'], trade.pair) + co = self.exchange.cancel_stoploss_order(order['id'], trade.pair) + trade.update_order(co) except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {order['id']} " f"for pair {trade.pair}") @@ -927,7 +1045,7 @@ class FreqtradeBot: logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) continue - fully_cancelled = self.update_trade_state(trade, order) + fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order) if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and ( fully_cancelled @@ -995,8 +1113,7 @@ class FreqtradeBot: if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): logger.info('Buy order fully cancelled. Removing %s from database.', trade) # if trade is not partially completed, just delete the trade - Trade.session.delete(trade) - Trade.session.flush() + trade.delete() was_trade_fully_canceled = True reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" else: @@ -1007,7 +1124,7 @@ class FreqtradeBot: # we need to fall back to the values from order if corder does not contain these keys. trade.amount = filled_amount trade.stake_amount = trade.amount * trade.open_rate - self.update_trade_state(trade, corder, trade.amount) + self.update_trade_state(trade, trade.open_order_id, corder) trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) @@ -1121,19 +1238,28 @@ class FreqtradeBot: logger.info(f"User requested abortion of selling {trade.pair}") return False - # Execute sell and update trade record - order = self.exchange.sell(pair=str(trade.pair), - ordertype=order_type, - amount=amount, rate=limit, - time_in_force=time_in_force - ) + try: + # Execute sell and update trade record + order = self.exchange.sell(pair=trade.pair, + ordertype=order_type, + amount=amount, rate=limit, + time_in_force=time_in_force + ) + except InsufficientFundsError as e: + logger.warning(f"Unable to place order {e}.") + # Try to figure out what went wrong + self.handle_insufficient_funds(trade) + return False + + order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell') + trade.orders.append(order_obj) trade.open_order_id = order['id'] trade.close_rate_requested = limit trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': - self.update_trade_state(trade, order) + self.update_trade_state(trade, trade.open_order_id, order) Trade.session.flush() # Lock pair for one candle to prevent immediate rebuys @@ -1230,30 +1356,35 @@ class FreqtradeBot: # Common update trade state methods # - def update_trade_state(self, trade: Trade, action_order: dict = None, - order_amount: float = None, sl_order: bool = False) -> bool: + def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, + stoploss_order: bool = False) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. + :param trade: Trade object of the trade we're analyzing + :param order_id: Order-id of the order we're analyzing + :param action_order: Already aquired order object :return: True if order has been cancelled without being filled partially, False otherwise """ - # Get order details for actual price per unit - if trade.open_order_id: - order_id = trade.open_order_id - elif trade.stoploss_order_id and sl_order: - order_id = trade.stoploss_order_id - else: + if not order_id: + logger.warning(f'Orderid for trade {trade} is empty.') return False + # Update trade with order values logger.info('Found open order for %s', trade) try: - order = action_order or self.exchange.fetch_order(order_id, trade.pair) + order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id, + trade.pair, + stoploss_order) except InvalidOrderException as exception: logger.warning('Unable to fetch order %s: %s', order_id, exception) return False + + trade.update_order(order) + # Try update amount (binance-fix) try: - new_amount = self.get_real_amount(trade, order, order_amount) + new_amount = self.get_real_amount(trade, order) if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount, abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount @@ -1291,7 +1422,7 @@ class FreqtradeBot: return real_amount return amount - def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: + def get_real_amount(self, trade: Trade, order: Dict) -> float: """ Detect and update trade fee. Calls trade.update_fee() uppon correct detection. @@ -1300,8 +1431,7 @@ class FreqtradeBot: :return: identical (or new) amount for the trade """ # Init variables - if order_amount is None: - order_amount = safe_value_fallback(order, 'filled', 'amount') + order_amount = safe_value_fallback(order, 'filled', 'amount') # Only run for closed orders if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': return order_amount @@ -1325,7 +1455,7 @@ class FreqtradeBot: """ fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. """ - trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, + trades = self.exchange.get_trades_for_order(order['id'], trade.pair, trade.open_date) if len(trades) == 0: diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py new file mode 100644 index 000000000..764856f2b --- /dev/null +++ b/freqtrade/persistence/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa: F401 + +from freqtrade.persistence.models import (Order, Trade, clean_dry_run_db, + cleanup, init) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py new file mode 100644 index 000000000..5089953b2 --- /dev/null +++ b/freqtrade/persistence/migrations.py @@ -0,0 +1,149 @@ +import logging +from typing import List + +from sqlalchemy import inspect + +logger = logging.getLogger(__name__) + + +def get_table_names_for_table(inspector, tabletype): + return [t for t in inspector.get_table_names() if t.startswith(tabletype)] + + +def has_column(columns: List, searchname: str) -> bool: + return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 + + +def get_column_def(columns: List, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + +def get_backup_name(tabs, backup_prefix: str): + table_back_name = backup_prefix + for i, table_back_name in enumerate(tabs): + table_back_name = f'{backup_prefix}{i}' + logger.debug(f'trying {table_back_name}') + + return table_back_name + + +def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List): + fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') + fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') + stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') + stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') + max_rate = get_column_def(cols, 'max_rate', '0.0') + min_rate = get_column_def(cols, 'min_rate', 'null') + sell_reason = get_column_def(cols, 'sell_reason', 'null') + strategy = get_column_def(cols, 'strategy', 'null') + # If ticker-interval existed use that, else null. + if has_column(cols, 'ticker_interval'): + timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') + else: + timeframe = get_column_def(cols, 'timeframe', 'null') + + open_trade_price = get_column_def(cols, 'open_trade_price', + f'amount * open_rate * (1 + {fee_open})') + close_profit_abs = get_column_def( + cols, 'close_profit_abs', + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') + amount_requested = get_column_def(cols, 'amount_requested', 'amount') + + # Schema migration necessary + engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute(f"""insert into trades + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, + stoploss_order_id, stoploss_last_update, + max_rate, min_rate, sell_reason, sell_order_status, strategy, + timeframe, open_trade_price, close_profit_abs + ) + select id, lower(exchange), + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, + {initial_stop_loss} initial_stop_loss, + {initial_stop_loss_pct} initial_stop_loss_pct, + {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, + {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, + {sell_order_status} sell_order_status, + {strategy} strategy, {timeframe} timeframe, + {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs + from {table_back_name} + """) + + +def migrate_open_orders_to_trades(engine): + engine.execute(""" + insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) + select id ft_trade_id, pair ft_pair, open_order_id, + case when close_rate_requested is null then 'buy' + else 'sell' end ft_order_side, 1 ft_is_open + from trades + where open_order_id is not null + union all + select id ft_trade_id, pair ft_pair, stoploss_order_id order_id, + 'stoploss' ft_order_side, 1 ft_is_open + from trades + where stoploss_order_id is not null + """) + + +def check_migrate(engine, decl_base, previous_tables) -> None: + """ + Checks if migration is necessary and migrates if necessary + """ + inspector = inspect(engine) + + cols = inspector.get_columns('trades') + tabs = get_table_names_for_table(inspector, 'trades') + table_back_name = get_backup_name(tabs, 'trades_bak') + + # Check for latest column + if not has_column(cols, 'amount_requested'): + logger.info(f'Running database migration for trades - backup: {table_back_name}') + migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) + # Reread columns - the above recreated the table! + inspector = inspect(engine) + cols = inspector.get_columns('trades') + + if 'orders' not in previous_tables: + logger.info('Moving open orders to Orders table.') + migrate_open_orders_to_trades(engine) + else: + pass + # Empty for now - as there is only one iteration of the orders table so far. + # table_back_name = get_backup_name(tabs, 'orders_bak') diff --git a/freqtrade/persistence.py b/freqtrade/persistence/models.py similarity index 74% rename from freqtrade/persistence.py rename to freqtrade/persistence/models.py index 9eebadd8d..816e23fd3 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence/models.py @@ -7,17 +7,19 @@ from decimal import Decimal from typing import Any, Dict, List, Optional import arrow -from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, - create_engine, desc, func, inspect) +from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, + String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Query +from sqlalchemy.orm import Query, relationship from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.schema import UniqueConstraint -from freqtrade.exceptions import OperationalException +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.misc import safe_value_fallback +from freqtrade.persistence.migrations import check_migrate logger = logging.getLogger(__name__) @@ -57,121 +59,18 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: # We should use the scoped_session object - not a seperately initialized version Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.query = Trade.session.query_property() + # Copy session attributes to order object too + Order.session = Trade.session + Order.query = Order.session.query_property() + previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) - check_migrate(engine) + check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables) # Clean dry_run DB if the db is not in-memory if clean_open_orders and db_url != 'sqlite://': clean_dry_run_db() -def has_column(columns: List, searchname: str) -> bool: - return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 - - -def get_column_def(columns: List, column: str, default: str) -> str: - return default if not has_column(columns, column) else column - - -def check_migrate(engine) -> None: - """ - Checks if migration is necessary and migrates if necessary - """ - inspector = inspect(engine) - - cols = inspector.get_columns('trades') - tabs = inspector.get_table_names() - table_back_name = 'trades_bak' - for i, table_back_name in enumerate(tabs): - table_back_name = f'trades_bak{i}' - logger.debug(f'trying {table_back_name}') - - # Check for latest column - if not has_column(cols, 'amount_requested'): - logger.info(f'Running database migration - backup available as {table_back_name}') - - fee_open = get_column_def(cols, 'fee_open', 'fee') - fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') - fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') - fee_close = get_column_def(cols, 'fee_close', 'fee') - fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') - fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') - open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') - close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') - stop_loss = get_column_def(cols, 'stop_loss', '0.0') - stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') - initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') - initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') - stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') - stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') - max_rate = get_column_def(cols, 'max_rate', '0.0') - min_rate = get_column_def(cols, 'min_rate', 'null') - sell_reason = get_column_def(cols, 'sell_reason', 'null') - strategy = get_column_def(cols, 'strategy', 'null') - # If ticker-interval existed use that, else null. - if has_column(cols, 'ticker_interval'): - timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') - else: - timeframe = get_column_def(cols, 'timeframe', 'null') - - open_trade_price = get_column_def(cols, 'open_trade_price', - f'amount * open_rate * (1 + {fee_open})') - close_profit_abs = get_column_def( - cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") - sell_order_status = get_column_def(cols, 'sell_order_status', 'null') - amount_requested = get_column_def(cols, 'amount_requested', 'amount') - - # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") - # let SQLAlchemy create the schema as required - _DECL_BASE.metadata.create_all(engine) - - # Copy data back - following the correct schema - engine.execute(f"""insert into trades - (id, exchange, pair, is_open, - fee_open, fee_open_cost, fee_open_currency, - fee_close, fee_close_cost, fee_open_currency, open_rate, - open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, amount_requested, open_date, close_date, open_order_id, - stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, - stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, sell_order_status, strategy, - timeframe, open_trade_price, close_profit_abs - ) - select id, lower(exchange), - case - when instr(pair, '_') != 0 then - substr(pair, instr(pair, '_') + 1) || '/' || - substr(pair, 1, instr(pair, '_') - 1) - else pair - end - pair, - is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, - {fee_open_currency} fee_open_currency, {fee_close} fee_close, - {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, - open_rate, {open_rate_requested} open_rate_requested, close_rate, - {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, - {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, - {initial_stop_loss} initial_stop_loss, - {initial_stop_loss_pct} initial_stop_loss_pct, - {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, - {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {sell_order_status} sell_order_status, - {strategy} strategy, {timeframe} timeframe, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs - from {table_back_name} - """) - - # Reread columns - the above recreated the table! - inspector = inspect(engine) - cols = inspector.get_columns('trades') - - def cleanup() -> None: """ Flushes all pending operations to disk. @@ -191,13 +90,117 @@ def clean_dry_run_db() -> None: trade.open_order_id = None +class Order(_DECL_BASE): + """ + Order database model + Keeps a record of all orders placed on the exchange + + One to many relationship with Trades: + - One trade can have many orders + - One Order can only be associated with one Trade + + Mirrors CCXT Order structure + """ + __tablename__ = 'orders' + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) + + id = Column(Integer, primary_key=True) + ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) + + trade = relationship("Trade", back_populates="orders") + + ft_order_side = Column(String, nullable=False) + ft_pair = Column(String, nullable=False) + ft_is_open = Column(Boolean, nullable=False, default=True, index=True) + + order_id = Column(String, nullable=False, index=True) + status = Column(String, nullable=True) + symbol = Column(String, nullable=True) + order_type = Column(String, nullable=True) + side = Column(String, nullable=True) + price = Column(Float, nullable=True) + amount = Column(Float, nullable=True) + filled = Column(Float, nullable=True) + remaining = Column(Float, nullable=True) + cost = Column(Float, nullable=True) + order_date = Column(DateTime, nullable=True, default=datetime.utcnow) + order_filled_date = Column(DateTime, nullable=True) + order_update_date = Column(DateTime, nullable=True) + + def __repr__(self): + + return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' + f'side={self.side}, order_type={self.order_type}, status={self.status})') + + def update_from_ccxt_object(self, order): + """ + Update Order from ccxt response + Only updates if fields are available from ccxt - + """ + if self.order_id != str(order['id']): + raise DependencyException("Order-id's don't match") + + self.status = order.get('status', self.status) + self.symbol = order.get('symbol', self.symbol) + self.order_type = order.get('type', self.order_type) + self.side = order.get('side', self.side) + self.price = order.get('price', self.price) + self.amount = order.get('amount', self.amount) + self.filled = order.get('filled', self.filled) + self.remaining = order.get('remaining', self.remaining) + self.cost = order.get('cost', self.cost) + if 'timestamp' in order and order['timestamp'] is not None: + self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) + + self.ft_is_open = True + if self.status in ('closed', 'canceled', 'cancelled'): + self.ft_is_open = False + if order.get('filled', 0) > 0: + self.order_filled_date = arrow.utcnow().datetime + self.order_update_date = arrow.utcnow().datetime + + @staticmethod + def update_orders(orders: List['Order'], order: Dict[str, Any]): + """ + Get all non-closed orders - useful when trying to batch-update orders + """ + filtered_orders = [o for o in orders if o.order_id == order['id']] + if filtered_orders: + oobj = filtered_orders[0] + oobj.update_from_ccxt_object(order) + else: + logger.warning(f"Did not find order for {order['id']}.") + + @staticmethod + def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order': + """ + Parse an order from a ccxt object and return a new order Object. + """ + o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair) + + o.update_from_ccxt_object(order) + return o + + @staticmethod + def get_open_orders() -> List['Order']: + """ + """ + return Order.query.filter(Order.ft_is_open.is_(True)).all() + + class Trade(_DECL_BASE): """ - Class used to define a trade structure + Trade database model. + Also handles updating and querying trades """ __tablename__ = 'trades' id = Column(Integer, primary_key=True) + + orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") + exchange = Column(String, nullable=False) pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) @@ -380,15 +383,18 @@ class Trade(_DECL_BASE): self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) self.recalc_open_trade_price() - logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) + if self.is_open: + logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None elif order_type in ('market', 'limit') and order['side'] == 'sell': + if self.is_open: + logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.') self.close(safe_value_fallback(order, 'average', 'price')) - logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self) elif order_type in ('stop_loss_limit', 'stop-loss', 'stop'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss - logger.info('%s is hit for %s.', order_type.upper(), self) + if self.is_open: + logger.info(f'{order_type.upper()} is hit for {self}.') self.close(order['average']) else: raise ValueError(f'Unknown order type: {order_type}') @@ -402,7 +408,7 @@ class Trade(_DECL_BASE): self.close_rate = Decimal(rate) self.close_profit = self.calc_profit_ratio() self.close_profit_abs = self.calc_profit() - self.close_date = datetime.utcnow() + self.close_date = self.close_date or datetime.utcnow() self.is_open = False self.sell_order_status = 'closed' self.open_order_id = None @@ -440,6 +446,17 @@ class Trade(_DECL_BASE): else: return False + def update_order(self, order: Dict) -> None: + Order.update_orders(self.orders, order) + + def delete(self) -> None: + + for order in self.orders: + Order.session.delete(order) + + Trade.session.delete(self) + Trade.session.flush() + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including open_fee. @@ -506,6 +523,21 @@ class Trade(_DECL_BASE): profit_ratio = (close_trade_price / self.open_trade_price) - 1 return float(f"{profit_ratio:.8f}") + def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: + """ + Finds latest order for this orderside and status + :param order_side: Side of the order (either 'buy' or 'sell') + :param is_open: Only search for open orders? + :return: latest Order object if it exists, else None + """ + orders = [o for o in self.orders if o.side == order_side] + if is_open is not None: + orders = [o for o in orders if o.ft_is_open == is_open] + if len(orders) > 0: + return orders[-1] + else: + return None + @staticmethod def get_trades(trade_filter=None) -> Query: """ @@ -537,6 +569,26 @@ class Trade(_DECL_BASE): """ return Trade.get_trades(Trade.open_order_id.isnot(None)).all() + @staticmethod + def get_open_trades_without_assigned_fees(): + """ + Returns all open trades which don't have open fees set correctly + """ + return Trade.get_trades([Trade.fee_open_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(True), + ]).all() + + @staticmethod + def get_sold_trades_without_assigned_fees(): + """ + Returns all closed trades which don't have fees set correctly + """ + return Trade.get_trades([Trade.fee_close_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(False), + ]).all() + @staticmethod def total_open_trades_stakes() -> float: """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0b9196f2e..f4e5d3b8e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -562,8 +562,7 @@ class RPC: except (ExchangeError): pass - Trade.session.delete(trade) - Trade.session.flush() + trade.delete() self._freqtrade.wallets.update() return { 'result': 'success', diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 26875fb7f..a35fc9fb8 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -18,6 +18,7 @@ from freqtrade.state import RunMode from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) +from tests.conftest_trades import MOCK_TRADE_COUNT def test_setup_utils_configuration(): @@ -1116,7 +1117,7 @@ def test_show_trades(mocker, fee, capsys, caplog): pargs = get_args(args) pargs['config'] = None start_show_trades(pargs) - assert log_has("Printing 4 Trades: ", caplog) + assert log_has(f"Printing {MOCK_TRADE_COUNT} Trades: ", caplog) captured = capsys.readouterr() assert "Trade(id=1" in captured.out assert "Trade(id=2" in captured.out diff --git a/tests/conftest.py b/tests/conftest.py index dbed08ec5..fe55c8784 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,8 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver from freqtrade.worker import Worker +from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, + mock_trade_4, mock_trade_5, mock_trade_6) logging.getLogger('').setLevel(logging.INFO) @@ -172,64 +174,22 @@ def create_mock_trades(fee): Create some fake trades ... """ # Simulate dry_run entries - trade = Trade( - pair='ETH/BTC', - stake_amount=0.001, - amount=123.0, - amount_requested=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='dry_run_buy_12345', - strategy='DefaultStrategy', - ) + trade = mock_trade_1(fee) Trade.session.add(trade) - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - amount_requested=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - close_rate=0.128, - close_profit=0.005, - exchange='bittrex', - is_open=False, - open_order_id='dry_run_sell_12345', - strategy='DefaultStrategy', - ) + trade = mock_trade_2(fee) Trade.session.add(trade) - trade = Trade( - pair='XRP/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.05, - close_rate=0.06, - close_profit=0.01, - exchange='bittrex', - is_open=False, - ) + trade = mock_trade_3(fee) Trade.session.add(trade) - # Simulate prod entry - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - amount_requested=124.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='prod_buy_12345', - strategy='DefaultStrategy', - ) + trade = mock_trade_4(fee) + Trade.session.add(trade) + + trade = mock_trade_5(fee) + Trade.session.add(trade) + + trade = mock_trade_6(fee) Trade.session.add(trade) @@ -823,22 +783,32 @@ def markets_empty(): @pytest.fixture(scope='function') -def limit_buy_order(): +def limit_buy_order_open(): return { 'id': 'mocked_limit_buy', 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), + 'timestamp': arrow.utcnow().timestamp, 'price': 0.00001099, 'amount': 90.99181073, - 'filled': 90.99181073, + 'filled': 0.0, 'cost': 0.0009999, - 'remaining': 0.0, - 'status': 'closed' + 'remaining': 90.99181073, + 'status': 'open' } +@pytest.fixture(scope='function') +def limit_buy_order(limit_buy_order_open): + order = deepcopy(limit_buy_order_open) + order['status'] = 'closed' + order['filled'] = order['amount'] + order['remaining'] = 0.0 + return order + + @pytest.fixture(scope='function') def market_buy_order(): return { @@ -1021,21 +991,31 @@ def limit_buy_order_canceled_empty(request): @pytest.fixture -def limit_sell_order(): +def limit_sell_order_open(): return { 'id': 'mocked_limit_sell', 'type': 'limit', 'side': 'sell', 'pair': 'mocked', 'datetime': arrow.utcnow().isoformat(), + 'timestamp': arrow.utcnow().timestamp, 'price': 0.00001173, 'amount': 90.99181073, - 'filled': 90.99181073, - 'remaining': 0.0, - 'status': 'closed' + 'filled': 0.0, + 'remaining': 90.99181073, + 'status': 'open' } +@pytest.fixture +def limit_sell_order(limit_sell_order_open): + order = deepcopy(limit_sell_order_open) + order['remaining'] = 0.0 + order['filled'] = order['amount'] + order['status'] = 'closed' + return order + + @pytest.fixture def order_book_l2(): return MagicMock(return_value={ diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py new file mode 100644 index 000000000..78388f022 --- /dev/null +++ b/tests/conftest_trades.py @@ -0,0 +1,279 @@ +from freqtrade.persistence.models import Order, Trade + + +MOCK_TRADE_COUNT = 6 + + +def mock_order_1(): + return { + 'id': '1234', + 'symbol': 'ETH/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_1(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_1(), 'ETH/BTC', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_2(): + return { + 'id': '1235', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_2_sell(): + return { + 'id': '12366', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'limit', + 'price': 0.128, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_2(fee): + """ + Closed trade... + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + close_rate=0.128, + close_profit=0.005, + exchange='bittrex', + is_open=False, + open_order_id='dry_run_sell_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_2_sell(), 'ETC/BTC', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_3(): + return { + 'id': '41231a12a', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.05, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_3_sell(): + return { + 'id': '41231a666a', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 0.06, + 'average': 0.06, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_3(fee): + """ + Closed trade + """ + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.05, + close_rate=0.06, + close_profit=0.01, + exchange='bittrex', + is_open=False, + ) + o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_3_sell(), 'XRP/BTC', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_4(): + return { + 'id': 'prod_buy_12345', + 'symbol': 'ETC/BTC', + 'status': 'open', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 0.0, + 'remaining': 123.0, + } + + +def mock_trade_4(fee): + """ + Simulate prod entry + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=124.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='prod_buy_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_4(), 'ETC/BTC', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_5(): + return { + 'id': 'prod_buy_3455', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_5_stoploss(): + return { + 'id': 'prod_stoploss_3455', + 'symbol': 'XRP/BTC', + 'status': 'open', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 0.0, + 'remaining': 123.0, + } + + +def mock_trade_5(fee): + """ + Simulate prod entry with stoploss + """ + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=124.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + strategy='SampleStrategy', + stoploss_order_id='prod_stoploss_3455' + ) + o = Order.parse_from_ccxt_object(mock_order_5(), 'XRP/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_5_stoploss(), 'XRP/BTC', 'stoploss') + trade.orders.append(o) + return trade + + +def mock_order_6(): + return { + 'id': 'prod_buy_6', + 'symbol': 'LTC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.15, + 'amount': 2.0, + 'filled': 2.0, + 'remaining': 0.0, + } + + +def mock_order_6_sell(): + return { + 'id': 'prod_sell_6', + 'symbol': 'LTC/BTC', + 'status': 'open', + 'side': 'sell', + 'type': 'limit', + 'price': 0.20, + 'amount': 2.0, + 'filled': 0.0, + 'remaining': 2.0, + } + + +def mock_trade_6(fee): + """ + Simulate prod entry with open sell order + """ + trade = Trade( + pair='LTC/BTC', + stake_amount=0.001, + amount=2.0, + amount_requested=2.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.15, + exchange='bittrex', + strategy='SampleStrategy', + open_order_id="prod_sell_6", + ) + o = Order.parse_from_ccxt_object(mock_order_6(), 'LTC/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_6_sell(), 'LTC/BTC', 'sell') + trade.orders.append(o) + return trade diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index e2ca66bd8..564dae0b1 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -20,6 +20,7 @@ from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, from freqtrade.data.history import load_data, load_pair_history from freqtrade.optimize.backtesting import BacktestResult from tests.conftest import create_mock_trades +from tests.conftest_trades import MOCK_TRADE_COUNT def test_get_latest_backtest_filename(testdatadir, mocker): @@ -110,7 +111,7 @@ def test_load_trades_from_db(default_conf, fee, mocker): trades = load_trades_from_db(db_url=default_conf['db_url']) assert init_mock.call_count == 1 - assert len(trades) == 4 + assert len(trades) == MOCK_TRADE_COUNT assert isinstance(trades, DataFrame) assert "pair" in trades.columns assert "open_date" in trades.columns diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index c254d6a09..e0b97d157 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access import copy import logging from datetime import datetime, timezone @@ -15,7 +13,8 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken -from freqtrade.exchange.common import API_RETRY_COUNT, calculate_backoff +from freqtrade.exchange.common import (API_RETRY_COUNT, API_FETCH_ORDER_RETRY_COUNT, + calculate_backoff) from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, @@ -808,7 +807,7 @@ def test_dry_run_order(default_conf, mocker, side, exchange_name): assert f'dry_run_{side}_' in order["id"] assert order["side"] == side assert order["type"] == "limit" - assert order["pair"] == "ETH/BTC" + assert order["symbol"] == "ETH/BTC" @pytest.mark.parametrize("side", [ @@ -1766,7 +1765,7 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name): cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC') assert order['id'] == cancel_order['id'] assert order['amount'] == cancel_order['amount'] - assert order['pair'] == cancel_order['pair'] + assert order['symbol'] == cancel_order['symbol'] assert cancel_order['status'] == 'canceled' @@ -1903,12 +1902,14 @@ def test_fetch_order(default_conf, mocker, exchange_name): # Ensure backoff is called assert tm.call_args_list[0][0][0] == 1 assert tm.call_args_list[1][0][0] == 2 - assert tm.call_args_list[2][0][0] == 5 - assert tm.call_args_list[3][0][0] == 10 - assert api_mock.fetch_order.call_count == 6 + if API_FETCH_ORDER_RETRY_COUNT > 2: + assert tm.call_args_list[2][0][0] == 5 + if API_FETCH_ORDER_RETRY_COUNT > 3: + assert tm.call_args_list[3][0][0] == 10 + assert api_mock.fetch_order.call_count == API_FETCH_ORDER_RETRY_COUNT + 1 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, - 'fetch_order', 'fetch_order', retries=6, + 'fetch_order', 'fetch_order', retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') @@ -1941,10 +1942,35 @@ def test_fetch_stoploss_order(default_conf, mocker, exchange_name): ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, 'fetch_stoploss_order', 'fetch_order', - retries=6, + retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') +def test_fetch_order_or_stoploss_order(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, id='binance') + fetch_order_mock = MagicMock() + fetch_stoploss_order_mock = MagicMock() + mocker.patch.multiple('freqtrade.exchange.Exchange', + fetch_order=fetch_order_mock, + fetch_stoploss_order=fetch_stoploss_order_mock, + ) + + exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', False) + assert fetch_order_mock.call_count == 1 + assert fetch_order_mock.call_args_list[0][0][0] == '1234' + assert fetch_order_mock.call_args_list[0][0][1] == 'ETH/BTC' + assert fetch_stoploss_order_mock.call_count == 0 + + fetch_order_mock.reset_mock() + fetch_stoploss_order_mock.reset_mock() + + exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', True) + assert fetch_order_mock.call_count == 0 + assert fetch_stoploss_order_mock.call_count == 1 + assert fetch_stoploss_order_mock.call_args_list[0][0][0] == '1234' + assert fetch_stoploss_order_mock.call_args_list[0][0][1] == 'ETH/BTC' + + @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_name(default_conf, mocker, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py index bed92d276..16789af2c 100644 --- a/tests/exchange/test_ftx.py +++ b/tests/exchange/test_ftx.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access from random import randint from unittest.mock import MagicMock @@ -7,6 +5,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException +from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT from tests.conftest import get_patched_exchange from .test_exchange import ccxt_exceptionhandlers @@ -154,5 +153,5 @@ def test_fetch_stoploss_order(default_conf, mocker): ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'ftx', 'fetch_stoploss_order', 'fetch_orders', - retries=6, + retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 9451c0b9e..8f774a7ec 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -1,5 +1,3 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access from random import randint from unittest.mock import MagicMock diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 102ed12fe..c2dee6439 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -313,7 +313,6 @@ def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog): with pytest.raises(RPCException, match='invalid argument'): rpc._rpc_delete('200') - create_mock_trades(fee) trades = Trade.query.all() trades[1].stoploss_order_id = '1234' trades[2].stoploss_order_id = '1234' @@ -717,11 +716,13 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: mocker.patch( 'freqtrade.exchange.Exchange.fetch_order', side_effect=[{ + 'id': '1234', 'status': 'open', 'type': 'limit', 'side': 'buy', 'filled': filled_amount }, { + 'id': '1234', 'status': 'closed', 'type': 'limit', 'side': 'buy', @@ -837,10 +838,10 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: assert counts["current"] == 1 -def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None: +def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order_open) -> None: default_conf['forcebuy_enable'] = True mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 762780111..6feacd4bd 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -253,7 +253,6 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) msg_mock = MagicMock() @@ -1007,7 +1006,6 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index ac6d3791a..0c12c05bb 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -13,10 +13,12 @@ import pytest from freqtrade.constants import (CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT) from freqtrade.exceptions import (DependencyException, ExchangeError, + InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade +from freqtrade.persistence.models import Order from freqtrade.rpc import RPCMessageType from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellCheckTuple, SellType @@ -25,6 +27,11 @@ from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, patch_wallet, patch_whitelist) +from tests.conftest_trades import (MOCK_TRADE_COUNT, mock_order_1, + mock_order_2, mock_order_2_sell, + mock_order_3, mock_order_3_sell, + mock_order_4, mock_order_5_stoploss, + mock_order_6_sell) def patch_RPCManager(mocker) -> MagicMock: @@ -170,7 +177,7 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]), (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]), ]) -def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order, +def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_open, amend_last, wallet, max_open, lsamr, expected) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -178,7 +185,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee ) default_conf['dry_run_wallet'] = wallet @@ -191,6 +198,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b for i in range(0, max_open): if expected[i] is not None: + limit_buy_order_open['id'] = str(i) result = freqtrade.get_trade_stake_amount('ETH/BTC') assert pytest.approx(result) == expected[i] freqtrade.execute_buy('ETH/BTC', result) @@ -216,13 +224,13 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: (0.50, 0.0025), ]) def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1, - limit_buy_order, fee, mocker) -> None: + limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee ) @@ -303,7 +311,6 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf 'ask': buy_price * 0.79, 'last': buy_price * 0.79 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) ############################################# @@ -343,7 +350,6 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, 'ask': buy_price * 0.85, 'last': buy_price * 0.85 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) ############################################# @@ -362,8 +368,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, assert freqtrade.handle_trade(trade) is False -def test_total_open_trades_stakes(mocker, default_conf, ticker, - limit_buy_order, fee) -> None: +def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['stake_amount'] = 0.00098751 @@ -371,7 +376,6 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -534,7 +538,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -568,7 +571,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -578,11 +580,11 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, freqtrade.create_trade('ETH/BTC') -def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, +def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mock = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -598,11 +600,11 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, assert rate * amount <= default_conf['stake_amount'] -def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, +def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mock = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, @@ -618,14 +620,14 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord assert not freqtrade.create_trade('ETH/BTC') -def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, ) @@ -639,14 +641,14 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, assert freqtrade.get_trade_stake_amount('ETH/BTC') == 0 -def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order, fee, +def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -702,7 +704,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @pytest.mark.parametrize("max_open", range(0, 5)) @pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)]) -def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, +def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_buy_order_open, max_open, tradable_balance_ratio, modifier) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -713,7 +715,7 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': "12355555"}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -727,14 +729,14 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, assert len(trades) == max(int(max_open * modifier), 0) -def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: +def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['max_open_trades'] = 4 mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': "12355555"}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -745,6 +747,8 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: freqtrade.execute_buy('NEO/BTC', default_conf['stake_amount']) assert len(Trade.get_open_trades()) == 2 + # Change order_id for new orders + limit_buy_order_open['id'] = '123444' # Create 2 new trades using create_trades assert freqtrade.create_trade('ETH/BTC') @@ -754,14 +758,14 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: assert len(trades) == 4 -def test_process_trade_creation(default_conf, ticker, limit_buy_order, +def test_process_trade_creation(default_conf, ticker, limit_buy_order, limit_buy_order_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), fetch_order=MagicMock(return_value=limit_buy_order), get_fee=fee, ) @@ -824,14 +828,14 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] -def test_process_trade_handling(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_process_trade_handling(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - fetch_order=MagicMock(return_value=limit_buy_order), + buy=MagicMock(return_value=limit_buy_order_open), + fetch_order=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -971,7 +975,7 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, assert not log_has("Using cached buy rate for ETH/BTC.", caplog) -def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: +def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) freqtrade = FreqtradeBot(default_conf) @@ -984,7 +988,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: get_buy_rate=buy_rate_mock, _get_min_pair_stake_amount=MagicMock(return_value=1) ) - buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -1003,6 +1007,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: assert freqtrade.strategy.confirm_trade_entry.call_count == 1 buy_rate_mock.reset_mock() + limit_buy_order_open['id'] = '22' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_buy(pair, stake_amount) assert buy_rate_mock.call_count == 1 @@ -1018,9 +1023,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: trade = Trade.query.first() assert trade assert trade.is_open is True - assert trade.open_order_id == limit_buy_order['id'] + assert trade.open_order_id == '22' # Test calling with price + limit_buy_order_open['id'] = '33' fix_price = 0.06 assert freqtrade.execute_buy(pair, stake_amount, fix_price) # Make sure get_buy_rate wasn't called again @@ -1036,6 +1042,8 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: limit_buy_order['status'] = 'closed' limit_buy_order['price'] = 10 limit_buy_order['cost'] = 100 + limit_buy_order['id'] = '444' + mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) assert freqtrade.execute_buy(pair, stake_amount) trade = Trade.query.all()[2] @@ -1051,11 +1059,12 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: limit_buy_order['remaining'] = 10.00 limit_buy_order['price'] = 0.5 limit_buy_order['cost'] = 40.495905365 + limit_buy_order['id'] = '555' mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) assert freqtrade.execute_buy(pair, stake_amount) trade = Trade.query.all()[3] assert trade - assert trade.open_order_id is None + assert trade.open_order_id == '555' assert trade.open_rate == 0.5 assert trade.stake_amount == 40.495905365 @@ -1066,6 +1075,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: limit_buy_order['remaining'] = 90.99181073 limit_buy_order['price'] = 0.5 limit_buy_order['cost'] = 0.0 + limit_buy_order['id'] = '66' mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order)) assert not freqtrade.execute_buy(pair, stake_amount) @@ -1093,9 +1103,11 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) - freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=ValueError) assert freqtrade.execute_buy(pair, stake_amount) + limit_buy_order['id'] = '222' freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=Exception) assert freqtrade.execute_buy(pair, stake_amount) + limit_buy_order['id'] = '2223' freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True) assert freqtrade.execute_buy(pair, stake_amount) @@ -1201,6 +1213,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert trade stoploss_order_hit = MagicMock(return_value={ + 'id': 100, 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, @@ -1209,7 +1222,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, }) mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True - assert log_has('STOP_LOSS_LIMIT is hit for {}.'.format(trade), caplog) + assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None assert trade.is_open is False @@ -1258,7 +1271,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - fetch_stoploss_order=MagicMock(return_value={'status': 'canceled'}), + fetch_stoploss_order=MagicMock(return_value={'status': 'canceled', 'id': 100}), stoploss=MagicMock(side_effect=ExchangeError()), ) freqtrade = FreqtradeBot(default_conf) @@ -1278,7 +1291,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, - limit_buy_order, limit_sell_order): + limit_buy_order_open, limit_sell_order): rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) @@ -1289,7 +1302,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, 'ask': 0.00001173, 'last': 0.00001172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), sell=sell_mock, get_fee=fee, fetch_order=MagicMock(return_value={'status': 'canceled'}), @@ -1320,7 +1333,46 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' -def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, +def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, + limit_buy_order_open, limit_sell_order): + sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=MagicMock(return_value={ + 'bid': 0.00001172, + 'ask': 0.00001173, + 'last': 0.00001172 + }), + buy=MagicMock(return_value=limit_buy_order_open), + sell=sell_mock, + get_fee=fee, + fetch_order=MagicMock(return_value={'status': 'canceled'}), + stoploss=MagicMock(side_effect=InsufficientFundsError()), + ) + patch_get_signal(freqtrade) + freqtrade.strategy.order_types['stoploss_on_exchange'] = True + + freqtrade.enter_positions() + trade = Trade.query.first() + caplog.clear() + freqtrade.create_stoploss_order(trade, 200) + # stoploss_orderid was empty before + assert trade.stoploss_order_id is None + assert mock_insuf.call_count == 1 + mock_insuf.reset_mock() + + trade.stoploss_order_id = 'stoploss_orderid' + freqtrade.create_stoploss_order(trade, 200) + # No change to stoploss-orderid + assert trade.stoploss_order_id == 'stoploss_orderid' + assert mock_insuf.call_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set stoploss = MagicMock(return_value={'id': 13434334}) @@ -1389,7 +1441,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, })) cancel_order_mock = MagicMock() - stoploss_order_mock = MagicMock() + stoploss_order_mock = MagicMock(return_value={'id': 13434334}) mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', cancel_order_mock) mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock) @@ -1691,8 +1743,10 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No open_date=arrow.utcnow().datetime, amount=11, ) + assert not freqtrade.update_trade_state(trade, None) + assert log_has_re(r'Orderid for trade .* is empty.', caplog) # Add datetime explicitly since sqlalchemy defaults apply only once written to database - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, '123') # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) assert trade.open_order_id is None @@ -1702,14 +1756,14 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) assert trade.amount != 90.81 # test amount modified by fee-logic - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, '123') assert trade.amount == 90.81 assert trade.open_order_id is None trade.is_open = True trade.open_order_id = None # Assert we call handle_trade() if trade is feasible for execution - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, '123') assert log_has_re('Found open order for.*', caplog) @@ -1734,7 +1788,7 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_ open_order_id="123456", is_open=True, ) - freqtrade.update_trade_state(trade, limit_buy_order) + freqtrade.update_trade_state(trade, '123456', limit_buy_order) assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] @@ -1756,11 +1810,11 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ open_rate=0.245441, fee_open=fee.return_value, fee_close=fee.return_value, - open_order_id="123456", + open_order_id='123456', is_open=True, open_date=arrow.utcnow().datetime, ) - freqtrade.update_trade_state(trade, limit_buy_order) + freqtrade.update_trade_state(trade, '123456', limit_buy_order) assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] assert log_has_re(r'Applying fee on amount for .*', caplog) @@ -1780,7 +1834,7 @@ def test_update_trade_state_exception(mocker, default_conf, 'freqtrade.freqtradebot.FreqtradeBot.get_real_amount', side_effect=DependencyException() ) - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, trade.open_order_id) assert log_has('Could not update trade amount: ', caplog) @@ -1795,12 +1849,13 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None # Test raise of OperationalException exception grm_mock = mocker.patch("freqtrade.freqtradebot.FreqtradeBot.get_real_amount", MagicMock()) - freqtrade.update_trade_state(trade) + freqtrade.update_trade_state(trade, trade.open_order_id) assert grm_mock.call_count == 0 assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) -def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order, mocker): +def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, + limit_sell_order, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) # fetch_order should not be called!! mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError)) @@ -1823,14 +1878,20 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde open_order_id="123456", is_open=True, ) - freqtrade.update_trade_state(trade, limit_sell_order) + order = Order.parse_from_ccxt_object(limit_sell_order_open, 'LTC/ETH', 'sell') + trade.orders.append(order) + assert order.status == 'open' + freqtrade.update_trade_state(trade, trade.open_order_id, limit_sell_order) assert trade.amount == limit_sell_order['amount'] # Wallet needs to be updated after closing a limit-sell order to reenable buying assert wallet_mock.call_count == 1 assert not trade.is_open + # Order is updated by update_trade_state + assert order.status == 'closed' -def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mocker) -> None: +def test_handle_trade(default_conf, limit_buy_order, limit_sell_order_open, limit_sell_order, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -1840,8 +1901,8 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock 'ask': 0.00001173, 'last': 0.00001172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - sell=MagicMock(return_value={'id': limit_sell_order['id']}), + buy=MagicMock(return_value=limit_buy_order), + sell=MagicMock(return_value=limit_sell_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -1870,13 +1931,14 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock assert trade.close_date is not None -def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -1921,7 +1983,7 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee, assert freqtrade.handle_trade(trades[0]) is True -def test_handle_trade_roi(default_conf, ticker, limit_buy_order, +def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) @@ -1929,7 +1991,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -1954,14 +2016,14 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, def test_handle_trade_use_sell_signal( - default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None: + default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None: # use_sell_signal is True buy default caplog.set_level(logging.DEBUG) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -1982,14 +2044,14 @@ def test_handle_trade_use_sell_signal( caplog) -def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, +def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -2807,7 +2869,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, fee, - limit_buy_order, mocker) -> None: + mocker) -> None: default_conf['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) @@ -2929,7 +2991,36 @@ def test_execute_sell_market_order(default_conf, ticker, fee, } == last_msg -def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, +def test_execute_sell_insufficient_funds_error(default_conf, ticker, fee, + ticker_sell_up, mocker) -> None: + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds') + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, + sell=MagicMock(side_effect=InsufficientFundsError()) + ) + patch_get_signal(freqtrade) + + # Create some test data + freqtrade.enter_positions() + + trade = Trade.query.first() + assert trade + + # Increase the price and sell it + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker_sell_up + ) + + assert not freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], + sell_reason=SellType.ROI) + assert mock_insuf.call_count == 1 + + +def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2940,7 +3031,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, 'ask': 0.00002173, 'last': 0.00002172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -2961,7 +3052,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, +def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2972,7 +3063,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, 'ask': 0.00002173, 'last': 0.00002172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -2992,7 +3083,8 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3002,7 +3094,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker 'ask': 0.00000173, 'last': 0.00000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3021,7 +3113,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker assert freqtrade.handle_trade(trade) is False -def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocker) -> None: +def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3031,7 +3124,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke 'ask': 0.0000173, 'last': 0.0000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3053,7 +3146,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke assert trade.sell_reason == SellType.SELL_SIGNAL.value -def test_sell_not_enough_balance(default_conf, limit_buy_order, +def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3064,7 +3157,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, 'ask': 0.00002173, 'last': 0.00002172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -3172,7 +3265,8 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo assert log_has(f"Pair {trade.pair} is currently locked.", caplog) -def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None: +def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3182,7 +3276,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> 'ask': 0.0000173, 'last': 0.0000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3206,7 +3300,8 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> assert trade.sell_reason == SellType.ROI.value -def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None: +def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, + fee, caplog, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( @@ -3216,7 +3311,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) 'ask': 0.00001099, 'last': 0.00001099 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['trailing_stop'] = True @@ -3256,7 +3351,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, +def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) @@ -3268,7 +3363,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['trailing_stop'] = True @@ -3313,7 +3408,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) -def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, +def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] patch_RPCManager(mocker) @@ -3325,7 +3420,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -3371,7 +3466,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value -def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, +def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, caplog, mocker) -> None: buy_price = limit_buy_order['price'] # buy_price: 0.00001099 @@ -3385,7 +3480,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, 'ask': buy_price, 'last': buy_price }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -3434,7 +3529,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, assert trade.stop_loss == 0.0000117705 -def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, +def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -3445,7 +3540,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, 'ask': 0.00000173, 'last': 0.00000172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) default_conf['ask_strategy'] = { @@ -3787,8 +3882,8 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker, assert walletmock.call_count == 1 -def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker, - order_book_l2): +def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order, + fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1 patch_RPCManager(mocker) @@ -3797,7 +3892,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -3912,8 +4007,8 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False -def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order, - fee, mocker, order_book_l2, caplog) -> None: +def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_order, fee, + limit_sell_order_open, mocker, order_book_l2, caplog) -> None: """ test order book ask strategy """ @@ -3932,8 +4027,8 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order 'ask': 0.00001173, 'last': 0.00001172 }), - buy=MagicMock(return_value={'id': limit_buy_order['id']}), - sell=MagicMock(return_value={'id': limit_sell_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), + sell=MagicMock(return_value=limit_sell_order_open), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) @@ -4078,7 +4173,7 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): @pytest.mark.usefixtures("init_persistence") -def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog): +def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_open, caplog): default_conf['dry_run'] = True # Initialize to 2 times stake amount default_conf['dry_run_wallet'] = 0.002 @@ -4088,7 +4183,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': limit_buy_order['id']}), + buy=MagicMock(return_value=limit_buy_order_open), get_fee=fee, ) @@ -4113,21 +4208,22 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limit_sell_order): default_conf['cancel_open_orders_on_exit'] = True mocker.patch('freqtrade.exchange.Exchange.fetch_order', - side_effect=[ExchangeError(), limit_sell_order, limit_buy_order]) + side_effect=[ + ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order]) buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy') sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell') freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) trades = Trade.query.all() - assert len(trades) == 4 + assert len(trades) == MOCK_TRADE_COUNT freqtrade.cancel_all_open_orders() assert buy_mock.call_count == 1 - assert sell_mock.call_count == 1 + assert sell_mock.call_count == 2 @pytest.mark.usefixtures("init_persistence") -def test_check_for_open_trades(mocker, default_conf, fee, limit_buy_order, limit_sell_order): +def test_check_for_open_trades(mocker, default_conf, fee): freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.check_for_open_trades() @@ -4140,3 +4236,246 @@ def test_check_for_open_trades(mocker, default_conf, fee, limit_buy_order, limit freqtrade.check_for_open_trades() assert freqtrade.rpc.send_msg.call_count == 1 assert 'Handle these trades manually' in freqtrade.rpc.send_msg.call_args[0][0]['status'] + + +@pytest.mark.usefixtures("init_persistence") +def test_update_open_orders(mocker, default_conf, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + create_mock_trades(fee) + + freqtrade.update_open_orders() + assert log_has_re(r"Error updating Order .*", caplog) + caplog.clear() + + assert len(Order.get_open_orders()) == 3 + matching_buy_order = mock_order_4() + matching_buy_order.update({ + 'status': 'closed', + }) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=matching_buy_order) + freqtrade.update_open_orders() + # Only stoploss and sell orders are kept open + assert len(Order.get_open_orders()) == 2 + + +@pytest.mark.usefixtures("init_persistence") +def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + def patch_with_fee(order): + order.update({'fee': {'cost': 0.1, 'rate': 0.2, + 'currency': order['symbol'].split('/')[0]}}) + return order + + mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + side_effect=[ + patch_with_fee(mock_order_2_sell()), + patch_with_fee(mock_order_3_sell()), + patch_with_fee(mock_order_1()), + patch_with_fee(mock_order_2()), + patch_with_fee(mock_order_3()), + patch_with_fee(mock_order_4()), + ] + ) + + create_mock_trades(fee) + trades = Trade.get_trades().all() + assert len(trades) == MOCK_TRADE_COUNT + for trade in trades: + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None + + freqtrade.update_closed_trades_without_assigned_fees() + + trades = Trade.get_trades().all() + assert len(trades) == MOCK_TRADE_COUNT + + for trade in trades: + if trade.is_open: + # Exclude Trade 4 - as the order is still open. + if trade.select_order('buy', False): + assert trade.fee_open_cost is not None + assert trade.fee_open_currency is not None + else: + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + + else: + assert trade.fee_close_cost is not None + assert trade.fee_close_currency is not None + + +@pytest.mark.usefixtures("init_persistence") +def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') + + create_mock_trades(fee) + trades = Trade.get_trades().all() + + freqtrade.reupdate_buy_order_fees(trades[0]) + assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) + assert mock_uts.call_count == 1 + assert mock_uts.call_args_list[0][0][0] == trades[0] + assert mock_uts.call_args_list[0][0][1] == mock_order_1()['id'] + assert log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) + mock_uts.reset_mock() + caplog.clear() + + # Test with trade without orders + trade = Trade( + pair='XRP/ETH', + stake_amount=0.001, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=arrow.utcnow().datetime, + is_open=True, + amount=20, + open_rate=0.01, + exchange='bittrex', + ) + Trade.session.add(trade) + + freqtrade.reupdate_buy_order_fees(trade) + assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) + assert mock_uts.call_count == 0 + assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) + + +@pytest.mark.usefixtures("init_persistence") +def test_handle_insufficient_funds(mocker, default_conf, fee): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order') + mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_buy_order_fees') + create_mock_trades(fee) + trades = Trade.get_trades().all() + + # Trade 0 has only a open buy order, no closed order + freqtrade.handle_insufficient_funds(trades[0]) + assert mock_rlo.call_count == 0 + assert mock_bof.call_count == 1 + + mock_rlo.reset_mock() + mock_bof.reset_mock() + + # Trade 1 has closed buy and sell orders + freqtrade.handle_insufficient_funds(trades[1]) + assert mock_rlo.call_count == 1 + assert mock_bof.call_count == 0 + + mock_rlo.reset_mock() + mock_bof.reset_mock() + + # Trade 2 has closed buy and sell orders + freqtrade.handle_insufficient_funds(trades[2]) + assert mock_rlo.call_count == 1 + assert mock_bof.call_count == 0 + + mock_rlo.reset_mock() + mock_bof.reset_mock() + + # Trade 3 has an opne buy order + freqtrade.handle_insufficient_funds(trades[3]) + assert mock_rlo.call_count == 0 + assert mock_bof.call_count == 1 + + +@pytest.mark.usefixtures("init_persistence") +def test_refind_lost_order(mocker, default_conf, fee, caplog): + caplog.set_level(logging.DEBUG) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') + + mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + return_value={'status': 'open'}) + + def reset_open_orders(trade): + trade.open_order_id = None + trade.stoploss_order_id = None + + create_mock_trades(fee) + trades = Trade.get_trades().all() + + caplog.clear() + + # No open order + trade = trades[0] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_1() + assert log_has_re(r"Order Order(.*order_id=" + order['id'] + ".*) is no longer open.", caplog) + assert mock_fo.call_count == 0 + assert mock_uts.call_count == 0 + # No change to orderid - as update_trade_state is mocked + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + caplog.clear() + mock_fo.reset_mock() + + # Open buy order + trade = trades[3] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_4() + assert log_has_re(r"Trying to refind Order\(.*", caplog) + assert mock_fo.call_count == 0 + assert mock_uts.call_count == 0 + # No change to orderid - as update_trade_state is mocked + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + caplog.clear() + mock_fo.reset_mock() + + # Open stoploss order + trade = trades[4] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_5_stoploss() + assert log_has_re(r"Trying to refind Order\(.*", caplog) + assert mock_fo.call_count == 1 + assert mock_uts.call_count == 1 + # stoploss_order_id is "refound" and added to the trade + assert trade.open_order_id is None + assert trade.stoploss_order_id is not None + + caplog.clear() + mock_fo.reset_mock() + mock_uts.reset_mock() + + # Open sell order + trade = trades[5] + reset_open_orders(trade) + assert trade.open_order_id is None + assert trade.stoploss_order_id is None + + freqtrade.refind_lost_order(trade) + order = mock_order_6_sell() + assert log_has_re(r"Trying to refind Order\(.*", caplog) + assert mock_fo.call_count == 1 + assert mock_uts.call_count == 1 + # sell-orderid is "refound" and added to the trade + assert trade.open_order_id == order['id'] + assert trade.stoploss_order_id is None + + caplog.clear() + + # Test error case + mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order', + side_effect=ExchangeError()) + order = mock_order_5_stoploss() + + freqtrade.refind_lost_order(trades[4]) + assert log_has(f"Error updating {order['id']}.", caplog) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 65c83e05b..adfa18876 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -7,9 +7,9 @@ import pytest from sqlalchemy import create_engine from freqtrade import constants -from freqtrade.exceptions import OperationalException -from freqtrade.persistence import Trade, clean_dry_run_db, init -from tests.conftest import log_has, create_mock_trades +from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.persistence import Order, Trade, clean_dry_run_db, init +from tests.conftest import create_mock_trades, log_has, log_has_re def test_init_create_session(default_conf): @@ -22,7 +22,7 @@ def test_init_create_session(default_conf): def test_init_custom_db_url(default_conf, mocker): # Update path to a value other than default, but still in-memory default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'}) - create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock()) init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 @@ -40,7 +40,7 @@ def test_init_prod_db(default_conf, mocker): default_conf.update({'dry_run': False}) default_conf.update({'db_url': constants.DEFAULT_DB_PROD_URL}) - create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock()) init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 @@ -51,7 +51,7 @@ def test_init_dryrun_db(default_conf, mocker): default_conf.update({'dry_run': True}) default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL}) - create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock()) + create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock()) init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 @@ -93,6 +93,8 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): stake_amount=0.001, open_rate=0.01, amount=5, + is_open=True, + open_date=arrow.utcnow().datetime, fee_open=fee.return_value, fee_close=fee.return_value, exchange='bittrex', @@ -107,9 +109,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): assert trade.open_rate == 0.00001099 assert trade.close_profit is None assert trade.close_date is None - assert log_has("LIMIT_BUY has been fulfilled for Trade(id=2, " - "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", - caplog) + assert log_has_re(r"LIMIT_BUY has been fulfilled for Trade\(id=2, " + r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).", + caplog) caplog.clear() trade.open_order_id = 'something' @@ -118,9 +120,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog): assert trade.close_rate == 0.00001173 assert trade.close_profit == 0.06201058 assert trade.close_date is not None - assert log_has("LIMIT_SELL has been fulfilled for Trade(id=2, " - "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).", - caplog) + assert log_has_re(r"LIMIT_SELL has been fulfilled for Trade\(id=2, " + r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).", + caplog) @pytest.mark.usefixtures("init_persistence") @@ -131,8 +133,10 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): stake_amount=0.001, amount=5, open_rate=0.01, + is_open=True, fee_open=fee.return_value, fee_close=fee.return_value, + open_date=arrow.utcnow().datetime, exchange='bittrex', ) @@ -142,20 +146,21 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog): assert trade.open_rate == 0.00004099 assert trade.close_profit is None assert trade.close_date is None - assert log_has("MARKET_BUY has been fulfilled for Trade(id=1, " - "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", - caplog) + assert log_has_re(r"MARKET_BUY has been fulfilled for Trade\(id=1, " + r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).", + caplog) caplog.clear() + trade.is_open = True trade.open_order_id = 'something' trade.update(market_sell_order) assert trade.open_order_id is None assert trade.close_rate == 0.00004173 assert trade.close_profit == 0.01297561 assert trade.close_date is not None - assert log_has("MARKET_SELL has been fulfilled for Trade(id=1, " - "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).", - caplog) + assert log_has_re(r"MARKET_SELL has been fulfilled for Trade\(id=1, " + r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).", + caplog) @pytest.mark.usefixtures("init_persistence") @@ -184,6 +189,36 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee): assert trade.calc_profit_ratio() == 0.06201058 +@pytest.mark.usefixtures("init_persistence") +def test_trade_close(limit_buy_order, limit_sell_order, fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + open_rate=0.01, + amount=5, + is_open=True, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_date=arrow.Arrow(2020, 2, 1, 15, 5, 1).datetime, + exchange='bittrex', + ) + assert trade.close_profit is None + assert trade.close_date is None + assert trade.is_open is True + trade.close(0.02) + assert trade.is_open is False + assert trade.close_profit == 0.99002494 + assert trade.close_date is not None + + new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime, + assert trade.close_date != new_date + # Close should NOT update close_date if the trade has been closed already + assert trade.is_open is False + trade.close_date = new_date + trade.close(0.02) + assert trade.close_date == new_date + + @pytest.mark.usefixtures("init_persistence") def test_calc_close_trade_price_exception(limit_buy_order, fee): trade = Trade( @@ -421,9 +456,9 @@ def test_migrate_old(mocker, default_conf, fee): PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" - insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, + insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee, open_rate, stake_amount, amount, open_date) - VALUES ('BITTREX', 'BTC_ETC', 1, {fee}, + VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee}, 0.00258580, {stake}, {amount}, '2017-11-28 12:44:24.000000') """.format(fee=fee.return_value, @@ -440,7 +475,7 @@ def test_migrate_old(mocker, default_conf, fee): amount=amount ) engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) @@ -481,6 +516,12 @@ def test_migrate_old(mocker, default_conf, fee): assert pytest.approx(trade.close_profit_abs) == trade.calc_profit() assert trade.sell_order_status is None + # Should've created one order + assert len(Order.query.all()) == 1 + order = Order.query.first() + assert order.order_id == '123123' + assert order.ft_order_side == 'buy' + def test_migrate_new(mocker, default_conf, fee, caplog): """ @@ -509,22 +550,25 @@ def test_migrate_new(mocker, default_conf, fee, caplog): sell_reason VARCHAR, strategy VARCHAR, ticker_interval INTEGER, + stoploss_order_id VARCHAR, PRIMARY KEY (id), CHECK (is_open IN (0, 1)) );""" insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee, open_rate, stake_amount, amount, open_date, - stop_loss, initial_stop_loss, max_rate, ticker_interval) + stop_loss, initial_stop_loss, max_rate, ticker_interval, + open_order_id, stoploss_order_id) VALUES ('binance', 'ETC/BTC', 1, {fee}, 0.00258580, {stake}, {amount}, '2019-11-28 12:44:24.000000', - 0.0, 0.0, 0.0, '5m') + 0.0, 0.0, 0.0, '5m', + 'buy_order', 'stop_order_id222') """.format(fee=fee.return_value, stake=default_conf.get("stake_amount"), amount=amount ) engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) @@ -558,14 +602,23 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert trade.sell_reason is None assert trade.strategy is None assert trade.timeframe == '5m' - assert trade.stoploss_order_id is None + assert trade.stoploss_order_id == 'stop_order_id222' assert trade.stoploss_last_update is None assert log_has("trying trades_bak1", caplog) assert log_has("trying trades_bak2", caplog) - assert log_has("Running database migration - backup available as trades_bak2", caplog) + assert log_has("Running database migration for trades - backup: trades_bak2", caplog) assert trade.open_trade_price == trade._calc_open_trade_price() assert trade.close_profit_abs is None + assert log_has("Moving open orders to Orders table.", caplog) + orders = Order.query.all() + assert len(orders) == 2 + assert orders[0].order_id == 'buy_order' + assert orders[0].ft_order_side == 'buy' + + assert orders[1].order_id == 'stop_order_id222' + assert orders[1].ft_order_side == 'stoploss' + def test_migrate_mid_state(mocker, default_conf, fee, caplog): """ @@ -601,7 +654,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): amount=amount ) engine = create_engine('sqlite://') - mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) + mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) @@ -626,7 +679,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): assert trade.initial_stop_loss == 0.0 assert trade.open_trade_price == trade._calc_open_trade_price() assert log_has("trying trades_bak0", caplog) - assert log_has("Running database migration - backup available as trades_bak0", caplog) + assert log_has("Running database migration for trades - backup: trades_bak0", caplog) def test_adjust_stop_loss(fee): @@ -713,10 +766,10 @@ def test_adjust_min_max_rates(fee): @pytest.mark.usefixtures("init_persistence") -def test_get_open(default_conf, fee): +def test_get_open(fee): create_mock_trades(fee) - assert len(Trade.get_open_trades()) == 2 + assert len(Trade.get_open_trades()) == 4 @pytest.mark.usefixtures("init_persistence") @@ -986,7 +1039,7 @@ def test_total_open_trades_stakes(fee): assert res == 0 create_mock_trades(fee) res = Trade.total_open_trades_stakes() - assert res == 0.002 + assert res == 0.004 @pytest.mark.usefixtures("init_persistence") @@ -1012,3 +1065,96 @@ def test_get_best_pair(fee): assert len(res) == 2 assert res[0] == 'XRP/BTC' assert res[1] == 0.01 + + +@pytest.mark.usefixtures("init_persistence") +def test_update_order_from_ccxt(): + # Most basic order return (only has orderid) + o = Order.parse_from_ccxt_object({'id': '1234'}, 'ETH/BTC', 'buy') + assert isinstance(o, Order) + assert o.ft_pair == 'ETH/BTC' + assert o.ft_order_side == 'buy' + assert o.order_id == '1234' + assert o.ft_is_open + ccxt_order = { + 'id': '1234', + 'side': 'buy', + 'symbol': 'ETH/BTC', + 'type': 'limit', + 'price': 1234.5, + 'amount': 20.0, + 'filled': 9, + 'remaining': 11, + 'status': 'open', + 'timestamp': 1599394315123 + } + o = Order.parse_from_ccxt_object(ccxt_order, 'ETH/BTC', 'buy') + assert isinstance(o, Order) + assert o.ft_pair == 'ETH/BTC' + assert o.ft_order_side == 'buy' + assert o.order_id == '1234' + assert o.order_type == 'limit' + assert o.price == 1234.5 + assert o.filled == 9 + assert o.remaining == 11 + assert o.order_date is not None + assert o.ft_is_open + assert o.order_filled_date is None + + # Order has been closed + ccxt_order.update({'filled': 20.0, 'remaining': 0.0, 'status': 'closed'}) + o.update_from_ccxt_object(ccxt_order) + + assert o.filled == 20.0 + assert o.remaining == 0.0 + assert not o.ft_is_open + assert o.order_filled_date is not None + + ccxt_order.update({'id': 'somethingelse'}) + with pytest.raises(DependencyException, match=r"Order-id's don't match"): + o.update_from_ccxt_object(ccxt_order) + + +@pytest.mark.usefixtures("init_persistence") +def test_select_order(fee): + create_mock_trades(fee) + + trades = Trade.get_trades().all() + + # Open buy order, no sell order + order = trades[0].select_order('buy', True) + assert order is None + order = trades[0].select_order('buy', False) + assert order is not None + order = trades[0].select_order('sell', None) + assert order is None + + # closed buy order, and open sell order + order = trades[1].select_order('buy', True) + assert order is None + order = trades[1].select_order('buy', False) + assert order is not None + order = trades[1].select_order('buy', None) + assert order is not None + order = trades[1].select_order('sell', True) + assert order is None + order = trades[1].select_order('sell', False) + assert order is not None + + # Has open buy order + order = trades[3].select_order('buy', True) + assert order is not None + order = trades[3].select_order('buy', False) + assert order is None + + # Open sell order + order = trades[4].select_order('buy', True) + assert order is None + order = trades[4].select_order('buy', False) + assert order is not None + + order = trades[4].select_order('sell', True) + assert order is not None + assert order.ft_order_side == 'stoploss' + order = trades[4].select_order('sell', False) + assert order is None