From 67beda6c929df56a1c9861202fdb839e5fb311e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:06:32 +0200 Subject: [PATCH 1/6] Add fetch_dry_run_order method --- freqtrade/exchange/exchange.py | 28 ++++++++++++++++++---------- freqtrade/exchange/ftx.py | 9 ++------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 93d8f7584..02445de92 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -591,6 +591,19 @@ class Exchange: closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order + def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: + """ + Return dry-run order + Only call if running in dry-run mode. + """ + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + def create_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict: try: @@ -1066,11 +1079,12 @@ class Exchange: @retrier def cancel_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - order = self._dry_run_open_orders.get(order_id) - if order: + try: + order = self.fetch_dry_run_order(order_id) + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) return order - else: + except InvalidOrderException: return {} try: @@ -1144,13 +1158,7 @@ class Exchange: @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + return self.fetch_dry_run_order(order_id) try: return self._api.fetch_order(order_id, pair) except ccxt.OrderNotFound as e: diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 9009e9492..105389828 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -93,13 +93,8 @@ class Ftx(Exchange): @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + return self.fetch_dry_run_order(order_id) + try: orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) From 4c277b3039bda30369ada1157368dd93d19e4dd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:17:50 +0200 Subject: [PATCH 2/6] Reorder exchange methods --- freqtrade/exchange/exchange.py | 332 +++++++++++++++++---------------- 1 file changed, 169 insertions(+), 163 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 02445de92..c3c4e8e5a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -550,6 +550,8 @@ class Exchange: # See also #2575 at github. return max(min_stake_amounts) * amount_reserve_percent + # Dry-run methods + def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' @@ -604,6 +606,8 @@ class Exchange: raise InvalidOrderException( f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + # Order handling + def create_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict: try: @@ -680,6 +684,128 @@ class Exchange: raise OperationalException(f"stoploss is not implemented for {self.name}.") + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + return self.fetch_dry_run_order(order_id) + try: + return self._api.fetch_order(order_id, pair) + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to fetch_stoploss_order to allow easy overriding in other classes + fetch_stoploss_order = fetch_order + + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + + def check_order_canceled_empty(self, order: Dict) -> bool: + """ + Verify if an order has been cancelled without being partially filled + :param order: Order dict as returned from fetch_order() + :return: True if order has been cancelled without being filled, False otherwise. + """ + return (order.get('status') in ('closed', 'canceled', 'cancelled') + and order.get('filled') == 0.0) + + @retrier + def cancel_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + try: + order = self.fetch_dry_run_order(order_id) + + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) + return order + except InvalidOrderException: + return {} + + try: + return self._api.cancel_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to cancel_stoploss_order to allow easy overriding in other classes + cancel_stoploss_order = cancel_order + + def is_cancel_order_result_suitable(self, corder) -> bool: + if not isinstance(corder, dict): + return False + + required = ('fee', 'status', 'amount') + return all(k in corder for k in required) + + def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: Orderid to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + try: + corder = self.cancel_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + except InvalidOrderException: + logger.warning(f"Could not cancel order {order_id} for {pair}.") + try: + order = self.fetch_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + + def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel stoploss order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: stoploss-order-id to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + corder = self.cancel_stoploss_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + try: + order = self.fetch_stoploss_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + @retrier def get_balances(self) -> dict: @@ -726,6 +852,8 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + # Pricing info + @retrier def fetch_ticker(self, pair: str) -> dict: try: @@ -742,6 +870,47 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @staticmethod + def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], + range_required: bool = True): + """ + Get next greater value in the list. + Used by fetch_l2_order_book if the api only supports a limited range + """ + if not limit_range: + return limit + + result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) + if not range_required and limit > result: + # Range is not required - we can use None as parameter. + return None + return result + + @retrier + def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: + """ + Get L2 order book from exchange. + Can be limited to a certain amount (if supported). + Returns a dict in the format + {'asks': [price, volume], 'bids': [price, volume]} + """ + limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], + self._ft_has['l2_limit_range_required']) + try: + + return self._api.fetch_l2_order_book(pair, limit1) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: """ @@ -1067,169 +1236,6 @@ class Exchange: self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) - def check_order_canceled_empty(self, order: Dict) -> bool: - """ - Verify if an order has been cancelled without being partially filled - :param order: Order dict as returned from fetch_order() - :return: True if order has been cancelled without being filled, False otherwise. - """ - return (order.get('status') in ('closed', 'canceled', 'cancelled') - and order.get('filled') == 0.0) - - @retrier - def cancel_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - try: - order = self.fetch_dry_run_order(order_id) - - order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) - return order - except InvalidOrderException: - return {} - - try: - return self._api.cancel_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Could not cancel order. Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - # Assign method to cancel_stoploss_order to allow easy overriding in other classes - cancel_stoploss_order = cancel_order - - def is_cancel_order_result_suitable(self, corder) -> bool: - if not isinstance(corder, dict): - return False - - required = ('fee', 'status', 'amount') - return all(k in corder for k in required) - - def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: - """ - Cancel order returning a result. - Creates a fake result if cancel order returns a non-usable result - and fetch_order does not work (certain exchanges don't return cancelled orders) - :param order_id: Orderid to cancel - :param pair: Pair corresponding to order_id - :param amount: Amount to use for fake response - :return: Result from either cancel_order if usable, or fetch_order - """ - try: - corder = self.cancel_order(order_id, pair) - if self.is_cancel_order_result_suitable(corder): - return corder - except InvalidOrderException: - logger.warning(f"Could not cancel order {order_id} for {pair}.") - try: - order = self.fetch_order(order_id, pair) - except InvalidOrderException: - logger.warning(f"Could not fetch cancelled order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} - - return order - - def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: - """ - Cancel stoploss order returning a result. - Creates a fake result if cancel order returns a non-usable result - and fetch_order does not work (certain exchanges don't return cancelled orders) - :param order_id: stoploss-order-id to cancel - :param pair: Pair corresponding to order_id - :param amount: Amount to use for fake response - :return: Result from either cancel_order if usable, or fetch_order - """ - corder = self.cancel_stoploss_order(order_id, pair) - if self.is_cancel_order_result_suitable(corder): - return corder - try: - order = self.fetch_stoploss_order(order_id, pair) - except InvalidOrderException: - logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} - - return order - - @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) - def fetch_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - return self.fetch_dry_run_order(order_id) - try: - return self._api.fetch_order(order_id, pair) - except ccxt.OrderNotFound as e: - raise RetryableOrderError( - f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - # Assign method to fetch_stoploss_order to allow easy overriding in other classes - fetch_stoploss_order = fetch_order - - def fetch_order_or_stoploss_order(self, order_id: str, pair: str, - stoploss_order: bool = False) -> Dict: - """ - Simple wrapper calling either fetch_order or fetch_stoploss_order depending on - the stoploss_order parameter - :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. - """ - if stoploss_order: - return self.fetch_stoploss_order(order_id, pair) - return self.fetch_order(order_id, pair) - - @staticmethod - def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], - range_required: bool = True): - """ - Get next greater value in the list. - Used by fetch_l2_order_book if the api only supports a limited range - """ - if not limit_range: - return limit - - result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) - if not range_required and limit > result: - # Range is not required - we can use None as parameter. - return None - return result - - @retrier - def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: - """ - Get L2 order book from exchange. - Can be limited to a certain amount (if supported). - Returns a dict in the format - {'asks': [price, volume], 'bids': [price, volume]} - """ - limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], - self._ft_has['l2_limit_range_required']) - try: - - return self._api.fetch_l2_order_book(pair, limit1) - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching order book.' - f'Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: """ From 4e1425023e0aa4c0530b5c2ffb8a2e2c20dfd772 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:20:26 +0200 Subject: [PATCH 3/6] Further reorder exchange methods --- freqtrade/exchange/exchange.py | 242 +++++++++++++++++---------------- 1 file changed, 124 insertions(+), 118 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c3c4e8e5a..7514572fc 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -911,6 +911,128 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + # Fee handling + + @retrier + def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ + if self._config['dry_run']: + return [] + if not self.exchange_has('fetchMyTrades'): + return [] + try: + # Allow 5s offset to catch slight time offsets (discovered in #1185) + # since needs to be int in milliseconds + my_trades = self._api.fetch_my_trades( + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + + return matched_trades + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def get_order_id_conditional(self, order: Dict[str, Any]) -> str: + return order['id'] + + @retrier + def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, + price: float = 1, taker_or_maker: str = 'maker') -> float: + try: + if self._config['dry_run'] and self._config.get('fee', None) is not None: + return self._config['fee'] + # validate that markets are loaded before trying to get fee + if self._api.markets is None or len(self._api.markets) == 0: + self._api.load_markets() + + return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round( + order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None + else: + # If Fee currency is a different currency + if not order['cost']: + # If cost is None or 0.0 -> falsy, return None + return None + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except ExchangeError: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: + """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + self.calculate_fee_rate(order)) + + # Historic data + def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: """ @@ -1078,6 +1200,8 @@ class Exchange: raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' f'for pair {pair}. Message: {e}') from e + # Fetch historic trades + @retrier_async async def _async_fetch_trades(self, pair: str, since: Optional[int] = None, @@ -1236,124 +1360,6 @@ class Exchange: self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) - @retrier - def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: - """ - Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. - The "since" argument passed in is coming from the database and is in UTC, - as timezone-native datetime object. - From the python documentation: - > Naive datetime instances are assumed to represent local time - Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the - transformation from local timezone to UTC. - This works for timezones UTC+ since then the result will contain trades from a few hours - instead of from the last 5 seconds, however fails for UTC- timezones, - since we're then asking for trades with a "since" argument in the future. - - :param order_id order_id: Order-id as given when creating the order - :param pair: Pair the order is for - :param since: datetime object of the order creation time. Assumes object is in UTC. - """ - if self._config['dry_run']: - return [] - if not self.exchange_has('fetchMyTrades'): - return [] - try: - # Allow 5s offset to catch slight time offsets (discovered in #1185) - # since needs to be int in milliseconds - my_trades = self._api.fetch_my_trades( - pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) - matched_trades = [trade for trade in my_trades if trade['order'] == order_id] - - return matched_trades - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - def get_order_id_conditional(self, order: Dict[str, Any]) -> str: - return order['id'] - - @retrier - def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, - price: float = 1, taker_or_maker: str = 'maker') -> float: - try: - if self._config['dry_run'] and self._config.get('fee', None) is not None: - return self._config['fee'] - # validate that markets are loaded before trying to get fee - if self._api.markets is None or len(self._api.markets) == 0: - self._api.load_markets() - - return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, - price=price, takerOrMaker=taker_or_maker)['rate'] - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - @staticmethod - def order_has_fee(order: Dict) -> bool: - """ - Verifies if the passed in order dict has the needed keys to extract fees, - and that these keys (currency, cost) are not empty. - :param order: Order or trade (one trade) dict - :return: True if the fee substructure contains currency and cost, false otherwise - """ - if not isinstance(order, dict): - return False - return ('fee' in order and order['fee'] is not None - and (order['fee'].keys() >= {'currency', 'cost'}) - and order['fee']['currency'] is not None - and order['fee']['cost'] is not None - ) - - def calculate_fee_rate(self, order: Dict) -> Optional[float]: - """ - Calculate fee rate if it's not given by the exchange. - :param order: Order or trade (one trade) dict - """ - if order['fee'].get('rate') is not None: - return order['fee'].get('rate') - fee_curr = order['fee']['currency'] - # Calculate fee based on order details - if fee_curr in self.get_pair_base_currency(order['symbol']): - # Base currency - divide by amount - return round( - order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) - elif fee_curr in self.get_pair_quote_currency(order['symbol']): - # Quote currency - divide by cost - return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None - else: - # If Fee currency is a different currency - if not order['cost']: - # If cost is None or 0.0 -> falsy, return None - return None - try: - comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) - tick = self.fetch_ticker(comb) - - fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') - return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) - except ExchangeError: - return None - - def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: - """ - Extract tuple of cost, currency, rate. - Requires order_has_fee to run first! - :param order: Order or trade (one trade) dict - :return: Tuple with cost, currency, rate of the given fee dict - """ - return (order['fee']['cost'], - order['fee']['currency'], - self.calculate_fee_rate(order)) - def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: return exchange_name in ccxt_exchanges(ccxt_module) From 12916243ecf241bb3332fd36bff844cb49011717 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:30:19 +0200 Subject: [PATCH 4/6] Move get_buy_rate to exchange class --- freqtrade/exchange/exchange.py | 51 +++++++++++++++++++++++++++++++-- freqtrade/freqtradebot.py | 49 ++----------------------------- tests/exchange/test_exchange.py | 44 ++++++++++++++++++++++++++++ tests/test_freqtradebot.py | 43 --------------------------- 4 files changed, 95 insertions(+), 92 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7514572fc..42b518566 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -22,8 +22,8 @@ from pandas import DataFrame from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, - InvalidOrderException, OperationalException, RetryableOrderError, - TemporaryError) + InvalidOrderException, OperationalException, PricingError, + RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, retrier_async) @@ -88,6 +88,7 @@ class Exchange: # Cache for 10 minutes ... self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10) + self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles self._klines: Dict[Tuple[str, str], DataFrame] = {} @@ -911,6 +912,52 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def get_buy_rate(self, pair: str, refresh: bool) -> float: + """ + Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: float: Price + :raises PricingError if orderbook price could not be determined. + """ + if not refresh: + rate = self._buy_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached buy rate for {pair}.") + return rate + + bid_strategy = self._config.get('bid_strategy', {}) + if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + + order_book_top = bid_strategy.get('order_book_top', 1) + order_book = self.fetch_l2_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) + # top 1 = index 0 + try: + rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + except (IndexError, KeyError) as e: + logger.warning( + "Buy Price from orderbook could not be determined." + f"Orderbook: {order_book}" + ) + raise PricingError from e + logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " + f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") + used_rate = rate_from_l2 + else: + logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[bid_strategy['price_side']] + if ticker['last'] and ticker_rate > ticker['last']: + balance = bid_strategy['ask_last_balance'] + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) + used_rate = ticker_rate + + self._buy_rate_cache[pair] = used_rate + + return used_rate + # Fee handling @retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d7369ad47..d15748864 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -62,7 +62,6 @@ class FreqtradeBot(LoggingMixin): # Caching only applies to RPC methods, so prices for open trades are still # refreshed once every iteration. self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -396,50 +395,6 @@ class FreqtradeBot(LoggingMixin): return trades_created - def get_buy_rate(self, pair: str, refresh: bool) -> float: - """ - Calculates bid target between current ask price and last price - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: float: Price - """ - if not refresh: - rate = self._buy_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached buy rate for {pair}.") - return rate - - bid_strategy = self.config.get('bid_strategy', {}) - if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): - - order_book_top = bid_strategy.get('order_book_top', 1) - order_book = self.exchange.fetch_l2_order_book(pair, order_book_top) - logger.debug('order_book %s', order_book) - # top 1 = index 0 - try: - rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] - except (IndexError, KeyError) as e: - logger.warning( - "Buy Price from orderbook could not be determined." - f"Orderbook: {order_book}" - ) - raise PricingError from e - logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " - f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") - used_rate = rate_from_l2 - else: - logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[bid_strategy['price_side']] - if ticker['last'] and ticker_rate > ticker['last']: - balance = bid_strategy['ask_last_balance'] - ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) - used_rate = ticker_rate - - self._buy_rate_cache[pair] = used_rate - - return used_rate def create_trade(self, pair: str) -> bool: """ @@ -532,7 +487,7 @@ class FreqtradeBot(LoggingMixin): buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.get_buy_rate(pair, True) + buy_limit_requested = self.exchange.get_buy_rate(pair, True) if not buy_limit_requested: raise PricingError('Could not determine buy price.') @@ -657,7 +612,7 @@ class FreqtradeBot(LoggingMixin): """ Sends rpc notification when a buy cancel occurred. """ - current_rate = self.get_buy_rate(trade.pair, False) + current_rate = self.exchange.get_buy_rate(trade.pair, False) msg = { 'trade_id': trade.id, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b6b395802..0e68d054d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1684,6 +1684,50 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('bid', 21, 20, 10, 0.0, 20), # Full bid side + ('bid', 21, 20, 10, 1.0, 10), # Full last side + ('bid', 21, 20, 10, 0.5, 15), # Between bid and last + ('bid', 21, 20, 10, 0.7, 13), # Between bid and last + ('bid', 21, 20, 10, 0.3, 17), # Between bid and last + ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 6, 5, None, 1, 5), # last not available - uses bid + ('bid', 6, 5, None, 0, 5), # last not available - uses bid +]) +def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + default_conf['bid_strategy']['ask_last_balance'] = last_ab + default_conf['bid_strategy']['price_side'] = side + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'last': last, 'bid': bid}) + + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert exchange.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4d9284a2f..68d861ef2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -751,49 +751,6 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] -@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ - ('ask', 20, 19, 10, 0.0, 20), # Full ask side - ('ask', 20, 19, 10, 1.0, 10), # Full last side - ('ask', 20, 19, 10, 0.5, 15), # Between ask and last - ('ask', 20, 19, 10, 0.7, 13), # Between ask and last - ('ask', 20, 19, 10, 0.3, 17), # Between ask and last - ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask - ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask - ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask - ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask - ('ask', 4, 5, None, 1, 4), # last not available - uses ask - ('ask', 4, 5, None, 0, 4), # last not available - uses ask - ('bid', 21, 20, 10, 0.0, 20), # Full bid side - ('bid', 21, 20, 10, 1.0, 10), # Full last side - ('bid', 21, 20, 10, 0.5, 15), # Between bid and last - ('bid', 21, 20, 10, 0.7, 13), # Between bid and last - ('bid', 21, 20, 10, 0.3, 17), # Between bid and last - ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid - ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid - ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid - ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid - ('bid', 6, 5, None, 1, 5), # last not available - uses bid - ('bid', 6, 5, None, 0, 5), # last not available - uses bid -]) -def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, - last, last_ab, expected) -> None: - caplog.set_level(logging.DEBUG) - default_conf['bid_strategy']['ask_last_balance'] = last_ab - default_conf['bid_strategy']['price_side'] = side - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': ask, 'last': last, 'bid': bid}) - - assert freqtrade.get_buy_rate('ETH/BTC', True) == expected - assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - - assert freqtrade.get_buy_rate('ETH/BTC', False) == expected - assert log_has("Using cached buy rate for ETH/BTC.", caplog) - # Running a 2nd time with Refresh on! - caplog.clear() - assert freqtrade.get_buy_rate('ETH/BTC', True) == expected - assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) From bd1984386e1377b3e5dd297a4da60eab6ae55988 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:39:18 +0200 Subject: [PATCH 5/6] Move get_sell_rate to exchange class --- freqtrade/exchange/exchange.py | 53 ++++++++++++++++ freqtrade/freqtradebot.py | 71 +++------------------ freqtrade/rpc/rpc.py | 8 +-- tests/exchange/test_exchange.py | 104 ++++++++++++++++++++++++++++++- tests/rpc/test_rpc.py | 6 +- tests/rpc/test_rpc_apiserver.py | 2 +- tests/test_freqtradebot.py | 105 +------------------------------- 7 files changed, 172 insertions(+), 177 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 42b518566..67676d4e0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -88,6 +88,10 @@ class Exchange: # Cache for 10 minutes ... self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10) + # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Caching only applies to RPC methods, so prices for open trades are still + # refreshed once every iteration. + self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles @@ -912,6 +916,15 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, + order_book_min: int = 1): + """ + Helper generator to query orderbook in loop (used for early sell-order placing) + """ + order_book = self.fetch_l2_order_book(pair, order_book_max) + for i in range(order_book_min, order_book_max + 1): + yield order_book[side][i - 1][0] + def get_buy_rate(self, pair: str, refresh: bool) -> float: """ Calculates bid target between current ask price and last price @@ -958,6 +971,46 @@ class Exchange: return used_rate + def get_sell_rate(self, pair: str, refresh: bool) -> float: + """ + Get sell rate - either using ticker bid or first bid based on orderbook + or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: Bid rate + :raises PricingError if price could not be determined. + """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached sell rate for {pair}.") + return rate + + ask_strategy = self._config.get('ask_strategy', {}) + if ask_strategy.get('use_order_book', False): + # This code is only used for notifications, selling uses the generator directly + logger.info( + f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." + ) + try: + rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) + except (IndexError, KeyError) as e: + logger.warning("Sell Price at location from orderbook could not be determined.") + raise PricingError from e + else: + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[ask_strategy['price_side']] + if ticker['last'] and ticker_rate < ticker['last']: + balance = ask_strategy.get('bid_last_balance', 0.0) + ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) + rate = ticker_rate + + if rate is None: + raise PricingError(f"Sell-Rate for {pair} was empty.") + self._sell_rate_cache[pair] = rate + return rate + # Fee handling @retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d15748864..8628931b6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,7 +10,6 @@ from threading import Lock from typing import Any, Dict, List, Optional import arrow -from cachetools import TTLCache from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency @@ -58,11 +57,6 @@ class FreqtradeBot(LoggingMixin): # Init objects self.config = config - # Cache values for 1800 to avoid frequent polling of the exchange for prices - # Caching only applies to RPC methods, so prices for open trades are still - # refreshed once every iteration. - self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options @@ -395,7 +389,6 @@ class FreqtradeBot(LoggingMixin): return trades_created - def create_trade(self, pair: str) -> bool: """ Check the implemented trading strategy for buy signals. @@ -678,56 +671,6 @@ class FreqtradeBot(LoggingMixin): return trades_closed - def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, - order_book_min: int = 1): - """ - Helper generator to query orderbook in loop (used for early sell-order placing) - """ - order_book = self.exchange.fetch_l2_order_book(pair, order_book_max) - for i in range(order_book_min, order_book_max + 1): - yield order_book[side][i - 1][0] - - def get_sell_rate(self, pair: str, refresh: bool) -> float: - """ - Get sell rate - either using ticker bid or first bid based on orderbook - The orderbook portion is only used for rpc messaging, which would otherwise fail - for BitMex (has no bid/ask in fetch_ticker) - or remain static in any other case since it's not updating. - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: Bid rate - """ - if not refresh: - rate = self._sell_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached sell rate for {pair}.") - return rate - - ask_strategy = self.config.get('ask_strategy', {}) - if ask_strategy.get('use_order_book', False): - # This code is only used for notifications, selling uses the generator directly - logger.info( - f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." - ) - try: - rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) - except (IndexError, KeyError) as e: - logger.warning("Sell Price at location from orderbook could not be determined.") - raise PricingError from e - else: - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[ask_strategy['price_side']] - if ticker['last'] and ticker_rate < ticker['last']: - balance = ask_strategy.get('bid_last_balance', 0.0) - ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) - rate = ticker_rate - - if rate is None: - raise PricingError(f"Sell-Rate for {pair} was empty.") - self._sell_rate_cache[pair] = rate - return rate - def handle_trade(self, trade: Trade) -> bool: """ Sells the current pair if the threshold is reached and updates the trade record. @@ -755,9 +698,9 @@ class FreqtradeBot(LoggingMixin): logger.debug(f'Using order book between {order_book_min} and {order_book_max} ' f'for selling {trade.pair}...') - order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", - order_book_min=order_book_min, - order_book_max=order_book_max) + order_book = self.exchange._order_book_gen( + trade.pair, f"{config_ask_strategy['price_side']}s", + order_book_min=order_book_min, order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): try: sell_rate = next(order_book) @@ -770,14 +713,14 @@ class FreqtradeBot(LoggingMixin): f"{sell_rate:0.8f}") # Assign sell-rate to cache - otherwise sell-rate is never updated in the cache, # resulting in outdated RPC messages - self._sell_rate_cache[trade.pair] = sell_rate + self.exchange._sell_rate_cache[trade.pair] = sell_rate if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') - sell_rate = self.get_sell_rate(trade.pair, True) + sell_rate = self.exchange.get_sell_rate(trade.pair, True) if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True @@ -1209,7 +1152,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) # Use cached rates here - it was updated seconds ago. - current_rate = self.get_sell_rate(trade.pair, False) if not fill else None + current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" @@ -1254,7 +1197,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, False) + current_rate = self.exchange.get_sell_rate(trade.pair, False) profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c609bccb8..40d5eb583 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -171,7 +171,7 @@ class RPC: # calculate profit and send message to user if trade.is_open: try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (ExchangeError, PricingError): current_rate = NAN else: @@ -230,7 +230,7 @@ class RPC: for trade in trades: # calculate profit and send message to user try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN trade_percent = (100 * trade.calc_profit_ratio(current_rate)) @@ -386,7 +386,7 @@ class RPC: else: # Get current rate try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN profit_ratio = trade.calc_profit_ratio(rate=current_rate) @@ -556,7 +556,7 @@ class RPC: if not fully_canceled: # Get current rate and execute sell - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL) self._freqtrade.execute_sell(trade, current_rate, sell_reason) # ---- EOF def _exec_forcesell ---- diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 0e68d054d..5fa94e6c1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -11,7 +11,7 @@ import pytest from pandas import DataFrame from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, - OperationalException, TemporaryError) + OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff) @@ -1728,6 +1728,108 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, assert not log_has("Using cached buy rate for ETH/BTC.", caplog) +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ + ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side + ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side + ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat + ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid + ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid + ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid + ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side + ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side + ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat + ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask + ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask + ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask + ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), + ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), + ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), + ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), +]) +def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['bid_last_balance'] = last_ab + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'bid': bid, 'last': last}) + pair = "ETH/BTC" + + # Test regular mode + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + # Use caching + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +@pytest.mark.parametrize('side,expected', [ + ('bid', 0.043936), # Value from order_book_l2 fiture - bids side + ('ask', 0.043949), # Value from order_book_l2 fiture - asks side +]) +def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): + caplog.set_level(logging.DEBUG) + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = 'ask' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + # Test What happens if the exchange returns an empty orderbook. + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'bids': [[]], 'asks': [[]]}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError): + exchange.get_sell_rate(pair, True) + assert log_has("Sell Price at location from orderbook could not be determined.", caplog) + + +def test_get_sell_rate_exception(default_conf, mocker, caplog): + # Ticker on one side can be empty in certain circumstances. + default_conf['ask_strategy']['price_side'] = 'ask' + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': None, 'bid': 0.12, 'last': None}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'bid' + assert exchange.get_sell_rate(pair, True) == 0.12 + # Reverse sides + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.13, 'bid': None, 'last': None}) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'ask' + assert exchange.get_sell_rate(pair, True) == 0.13 + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index e7a968e37..7556dde6d 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -109,7 +109,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -217,7 +217,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert '-0.41% (-0.06)' == result[0][3] assert '-0.06' == f'{fiat_profit_sum:.2f}' - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] @@ -427,7 +427,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1a66b2e81..def2e43c6 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -834,7 +834,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) rc = client_get(client, f"{BASE_URI}/status") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 68d861ef2..9039328b0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -751,7 +751,6 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] - def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2480,7 +2479,7 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'freqtrade.exchange.Exchange', cancel_order=cancel_order_mock, ) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', return_value=0.245441) + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', return_value=0.245441) freqtrade = FreqtradeBot(default_conf) @@ -4029,108 +4028,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o assert log_has('Sell Price at location 1 from orderbook could not be determined.', caplog) -@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ - ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side - ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side - ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat - ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid - ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid - ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid - ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), - ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side - ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side - ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat - ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask - ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask - ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask - ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), - ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), - ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), - ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), -]) -def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, - last, last_ab, expected) -> None: - caplog.set_level(logging.DEBUG) - - default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['bid_last_balance'] = last_ab - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': ask, 'bid': bid, 'last': last}) - pair = "ETH/BTC" - - # Test regular mode - ft = get_patched_freqtradebot(mocker, default_conf) - rate = ft.get_sell_rate(pair, True) - assert not log_has("Using cached sell rate for ETH/BTC.", caplog) - assert isinstance(rate, float) - assert rate == expected - # Use caching - rate = ft.get_sell_rate(pair, False) - assert rate == expected - assert log_has("Using cached sell rate for ETH/BTC.", caplog) - - -@pytest.mark.parametrize('side,expected', [ - ('bid', 0.043936), # Value from order_book_l2 fiture - bids side - ('ask', 0.043949), # Value from order_book_l2 fiture - asks side -]) -def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): - caplog.set_level(logging.DEBUG) - # Test orderbook mode - default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_min'] = 1 - default_conf['ask_strategy']['order_book_max'] = 2 - pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - ft = get_patched_freqtradebot(mocker, default_conf) - rate = ft.get_sell_rate(pair, True) - assert not log_has("Using cached sell rate for ETH/BTC.", caplog) - assert isinstance(rate, float) - assert rate == expected - rate = ft.get_sell_rate(pair, False) - assert rate == expected - assert log_has("Using cached sell rate for ETH/BTC.", caplog) - - -def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): - # Test orderbook mode - default_conf['ask_strategy']['price_side'] = 'ask' - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_min'] = 1 - default_conf['ask_strategy']['order_book_max'] = 2 - pair = "ETH/BTC" - # Test What happens if the exchange returns an empty orderbook. - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', - return_value={'bids': [[]], 'asks': [[]]}) - ft = get_patched_freqtradebot(mocker, default_conf) - with pytest.raises(PricingError): - ft.get_sell_rate(pair, True) - assert log_has("Sell Price at location from orderbook could not be determined.", caplog) - - -def test_get_sell_rate_exception(default_conf, mocker, caplog): - # Ticker on one side can be empty in certain circumstances. - default_conf['ask_strategy']['price_side'] = 'ask' - pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': None, 'bid': 0.12, 'last': None}) - ft = get_patched_freqtradebot(mocker, default_conf) - with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - ft.get_sell_rate(pair, True) - - ft.config['ask_strategy']['price_side'] = 'bid' - assert ft.get_sell_rate(pair, True) == 0.12 - # Reverse sides - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': 0.13, 'bid': None, 'last': None}) - with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - ft.get_sell_rate(pair, True) - - ft.config['ask_strategy']['price_side'] = 'ask' - assert ft.get_sell_rate(pair, True) == 0.13 - - def test_startup_state(default_conf, mocker): default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} From 387f3bbc5d06b1123e03b586c7fcad4cce9d3868 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:43:47 +0200 Subject: [PATCH 6/6] Adjust missed tests --- tests/test_freqtradebot.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 9039328b0..66866a8fc 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -759,13 +759,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order stake_amount = 2 bid = 0.11 buy_rate_mock = MagicMock(return_value=bid) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_buy_rate=buy_rate_mock, - ) buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + get_buy_rate=buy_rate_mock, fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, @@ -856,7 +853,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order assert not freqtrade.execute_buy(pair, stake_amount) # Fail to get price... - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_buy_rate', MagicMock(return_value=0.0)) + mocker.patch('freqtrade.exchange.Exchange.get_buy_rate', MagicMock(return_value=0.0)) with pytest.raises(PricingError, match="Could not determine buy price."): freqtrade.execute_buy(pair, stake_amount) @@ -864,10 +861,6 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_buy_rate=MagicMock(return_value=0.11), - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -876,6 +869,7 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) - 'last': 0.00001172 }), buy=MagicMock(return_value=limit_buy_order), + get_buy_rate=MagicMock(return_value=0.11), get_min_pair_stake_amount=MagicMock(return_value=1), get_fee=fee, ) @@ -3934,7 +3928,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_buy_rate('ETH/BTC', True) == 0.043935 + assert freqtrade.exchange.get_buy_rate('ETH/BTC', True) == 0.043935 assert ticker_mock.call_count == 0 @@ -3956,7 +3950,7 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. with pytest.raises(PricingError): - freqtrade.get_buy_rate('ETH/BTC', refresh=True) + freqtrade.exchange.get_buy_rate('ETH/BTC', refresh=True) assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog)