From 15692421d96eb9175cb6d4e0117ab28a19823de9 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Wed, 31 Jan 2018 23:25:01 +0100 Subject: [PATCH 01/17] Add Binance exchange support --- freqtrade/exchange/__init__.py | 14 +- freqtrade/exchange/binance.py | 374 ++++++++++++++++++ freqtrade/exchange/bittrex.py | 9 +- freqtrade/exchange/interface.py | 12 +- freqtrade/main.py | 43 +- freqtrade/optimize/backtesting.py | 27 +- freqtrade/rpc/telegram.py | 37 ++ freqtrade/tests/exchange/test_exchange.py | 12 +- .../tests/exchange/test_exchange_binance.py | 349 ++++++++++++++++ .../tests/exchange/test_exchange_bittrex.py | 12 +- requirements.txt | 1 + 11 files changed, 859 insertions(+), 31 deletions(-) create mode 100644 freqtrade/exchange/binance.py create mode 100644 freqtrade/tests/exchange/test_exchange_binance.py diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index dc85bfedb..8d36c51e3 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -11,6 +11,7 @@ from cachetools import cached, TTLCache from freqtrade import OperationalException from freqtrade.exchange.bittrex import Bittrex +from freqtrade.exchange.binance import Binance from freqtrade.exchange.interface import Exchange logger = logging.getLogger(__name__) @@ -28,6 +29,7 @@ class Exchanges(enum.Enum): Maps supported exchange names to correspondent classes. """ BITTREX = Bittrex + BINANCE = Binance def init(config: dict) -> None: @@ -143,14 +145,14 @@ def get_ticker_history(pair: str, tick_interval) -> List[Dict]: return _API.get_ticker_history(pair, tick_interval) -def cancel_order(order_id: str) -> None: +def cancel_order(order_id: str, pair: str) -> None: if _CONF['dry_run']: return - return _API.cancel_order(order_id) + return _API.cancel_order(order_id, pair) -def get_order(order_id: str) -> Dict: +def get_order(order_id: str, pair: str) -> Dict: if _CONF['dry_run']: order = _DRY_RUN_OPEN_ORDERS[order_id] order.update({ @@ -158,7 +160,7 @@ def get_order(order_id: str) -> Dict: }) return order - return _API.get_order(order_id) + return _API.get_order(order_id, pair) def get_pair_detail_url(pair: str) -> str: @@ -183,3 +185,7 @@ def get_fee() -> float: def get_wallet_health() -> List[Dict]: return _API.get_wallet_health() + + +def get_trade_qty(pair: str) -> tuple: + return _API.get_trade_qty(pair) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py new file mode 100644 index 000000000..c1dad53cd --- /dev/null +++ b/freqtrade/exchange/binance.py @@ -0,0 +1,374 @@ +import logging +import datetime +import json +import http +from typing import List, Dict, Optional + +from binance.client import Client as _Binance +from binance.exceptions import BinanceAPIException +from binance.enums import * +from decimal import Decimal + +from freqtrade import OperationalException +from freqtrade.exchange.interface import Exchange + +logger = logging.getLogger(__name__) + +_API: _Binance = None +_EXCHANGE_CONF: dict = {} +_CONF: dict = {} + + +class Binance(Exchange): + """ + Binance API wrapper. + """ + # Base URL and API endpoints + BASE_URL: str = 'https://www.binance.com' + + def __init__(self, config: dict) -> None: + global _API, _EXCHANGE_CONF, _CONF + + _EXCHANGE_CONF.update(config) + + _API = _Binance(_EXCHANGE_CONF['key'], _EXCHANGE_CONF['secret']) + + def _pair_to_symbol(self, pair, seperator='') -> str: + """ + Turns freqtrade pair into Binance symbol + - Freqtrade pair = _ + i.e.: BTC_XALT + - Binance symbol = + i.e.: XALTBTC + """ + + pair_currencies = pair.split('_') + + return '{0}{1}{2}'.format(pair_currencies[1], seperator, pair_currencies[0]) + + def _symbol_to_pair(self, symbol) -> str: + """ + Turns Binance symbol into freqtrade pair + - Freqtrade pair = _ + i.e.: BTC_XALT + - Binance symbol = + i.e.: XALTBTC + """ + stake = _EXCHANGE_CONF['stake_currency'] + + symbol_stake_currency = symbol[-len(stake):] + symbol_currency = symbol[:-len(stake)] + + return '{0}_{1}'.format(symbol_stake_currency, symbol_currency) + + @staticmethod + def _handle_exception(excepter) -> None: + """ + Validates the given Binance response/exception + and raises a ContentDecodingError if a non-fatal issue happened. + """ + # Could to alternate exception handling for specific exceptions/errors + # See: http://python-binance.readthedocs.io/en/latest/binance.html#module-binance.exceptions + if type(excepter) == http.client.RemoteDisconnected: + logger.info( + 'Got HTTP error from Binance: %s' % excepter + ) + return True + + if type(excepter) == json.decoder.JSONDecodeError: + logger.info( + 'Got JSON error from Binance: %s' % excepter + ) + return True + + if type(excepter) == BinanceAPIException: + + logger.info( + 'Got API error from Binance: %s' % excepter + ) + + return True + + raise type(excepter)(excepter.args) + + @property + def fee(self) -> float: + # 0.1 %: See https://support.binance.com/hc/en-us + # /articles/115000429332-Fee-Structure-on-Binance + return 0.001 + + def buy(self, pair: str, rate: float, amount: float) -> str: + + symbol = self._pair_to_symbol(pair) + + try: + data = _API.order_limit_buy( + symbol=symbol, + quantity="{0:.8f}".format(amount), + price="{0:.8f}".format(rate)) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=str(e), + pair=pair, + rate=Decimal(rate), + amount=Decimal(amount))) + + return data['orderId'] + + def sell(self, pair: str, rate: float, amount: float) -> str: + + symbol = self._pair_to_symbol(pair) + + try: + data = _API.order_limit_sell( + symbol=symbol, + quantity="{0:.8f}".format(amount), + price="{0:.8f}".format(rate)) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException( + '{message} params=({pair}, {rate}, {amount})'.format( + message=str(e), + pair=pair, + rate=rate, + amount=amount)) + + return data['orderId'] + + def get_balance(self, currency: str) -> float: + + try: + data = _API.get_asset_balance(asset=currency) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message} params=({currency})'.format( + message=str(e), + currency=currency)) + + return float(data['free'] or 0.0) + + def get_balances(self) -> List[Dict]: + + try: + data = _API.get_account() + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message}'.format(message=str(e))) + + balances = data['balances'] + + currency_balances = [] + for currency in balances: + balance = {} + + if float(currency['free']) == 0 and float(currency['locked']) == 0: + continue + balance['Currency'] = currency.pop('asset') + balance['Available'] = currency.pop('free') + balance['Pending'] = currency.pop('locked') + balance['Balance'] = float(balance['Available']) + float(balance['Pending']) + + currency_balances.append(balance) + + return currency_balances + + def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: + + symbol = self._pair_to_symbol(pair) + + try: + data = _API.get_ticker(symbol=symbol) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message} params=({pair})'.format( + message=str(e), + pair=pair)) + + return { + 'bid': float(data['bidPrice']), + 'ask': float(data['askPrice']), + 'last': float(data['lastPrice']), + } + + def get_ticker_history(self, pair: str, tick_interval: int) -> List[Dict]: + + INTERVAL_ENUM = eval('KLINE_INTERVAL_' + str(tick_interval) + 'MINUTE') + + if INTERVAL_ENUM in ['', None]: + raise ValueError('Cannot parse tick_interval: {}'.format(tick_interval)) + + symbol = self._pair_to_symbol(pair) + + try: + data = _API.get_klines(symbol=symbol, interval=INTERVAL_ENUM) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message} params=({pair})'.format( + message=str(e), + pair=pair)) + + tick_data = [] + + for tick in data: + t = {} + t['O'] = float(tick[1]) + t['H'] = float(tick[2]) + t['L'] = float(tick[3]) + t['C'] = float(tick[4]) + t['V'] = float(tick[5]) + t['T'] = datetime.datetime.fromtimestamp(int(tick[6])/1000).isoformat() + t['BV'] = float(tick[7]) + + tick_data.append(t) + + return tick_data + + def get_order(self, order_id: str, pair: str) -> Dict: + + symbol = self._pair_to_symbol(pair) + + try: + data = _API.get_all_orders(symbol=symbol, orderId=order_id) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException( + '{message} params=({symbol},{order_id})'.format( + message=str(e), + symbol=symbol, + order_id=order_id)) + + order = {} + + for o in data: + + if o['orderId'] == int(order_id): + + order['id'] = o['orderId'] + order['type'] = "{}_{}".format(o['type'], o['side']) + order['pair'] = self._symbol_to_pair(o['symbol']) + order['opened'] = datetime.datetime.fromtimestamp( + int(o['time'])/1000).isoformat() + order['closed'] = datetime.datetime.fromtimestamp( + int(o['time'])/1000).isoformat()\ + if o['status'] == 'FILLED' else None + order['rate'] = float(o['price']) + order['amount'] = float(o['origQty']) + order['remaining'] = int(float(o['origQty']) - float(o['executedQty'])) + + return order + + def cancel_order(self, order_id: str, pair: str) -> None: + + symbol = self._pair_to_symbol(pair) + + try: + data = _API.cancel_order(symbol=symbol, orderId=order_id) + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message} params=({order_id})'.format( + message=str(e), + order_id=order_id)) + + return data + + def get_pair_detail_url(self, pair: str) -> str: + symbol = self._pair_to_symbol(pair, '_') + return 'https://www.binance.com/indexSpa.html#/trade/index?symbol={}'.format(symbol) + + def get_markets(self) -> List[str]: + try: + data = _API.get_all_tickers() + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message}'.format(message=str(e))) + + markets = [] + + stake = _EXCHANGE_CONF['stake_currency'] + + for t in data: + symbol = t['symbol'] + symbol_stake_currency = symbol[-len(stake):] + + if symbol_stake_currency == stake: + pair = self._symbol_to_pair(symbol) + markets.append(pair) + + return markets + + def get_market_summaries(self) -> List[Dict]: + + try: + data = _API.get_ticker() + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message}'.format(message=str(e))) + + market_summaries = [] + + for t in data: + market = {} + + # Looks like this one is only one actually used + market['MarketName'] = self._symbol_to_pair(t['symbol']) + + market['High'] = t['highPrice'] + market['Low'] = t['lowPrice'] + market['Volume'] = t['volume'] + market['Last'] = t['lastPrice'] + market['TimeStamp'] = t['closeTime'] + market['BaseVolume'] = t['volume'] + market['Bid'] = t['bidPrice'] + market['Ask'] = t['askPrice'] + market['OpenBuyOrders'] = None # TODO: Implement me (or dont care) + market['OpenSellOrders'] = None # TODO: Implement me (or dont care) + market['PrevDay'] = t['prevClosePrice'] + market['Created'] = None # TODO: Implement me (or don't care) + + market_summaries.append(market) + + return market_summaries + + def get_trade_qty(self, pair: str) -> tuple: + + try: + data = _API.get_exchange_info() + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message}'.format(message=str(e))) + + symbol = self._pair_to_symbol(pair) + + for s in data['symbols']: + + if symbol == s['symbol']: + + for f in s['filters']: + + if f['filterType'] == 'LOT_SIZE': + + return (float(f['minQty']), float(f['maxQty']), float(f['stepSize'])) + + return (None, None, None) + + def get_wallet_health(self) -> List[Dict]: + + try: + data = _API.get_exchange_info() + except Exception as e: + Binance._handle_exception(e) + raise OperationalException('{message}'.format(message=str(e))) + + wallet_health = [] + + for s in data['symbols']: + wallet = {} + wallet['Currency'] = s['baseAsset'] + wallet['IsActive'] = True if s['status'] == 'TRADING' else False + wallet['LastChecked'] = None # TODO + wallet['Notice'] = s['status'] if s['status'] != 'TRADING' else '' + + wallet_health.append(wallet) + + return wallet_health diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 2be81be2d..63ad61d5e 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -157,7 +157,8 @@ class Bittrex(Exchange): return data['result'] - def get_order(self, order_id: str) -> Dict: + def get_order(self, order_id: str, pair: str) -> Dict: + data = _API.get_order(order_id) if not data['success']: Bittrex._validate_response(data) @@ -176,7 +177,8 @@ class Bittrex(Exchange): 'closed': data['Closed'], } - def cancel_order(self, order_id: str) -> None: + def cancel_order(self, order_id: str, pair: str) -> None: + data = _API.cancel(order_id) if not data['success']: Bittrex._validate_response(data) @@ -212,3 +214,6 @@ class Bittrex(Exchange): 'LastChecked': entry['Health']['LastChecked'], 'Notice': entry['Currency'].get('Notice'), } for entry in data['result']] + + def get_trade_qty(self, pair: str) -> tuple: + return (None, None, None) diff --git a/freqtrade/exchange/interface.py b/freqtrade/exchange/interface.py index 6121a98b3..cb1b01c5a 100644 --- a/freqtrade/exchange/interface.py +++ b/freqtrade/exchange/interface.py @@ -94,7 +94,7 @@ class Exchange(ABC): ] """ - def get_order(self, order_id: str) -> Dict: + def get_order(self, order_id: str, pair: str) -> Dict: """ Get order details for the given order_id. :param order_id: ID as str @@ -111,7 +111,7 @@ class Exchange(ABC): """ @abstractmethod - def cancel_order(self, order_id: str) -> None: + def cancel_order(self, order_id: str, pair: str) -> None: """ Cancels order for given order_id. :param order_id: ID as str @@ -170,3 +170,11 @@ class Exchange(ABC): }, ... """ + + @abstractmethod + def get_trade_qty(self, pair: str) -> tuple: + """ + Returns a tuple of trade quantity limits + :return: tuple, format: ( min_qty: str, max_qty: str, step_qty: str ) + ... + """ diff --git a/freqtrade/main.py b/freqtrade/main.py index 163597eee..3b365f549 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -84,7 +84,7 @@ def process_maybe_execute_sell(trade: Trade, interval: int) -> bool: if trade.open_order_id: # Update trade with order values logger.info('Got open order for %s', trade) - trade.update(exchange.get_order(trade.open_order_id)) + trade.update(exchange.get_order(trade.open_order_id, trade.pair)) if trade.is_open and trade.open_order_id is None: # Check if we can sell our current pair @@ -151,7 +151,7 @@ def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool: """Buy timeout - cancel order :return: True if order was fully cancelled """ - exchange.cancel_order(trade.open_order_id) + exchange.cancel_order(trade.open_order_id, trade.pair) if order['remaining'] == order['amount']: # if trade is not partially completed, just delete the trade Trade.session.delete(trade) @@ -182,7 +182,7 @@ def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool: """ if order['remaining'] == order['amount']: # if trade is not partially completed, just cancel the trade - exchange.cancel_order(trade.open_order_id) + exchange.cancel_order(trade.open_order_id, trade.pair) trade.close_rate = None trade.close_profit = None trade.close_date = None @@ -207,7 +207,7 @@ def check_handle_timedout(timeoutvalue: int) -> None: for trade in Trade.query.filter(Trade.open_order_id.isnot(None)).all(): try: - order = exchange.get_order(trade.open_order_id) + order = exchange.get_order(trade.open_order_id, trade.pair) except requests.exceptions.RequestException: logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) continue @@ -400,15 +400,42 @@ def create_trade(stake_amount: float, interval: int) -> bool: else: return False - # Calculate amount + min_qty = None + max_qty = None + step_qty = None + + (min_qty, max_qty, step_qty) = exchange.get_trade_qty(pair) + + # Calculate bid price buy_limit = get_target_bid(exchange.get_ticker(pair)) + + # Calculate base amount amount = stake_amount / buy_limit - order_id = exchange.buy(pair, buy_limit, amount) + # if amount above max qty: just buy max qty + if max_qty: + if amount > max_qty: + amount = max_qty + + if min_qty: + if amount < min_qty: + raise DependencyException( + 'stake amount is too low (min_qty={})'.format(min_qty) + ) + + # make trade exact amount of step qty + if step_qty: + real_amount = (amount // step_qty) * step_qty + else: + real_amount = amount + + order_id = exchange.buy(pair, buy_limit, real_amount) + + real_stake_amount = buy_limit * real_amount fiat_converter = CryptoToFiatConverter() stake_amount_fiat = fiat_converter.convert_amount( - stake_amount, + real_stake_amount, _CONF['stake_currency'], _CONF['fiat_display_currency'] ) @@ -418,7 +445,7 @@ def create_trade(stake_amount: float, interval: int) -> bool: exchange.get_name().upper(), pair.replace('_', '/'), exchange.get_pair_detail_url(pair), - buy_limit, stake_amount, _CONF['stake_currency'], + buy_limit, real_stake_amount, _CONF['stake_currency'], stake_amount_fiat, _CONF['fiat_display_currency'] )) # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d6d016aba..542ae19f6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,12 +6,12 @@ from typing import Dict, Tuple import arrow from pandas import DataFrame, Series from tabulate import tabulate +from freqtrade import OperationalException import freqtrade.misc as misc import freqtrade.optimize as optimize from freqtrade import exchange from freqtrade.analyze import populate_buy_trend, populate_sell_trend -from freqtrade.exchange import Bittrex from freqtrade.main import should_sell from freqtrade.persistence import Trade from freqtrade.strategy.strategy import Strategy @@ -116,7 +116,19 @@ def backtest(args) -> DataFrame: records = [] trades = [] trade_count_lock: dict = {} - exchange._API = Bittrex({'key': '', 'secret': ''}) + + # Monkey patch config + from freqtrade import main + exchange_config = main._CONF['exchange'] + + exchange_name = exchange_config['name'] + try: + exchange_class = exchange.Exchanges[exchange_name.upper()].value + except KeyError: + raise OperationalException('Exchange {} is not supported'.format( + exchange_name)) + + exchange._API = exchange_class({'key': '', 'secret': ''}) for pair, pair_data in processed.items(): pair_data['buy'], pair_data['sell'] = 0, 0 ticker = populate_sell_trend(populate_buy_trend(pair_data)) @@ -167,8 +179,6 @@ def start(args): format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', ) - exchange._API = Bittrex({'key': '', 'secret': ''}) - logger.info('Using config: %s ...', args.config) config = misc.load_config(args.config) @@ -184,6 +194,15 @@ def start(args): logger.info('Using ticker_interval: %d ...', strategy.ticker_interval) + exchange_name = config['exchange']['name'] + try: + exchange_class = exchange.Exchanges[exchange_name.upper()].value + except KeyError: + raise OperationalException('Exchange {} is not supported'.format( + exchange_name)) + + exchange._API = exchange_class({'key': '', 'secret': ''}) + data = {} pairs = config['exchange']['pair_whitelist'] if args.live: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ea170baa1..0cbff321d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -396,6 +396,43 @@ def _version(bot: Bot, update: Update) -> None: send_msg('*Version:* `{}`'.format(__version__), bot=bot) +<<<<<<< HEAD +======= +def shorten_date(_date): + """ + Trim the date so it fits on small screens + """ + new_date = re.sub('seconds?', 'sec', _date) + new_date = re.sub('minutes?', 'min', new_date) + new_date = re.sub('hours?', 'h', new_date) + new_date = re.sub('days?', 'd', new_date) + new_date = re.sub('^an?', '1', new_date) + return new_date + + +def _exec_forcesell(trade: Trade) -> None: + # Check if there is there is an open order + if trade.open_order_id: + order = exchange.get_order(trade.open_order_id, trade.pair) + + # Cancel open LIMIT_BUY orders and close trade + if order and not order['closed'] and order['type'] == 'LIMIT_BUY': + exchange.cancel_order(trade.open_order_id, trade.pair) + trade.close(order.get('rate') or trade.open_rate) + # TODO: sell amount which has been bought already + return + + # Ignore trades with an attached LIMIT_SELL order + if order and not order['closed'] and order['type'] == 'LIMIT_SELL': + return + + # Get current rate and execute sell + current_rate = exchange.get_ticker(trade.pair, False)['bid'] + from freqtrade.main import execute_sell + execute_sell(trade, current_rate) + + +>>>>>>> Add Binance exchange support def send_msg(msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 526ff54ab..605e7d9ab 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -227,7 +227,7 @@ def test_cancel_order_dry_run(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert cancel_order(order_id='123') is None + assert cancel_order(order_id='123',pair='ABC_XYZ') is None # Ensure that if not dry_run, we should call API @@ -237,7 +237,7 @@ def test_cancel_order(default_conf, mocker): api_mock = MagicMock() api_mock.cancel_order = MagicMock(return_value=123) mocker.patch('freqtrade.exchange._API', api_mock) - assert cancel_order(order_id='_') == 123 + assert cancel_order(order_id='_',pair='ABC_XYZ') == 123 def test_get_order(default_conf, mocker): @@ -245,16 +245,18 @@ def test_get_order(default_conf, mocker): mocker.patch.dict('freqtrade.exchange._CONF', default_conf) order = MagicMock() order.myid = 123 + order.pair = 'ABC_XYZ' exchange._DRY_RUN_OPEN_ORDERS['X'] = order - print(exchange.get_order('X')) - assert exchange.get_order('X').myid == 123 + print(exchange.get_order('X','ABC_XYZ')) + assert exchange.get_order('X','ABC_XYZ').myid == 123 + assert exchange.get_order('X','ABC_XYZ').pair == 'ABC_XYZ' default_conf['dry_run'] = False mocker.patch.dict('freqtrade.exchange._CONF', default_conf) api_mock = MagicMock() api_mock.get_order = MagicMock(return_value=456) mocker.patch('freqtrade.exchange._API', api_mock) - assert exchange.get_order('X') == 456 + assert exchange.get_order('X', 'ABC_XYZ') == 456 def test_get_name(default_conf, mocker): diff --git a/freqtrade/tests/exchange/test_exchange_binance.py b/freqtrade/tests/exchange/test_exchange_binance.py new file mode 100644 index 000000000..3c8ea909d --- /dev/null +++ b/freqtrade/tests/exchange/test_exchange_binance.py @@ -0,0 +1,349 @@ +# pragma pylint: disable=missing-docstring, C0103, protected-access, unused-argument + +from unittest.mock import MagicMock +import pytest +from freqtrade.exchange.binance import Binance +import freqtrade.exchange.binance as bin + + +# Eat this flake8 +# +------------------+ +# | binance.Binance | +# +------------------+ +# | +# (mock Fake_binance) +# | +# +-----------------------------+ +# | freqtrade.exchange.Binance | +# +-----------------------------+ +# Call into Binance will flow up to the +# external package binance.Binance. +# By inserting a mock, we redirect those +# calls. +# The faked binance API is called just 'fb' +# The freqtrade.exchange.Binance is a +# wrapper, and is called 'wb' + + +def _stub_config(): + return {'key': '', + 'secret': ''} + + +class FakeBinance(): + def __init__(self, success=True): + self.success = True # Believe in yourself + self.result = None + self.get_ticker_call_count = 0 + # This is really ugly, doing side-effect during instance creation + # But we're allowed to in testing-code + bin._API = MagicMock() + bin._API.order_limit_buy = self.fake_order_limit_buy + bin._API.order_limit_sell = self.fake_order_limit_sell + bin._API.get_asset_balance = self.fake_get_asset_balance + bin._API.get_account = self.fake_get_account + bin._API.get_ticker = self.fake_get_ticker + bin._API.get_klines = self.fake_get_klines + bin._API.get_all_orders = self.fake_get_all_orders + bin._API.cancel_order = self.fake_cancel_order + bin._API.get_all_tickers = self.fake_get_all_tickers + bin._API.get_exchange_info = self.fake_get_exchange_info + bin._EXCHANGE_CONF = {'stake_currency': 'BTC'} + + def fake_order_limit_buy(self, symbol, quantity, price): + return {"symbol": "BTCETH", + "orderId": 42, + "clientOrderId": "6gCrw2kRUAF9CvJDGP16IP", + "transactTime": 1507725176595, + "price": "0.00000000", + "origQty": "10.00000000", + "executedQty": "10.00000000", + "status": "FILLED", + "timeInForce": "GTC", + "type": "LIMIT", + "side": "BUY"} + + def fake_order_limit_sell(self, symbol, quantity, price): + return {"symbol": "BTCETH", + "orderId": 42, + "clientOrderId": "6gCrw2kRUAF9CvJDGP16IP", + "transactTime": 1507725176595, + "price": "0.00000000", + "origQty": "10.00000000", + "executedQty": "10.00000000", + "status": "FILLED", + "timeInForce": "GTC", + "type": "LIMIT", + "side": "SELL"} + + def fake_get_asset_balance(self, asset): + return { + "asset": "BTC", + "free": "4723846.89208129", + "locked": "0.00000000" + } + + def fake_get_account(self): + return { + "makerCommission": 15, + "takerCommission": 15, + "buyerCommission": 0, + "sellerCommission": 0, + "canTrade": True, + "canWithdraw": True, + "canDeposit": True, + "balances": [ + { + "asset": "BTC", + "free": "4723846.89208129", + "locked": "0.00000000" + }, + { + "asset": "LTC", + "free": "4763368.68006011", + "locked": "0.00000000" + } + ] + } + + def fake_get_ticker(self, symbol=None): + self.get_ticker_call_count += 1 + t = {"symbol": "ETHBTC", + "priceChange": "-94.99999800", + "priceChangePercent": "-95.960", + "weightedAvgPrice": "0.29628482", + "prevClosePrice": "0.10002000", + "lastPrice": "4.00000200", + "bidPrice": "4.00000000", + "askPrice": "4.00000200", + "openPrice": "99.00000000", + "highPrice": "100.00000000", + "lowPrice": "0.10000000", + "volume": "8913.30000000", + "openTime": 1499783499040, + "closeTime": 1499869899040, + "fristId": 28385, + "lastId": 28460, + "count": 76} + return t if symbol else [t] + + def fake_get_klines(self, symbol, interval): + return [[0, + "0", + "0", + "0", + "0", + "0", + 0, + "0", + 0, + "0", + "0", + "0"]] + + def fake_get_all_orders(self, symbol, orderId): + return [{"symbol": "LTCBTC", + "orderId": 42, + "clientOrderId": "myOrder1", + "price": "0.1", + "origQty": "1.0", + "executedQty": "0.0", + "status": "NEW", + "timeInForce": "GTC", + "type": "LIMIT", + "side": "BUY", + "stopPrice": "0.0", + "icebergQty": "0.0", + "status_code": "200", + "time": 1499827319559}] + + def fake_cancel_order(self, symbol, orderId): + return {"symbol": "LTCBTC", + "origClientOrderId": "myOrder1", + "orderId": 42, + "clientOrderId": "cancelMyOrder1"} + + def fake_get_all_tickers(self): + return [{"symbol": "LTCBTC", + "price": "4.00000200"}, + {"symbol": "ETHBTC", + "price": "0.07946600"}] + + def fake_get_exchange_info(self): + return { + "timezone": "UTC", + "serverTime": 1508631584636, + "rateLimits": [ + { + "rateLimitType": "REQUESTS", + "interval": "MINUTE", + "limit": 1200 + }, + { + "rateLimitType": "ORDERS", + "interval": "SECOND", + "limit": 10 + }, + { + "rateLimitType": "ORDERS", + "interval": "DAY", + "limit": 100000 + } + ], + "exchangeFilters": [], + "symbols": [ + { + "symbol": "ETHBTC", + "status": "TRADING", + "baseAsset": "ETH", + "baseAssetPrecision": 8, + "quoteAsset": "BTC", + "quotePrecision": 8, + "orderTypes": ["LIMIT", "MARKET"], + "icebergAllowed": False, + "filters": [ + { + "filterType": "PRICE_FILTER", + "minPrice": "0.00000100", + "maxPrice": "100000.00000000", + "tickSize": "0.00000100" + }, { + "filterType": "LOT_SIZE", + "minQty": "0.00100000", + "maxQty": "100000.00000000", + "stepSize": "0.00100000" + }, { + "filterType": "MIN_NOTIONAL", + "minNotional": "0.00100000" + } + ] + } + ] + } + + +# The freqtrade.exchange.binance is called wrap_binance +# to not confuse naming with binance.binance +def make_wrap_binance(): + conf = _stub_config() + wb = bin.Binance(conf) + return wb + + +def test_exchange_binance_class(): + conf = _stub_config() + b = Binance(conf) + assert isinstance(b, Binance) + slots = dir(b) + for name in ['fee', 'buy', 'sell', 'get_balance', 'get_balances', + 'get_ticker', 'get_ticker_history', 'get_order', + 'cancel_order', 'get_pair_detail_url', 'get_markets', + 'get_market_summaries', 'get_wallet_health']: + assert name in slots + # FIX: ensure that the slot is also a method in the class + # getattr(b, name) => bound method Binance.buy + # type(getattr(b, name)) => class 'method' + + +def test_exchange_binance_fee(): + fee = Binance.fee.__get__(Binance) + assert fee >= 0 and fee < 0.1 # Fee is 0-10 % + + +def test_exchange_binance_buy_good(): + wb = make_wrap_binance() + fb = FakeBinance() + uuid = wb.buy('BTC_ETH', 1, 1) + assert uuid == fb.fake_order_limit_buy(1, 2, 3)['orderId'] + + with pytest.raises(IndexError, match=r'.*'): + wb.buy('BAD', 1, 1) + + +def test_exchange_binance_sell_good(): + wb = make_wrap_binance() + fb = FakeBinance() + uuid = wb.sell('BTC_ETH', 1, 1) + assert uuid == fb.fake_order_limit_sell(1, 2, 3)['orderId'] + + with pytest.raises(IndexError, match=r'.*'): + uuid = wb.sell('BAD', 1, 1) + + +def test_exchange_binance_get_balance(): + wb = make_wrap_binance() + fb = FakeBinance() + bal = wb.get_balance('BTC') + assert str(bal) == fb.fake_get_asset_balance(1)['free'] + + +def test_exchange_binance_get_balances(): + wb = make_wrap_binance() + fb = FakeBinance() + bals = wb.get_balances() + assert len(bals) <= len(fb.fake_get_account()['balances']) + + +def test_exchange_binance_get_ticker(): + wb = make_wrap_binance() + FakeBinance() + + # Poll ticker, which updates the cache + tick = wb.get_ticker('BTC_ETH') + for x in ['bid', 'ask', 'last']: + assert x in tick + + +def test_exchange_binance_get_ticker_history_intervals(): + wb = make_wrap_binance() + FakeBinance() + for tick_interval in [1, 5]: + assert ([{'O': 0.0, 'H': 0.0, + 'L': 0.0, 'C': 0.0, + 'V': 0.0, 'T': '1970-01-01T01:00:00', 'BV': 0.0}] == + wb.get_ticker_history('BTC_ETH', tick_interval)) + + +def test_exchange_binance_get_ticker_history(): + wb = make_wrap_binance() + FakeBinance() + assert wb.get_ticker_history('BTC_ETH', 5) + + +def test_exchange_binance_get_order(): + wb = make_wrap_binance() + FakeBinance() + order = wb.get_order('42', 'BTC_LTC') + assert order['id'] == 42 + + +def test_exchange_binance_cancel_order(): + wb = make_wrap_binance() + FakeBinance() + assert wb.cancel_order('42', 'BTC_LTC')['orderId'] == 42 + + +def test_exchange_get_pair_detail_url(): + wb = make_wrap_binance() + FakeBinance() + assert wb.get_pair_detail_url('BTC_ETH') + + +def test_exchange_get_markets(): + wb = make_wrap_binance() + FakeBinance() + x = wb.get_markets() + assert len(x) > 0 + + +def test_exchange_get_market_summaries(): + wb = make_wrap_binance() + FakeBinance() + assert wb.get_market_summaries() + + +def test_exchange_get_wallet_health(): + wb = make_wrap_binance() + FakeBinance() + x = wb.get_wallet_health() + assert x[0]['Currency'] == 'ETH' diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 99a964815..7f5a53cfd 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -264,27 +264,27 @@ def test_exchange_bittrex_get_ticker_history(): def test_exchange_bittrex_get_order(): wb = make_wrap_bittrex() fb = FakeBittrex() - order = wb.get_order('someUUID') + order = wb.get_order('someUUID','somePAIR') assert order['id'] == 'ABC123' fb.success = False with pytest.raises(btx.OperationalException, match=r'lost'): - wb.get_order('someUUID') + wb.get_order('someUUID','somePAIR') def test_exchange_bittrex_cancel_order(): wb = make_wrap_bittrex() fb = FakeBittrex() - wb.cancel_order('someUUID') + wb.cancel_order('someUUID','somePAIR') with pytest.raises(btx.OperationalException, match=r'no such order'): fb.success = False - wb.cancel_order('someUUID') + wb.cancel_order('someUUID','somePAIR') # Note: this can be a bug in exchange.bittrex._validate_response with pytest.raises(KeyError): fb.result = {'success': False} # message is missing! - wb.cancel_order('someUUID') + wb.cancel_order('someUUID','somePAIR') with pytest.raises(btx.OperationalException, match=r'foo'): fb.result = {'success': False, 'message': 'foo'} - wb.cancel_order('someUUID') + wb.cancel_order('someUUID','somePAIR') def test_exchange_get_pair_detail_url(): diff --git a/requirements.txt b/requirements.txt index b65dcfc2e..ae4f8b91b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ python-bittrex==0.3.0 +python-binance==0.6.1 SQLAlchemy==1.2.2 python-telegram-bot==9.0.0 arrow==0.12.1 From 3c60b33793f66d5dad37645dd341b04c23dd38e5 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 00:04:10 +0100 Subject: [PATCH 02/17] Fix hyperopt exchange error --- freqtrade/optimize/backtesting.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 542ae19f6..6f80b23ac 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -113,15 +113,11 @@ def backtest(args) -> DataFrame: max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', True) record = args.get('record', None) + exchange_name = args.get('exchange_name', 'bittrex') records = [] trades = [] trade_count_lock: dict = {} - # Monkey patch config - from freqtrade import main - exchange_config = main._CONF['exchange'] - - exchange_name = exchange_config['name'] try: exchange_class = exchange.Exchanges[exchange_name.upper()].value except KeyError: @@ -246,7 +242,8 @@ def start(args): 'sell_profit_only': sell_profit_only, 'use_sell_signal': use_sell_signal, 'stoploss': strategy.stoploss, - 'record': args.export + 'record': args.export, + 'exchange_name': exchange_name }) logger.info( '\n==================================== BACKTESTING REPORT ====================================\n%s', # noqa From 10dbf468c4d8b532e85402104579d162349a6878 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 00:39:23 +0100 Subject: [PATCH 03/17] Fix PEP8 errors --- freqtrade/tests/exchange/test_exchange.py | 10 +++++----- freqtrade/tests/exchange/test_exchange_bittrex.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange.py b/freqtrade/tests/exchange/test_exchange.py index 605e7d9ab..d9bea3e60 100644 --- a/freqtrade/tests/exchange/test_exchange.py +++ b/freqtrade/tests/exchange/test_exchange.py @@ -227,7 +227,7 @@ def test_cancel_order_dry_run(default_conf, mocker): default_conf['dry_run'] = True mocker.patch.dict('freqtrade.exchange._CONF', default_conf) - assert cancel_order(order_id='123',pair='ABC_XYZ') is None + assert cancel_order(order_id='123', pair='ABC_XYZ') is None # Ensure that if not dry_run, we should call API @@ -237,7 +237,7 @@ def test_cancel_order(default_conf, mocker): api_mock = MagicMock() api_mock.cancel_order = MagicMock(return_value=123) mocker.patch('freqtrade.exchange._API', api_mock) - assert cancel_order(order_id='_',pair='ABC_XYZ') == 123 + assert cancel_order(order_id='_', pair='ABC_XYZ') == 123 def test_get_order(default_conf, mocker): @@ -247,9 +247,9 @@ def test_get_order(default_conf, mocker): order.myid = 123 order.pair = 'ABC_XYZ' exchange._DRY_RUN_OPEN_ORDERS['X'] = order - print(exchange.get_order('X','ABC_XYZ')) - assert exchange.get_order('X','ABC_XYZ').myid == 123 - assert exchange.get_order('X','ABC_XYZ').pair == 'ABC_XYZ' + print(exchange.get_order('X', 'ABC_XYZ')) + assert exchange.get_order('X', 'ABC_XYZ').myid == 123 + assert exchange.get_order('X', 'ABC_XYZ').pair == 'ABC_XYZ' default_conf['dry_run'] = False mocker.patch.dict('freqtrade.exchange._CONF', default_conf) diff --git a/freqtrade/tests/exchange/test_exchange_bittrex.py b/freqtrade/tests/exchange/test_exchange_bittrex.py index 7f5a53cfd..08e438e47 100644 --- a/freqtrade/tests/exchange/test_exchange_bittrex.py +++ b/freqtrade/tests/exchange/test_exchange_bittrex.py @@ -264,27 +264,27 @@ def test_exchange_bittrex_get_ticker_history(): def test_exchange_bittrex_get_order(): wb = make_wrap_bittrex() fb = FakeBittrex() - order = wb.get_order('someUUID','somePAIR') + order = wb.get_order('someUUID', 'somePAIR') assert order['id'] == 'ABC123' fb.success = False with pytest.raises(btx.OperationalException, match=r'lost'): - wb.get_order('someUUID','somePAIR') + wb.get_order('someUUID', 'somePAIR') def test_exchange_bittrex_cancel_order(): wb = make_wrap_bittrex() fb = FakeBittrex() - wb.cancel_order('someUUID','somePAIR') + wb.cancel_order('someUUID', 'somePAIR') with pytest.raises(btx.OperationalException, match=r'no such order'): fb.success = False - wb.cancel_order('someUUID','somePAIR') + wb.cancel_order('someUUID', 'somePAIR') # Note: this can be a bug in exchange.bittrex._validate_response with pytest.raises(KeyError): fb.result = {'success': False} # message is missing! - wb.cancel_order('someUUID','somePAIR') + wb.cancel_order('someUUID', 'somePAIR') with pytest.raises(btx.OperationalException, match=r'foo'): fb.result = {'success': False, 'message': 'foo'} - wb.cancel_order('someUUID','somePAIR') + wb.cancel_order('someUUID', 'somePAIR') def test_exchange_get_pair_detail_url(): From 5d2dbeb54c6940fe3859be65b2ff8434cbe930ba Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 00:52:40 +0100 Subject: [PATCH 04/17] More PEP8 fixes --- freqtrade/exchange/binance.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index c1dad53cd..16915e0ea 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -6,7 +6,10 @@ from typing import List, Dict, Optional from binance.client import Client as _Binance from binance.exceptions import BinanceAPIException -from binance.enums import * +from binance.enums import (KLINE_INTERVAL_1MINUTE, KLINE_INTERVAL_5MINUTE, + KLINE_INTERVAL_30MINUTE, KLINE_INTERVAL_1HOUR, + KLINE_INTERVAL_1DAY) + from decimal import Decimal from freqtrade import OperationalException @@ -193,9 +196,17 @@ class Binance(Exchange): def get_ticker_history(self, pair: str, tick_interval: int) -> List[Dict]: - INTERVAL_ENUM = eval('KLINE_INTERVAL_' + str(tick_interval) + 'MINUTE') - - if INTERVAL_ENUM in ['', None]: + if tick_interval == 1: + INTERVAL_ENUM = KLINE_INTERVAL_1MINUTE + elif tick_interval == 5: + INTERVAL_ENUM = KLINE_INTERVAL_5MINUTE + elif tick_interval == 30: + INTERVAL_ENUM = KLINE_INTERVAL_30MINUTE + elif tick_interval == 60: + INTERVAL_ENUM = KLINE_INTERVAL_1HOUR + elif tick_interval == 1440: + INTERVAL_ENUM = KLINE_INTERVAL_1DAY + else: raise ValueError('Cannot parse tick_interval: {}'.format(tick_interval)) symbol = self._pair_to_symbol(pair) From 5bc80d1e78051219db9d6b93c30ec4ed1576813b Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 01:27:41 +0100 Subject: [PATCH 05/17] Fix another PEP8 thing --- freqtrade/optimize/backtesting.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6f80b23ac..d4a28b85c 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -118,11 +118,7 @@ def backtest(args) -> DataFrame: trades = [] trade_count_lock: dict = {} - try: - exchange_class = exchange.Exchanges[exchange_name.upper()].value - except KeyError: - raise OperationalException('Exchange {} is not supported'.format( - exchange_name)) + exchange_class = exchange.Exchanges[exchange_name.upper()].value exchange._API = exchange_class({'key': '', 'secret': ''}) for pair, pair_data in processed.items(): From f4e38b9a4b559ef350ef176810fb2aab32aeec71 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 01:47:48 +0100 Subject: [PATCH 06/17] Change/fix datetime test of ticker history --- freqtrade/tests/exchange/test_exchange_binance.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange_binance.py b/freqtrade/tests/exchange/test_exchange_binance.py index 3c8ea909d..4193447da 100644 --- a/freqtrade/tests/exchange/test_exchange_binance.py +++ b/freqtrade/tests/exchange/test_exchange_binance.py @@ -2,6 +2,8 @@ from unittest.mock import MagicMock import pytest +import datetime +import dateutil from freqtrade.exchange.binance import Binance import freqtrade.exchange.binance as bin @@ -298,10 +300,12 @@ def test_exchange_binance_get_ticker_history_intervals(): wb = make_wrap_binance() FakeBinance() for tick_interval in [1, 5]: - assert ([{'O': 0.0, 'H': 0.0, + h = wb.get_ticker_history('BTC_ETH', tick_interval) + assert type(dateutil.parser.parse(h[0]['T'])) is datetime.datetime + del h[0]['T'] + assert [{'O': 0.0, 'H': 0.0, 'L': 0.0, 'C': 0.0, - 'V': 0.0, 'T': '1970-01-01T01:00:00', 'BV': 0.0}] == - wb.get_ticker_history('BTC_ETH', tick_interval)) + 'V': 0.0, 'BV': 0.0}] == h def test_exchange_binance_get_ticker_history(): From 8f570e46f5a65776d8994fab288228394d36b68e Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 01:52:07 +0100 Subject: [PATCH 07/17] Fix PEP8 indentation --- freqtrade/tests/exchange/test_exchange_binance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/tests/exchange/test_exchange_binance.py b/freqtrade/tests/exchange/test_exchange_binance.py index 4193447da..294f8094a 100644 --- a/freqtrade/tests/exchange/test_exchange_binance.py +++ b/freqtrade/tests/exchange/test_exchange_binance.py @@ -304,8 +304,8 @@ def test_exchange_binance_get_ticker_history_intervals(): assert type(dateutil.parser.parse(h[0]['T'])) is datetime.datetime del h[0]['T'] assert [{'O': 0.0, 'H': 0.0, - 'L': 0.0, 'C': 0.0, - 'V': 0.0, 'BV': 0.0}] == h + 'L': 0.0, 'C': 0.0, + 'V': 0.0, 'BV': 0.0}] == h def test_exchange_binance_get_ticker_history(): From f371fba04b8abd7f5237db6a2091cc308d2d6d54 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Thu, 1 Feb 2018 01:57:37 +0100 Subject: [PATCH 08/17] Tick Binance checkbox in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cac43731d..f92c6b133 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ your current trades. ### Exchange supported - [x] Bittrex -- [ ] Binance +- [x] Binance - [ ] Others ## Quick start From 15b31794cefa58c388615a688521347ba45eee6e Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Fri, 2 Feb 2018 00:20:35 +0100 Subject: [PATCH 09/17] Update docs, fix stake currency exchange config and more generic backtest tests --- docs/configuration.md | 8 ++-- docs/pre-requisite.md | 14 +++++-- freqtrade/exchange/__init__.py | 1 + freqtrade/optimize/backtesting.py | 3 +- freqtrade/tests/optimize/test_backtesting.py | 42 +++++++++++--------- 5 files changed, 41 insertions(+), 27 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5e3b15925..66db68488 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -24,7 +24,7 @@ The table below will list all configuration parameters. | `stoploss` | -0.10 | No | Value of the stoploss in percent used by the bot. More information below. If set, this parameter will override `stoploss` from your strategy file. | `unfilledtimeout` | 0 | No | How long (in minutes) the bot will wait for an unfilled order to complete, after which the order will be cancelled. | `bid_strategy.ask_last_balance` | 0.0 | Yes | Set the bidding price. More information below. -| `exchange.name` | bittrex | Yes | Name of the exchange class to use. +| `exchange.name` | bittrex | Yes | Name of the exchange class to use. Valid values are: `bittrex` or `binance` | `exchange.key` | key | No | API key to use for the exchange. Only required when you are in production mode. | `exchange.secret` | secret | No | API secret to use for the exchange. Only required when you are in production mode. | `exchange.pair_whitelist` | [] | No | List of currency to use by the bot. Can be overrided with `--dynamic-whitelist` param. @@ -94,7 +94,7 @@ creating trades. "dry_run": true, ``` -3. Remove your Bittrex API key (change them by fake api credentials) +3. Remove your exchange API key (change them by fake api credentials) ```json "exchange": { "name": "bittrex", @@ -120,7 +120,7 @@ you run it in production mode. "dry_run": false, ``` -3. Insert your Bittrex API key (change them by fake api keys) +3. Insert your exchange API key (change them by fake api keys) ```json "exchange": { "name": "bittrex", @@ -129,7 +129,7 @@ you run it in production mode. ... } ``` -If you have not your Bittrex API key yet, +If you have not your exchange API key yet, [see our tutorial](https://github.com/gcarq/freqtrade/blob/develop/docs/pre-requisite.md). diff --git a/docs/pre-requisite.md b/docs/pre-requisite.md index 931d12d38..cdcbd8bee 100644 --- a/docs/pre-requisite.md +++ b/docs/pre-requisite.md @@ -1,15 +1,23 @@ # Pre-requisite Before running your bot in production you will need to setup few -external API. In production mode, the bot required valid Bittrex API +external API. In production mode, the bot requires valid exchange API credentials and a Telegram bot (optional but recommended). ## Table of Contents -- [Setup your Bittrex account](#setup-your-bittrex-account) +- [Setup your Exchange account](#setup-your-exchange-account) - [Backtesting commands](#setup-your-telegram-bot) -## Setup your Bittrex account +## Setup your exchange account + +### Bittrex *To be completed, please feel free to complete this section.* +### Binance +- Go to: https://www.binance.com/userCenter/createApi.html +- Enter API key label: "freqtrade bot" and click "Create New Key" +- Check the "Enable Trading" checkbox +- Write down the API key and secret to put in: config.json + ## Setup your Telegram bot The only things you need is a working Telegram bot and its API token. Below we explain how to create your Telegram Bot, and how to get your diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 8d36c51e3..75081c52c 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -56,6 +56,7 @@ def init(config: dict) -> None: except KeyError: raise OperationalException('Exchange {} is not supported'.format(name)) + exchange_config['stake_currency'] = config['stake_currency'] _API = exchange_class(exchange_config) # Check if all pairs are available diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index d4a28b85c..370ed0280 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -107,13 +107,14 @@ def backtest(args) -> DataFrame: sell_profit_only: sell if profit only use_sell_signal: act on sell-signal stoploss: use stoploss + exchange_name: which exchange to use :return: DataFrame """ processed = args['processed'] max_open_trades = args.get('max_open_trades', 0) realistic = args.get('realistic', True) record = args.get('record', None) - exchange_name = args.get('exchange_name', 'bittrex') + exchange_name = args.get('exchange_name', None) records = [] trades = [] trade_count_lock: dict = {} diff --git a/freqtrade/tests/optimize/test_backtesting.py b/freqtrade/tests/optimize/test_backtesting.py index 0dd4f777a..be0285e27 100644 --- a/freqtrade/tests/optimize/test_backtesting.py +++ b/freqtrade/tests/optimize/test_backtesting.py @@ -5,7 +5,6 @@ import math from unittest.mock import MagicMock import pandas as pd from freqtrade import exchange, optimize -from freqtrade.exchange import Bittrex from freqtrade.optimize import preprocess from freqtrade.optimize.backtesting import backtest, generate_text_table, get_timeframe import freqtrade.optimize.backtesting as backtesting @@ -47,29 +46,31 @@ def test_get_timeframe(default_strategy): def test_backtest(default_strategy, default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 10, - 'realistic': True}) - assert not results.empty + for exch in exchange.Exchanges.__members__.keys(): + results = backtest({'stake_amount': default_conf['stake_amount'], + 'processed': optimize.preprocess(data), + 'max_open_trades': 10, + 'realistic': True, + 'exchange_name': exch.lower()}) + assert not results.empty def test_backtest_1min_ticker_interval(default_strategy, default_conf, mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) - exchange._API = Bittrex({'key': '', 'secret': ''}) # Run a backtesting for an exiting 5min ticker_interval data = optimize.load_data(None, ticker_interval=1, pairs=['BTC_UNITEST']) data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 1, - 'realistic': True}) - assert not results.empty + for exch in exchange.Exchanges.__members__.keys(): + results = backtest({'stake_amount': default_conf['stake_amount'], + 'processed': optimize.preprocess(data), + 'max_open_trades': 1, + 'realistic': True, + 'exchange_name': exch.lower()}) + assert not results.empty def load_data_test(what): @@ -122,7 +123,8 @@ def simple_backtest(config, contour, num_results): results = backtest({'stake_amount': config['stake_amount'], 'processed': processed, 'max_open_trades': 1, - 'realistic': True}) + 'realistic': True, + 'exchange_name': config['exchange']['name']}) # results :: assert len(results) == num_results @@ -135,11 +137,13 @@ def test_backtest2(default_conf, mocker, default_strategy): mocker.patch.dict('freqtrade.main._CONF', default_conf) data = optimize.load_data(None, ticker_interval=5, pairs=['BTC_ETH']) data = trim_dictlist(data, -200) - results = backtest({'stake_amount': default_conf['stake_amount'], - 'processed': optimize.preprocess(data), - 'max_open_trades': 10, - 'realistic': True}) - assert not results.empty + for exch in exchange.Exchanges.__members__.keys(): + results = backtest({'stake_amount': default_conf['stake_amount'], + 'processed': optimize.preprocess(data), + 'max_open_trades': 10, + 'realistic': True, + 'exchange_name': exch.lower()}) + assert not results.empty def test_processed(default_conf, mocker, default_strategy): From 9d9319bec555e1058997440c634c680a7396d87e Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Fri, 2 Feb 2018 00:34:57 +0100 Subject: [PATCH 10/17] Resolve merge conflict --- freqtrade/rpc/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 163e0a8aa..b0a023df6 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -84,7 +84,7 @@ def rpc_trade_status(): for trade in trades: order = None if trade.open_order_id: - order = exchange.get_order(trade.open_order_id) + order = exchange.get_order(trade.open_order_id, trade.pair) # calculate profit and send message to user current_rate = exchange.get_ticker(trade.pair, False)['bid'] current_profit = trade.calc_profit_percent(current_rate) @@ -340,11 +340,11 @@ def rpc_forcesell(trade_id) -> None: def _exec_forcesell(trade: Trade) -> str: # Check if there is there is an open order if trade.open_order_id: - order = exchange.get_order(trade.open_order_id) + order = exchange.get_order(trade.open_order_id, trade.pair) # Cancel open LIMIT_BUY orders and close trade if order and not order['closed'] and order['type'] == 'LIMIT_BUY': - exchange.cancel_order(trade.open_order_id) + exchange.cancel_order(trade.open_order_id, trade.pair) trade.close(order.get('rate') or trade.open_rate) # TODO: sell amount which has been bought already return From 9ca61389cba8e18744b07826fb0e4d64d71c08e7 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Fri, 2 Feb 2018 00:43:46 +0100 Subject: [PATCH 11/17] Merge conflict fix --- freqtrade/rpc/telegram.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 0cbff321d..ea170baa1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -396,43 +396,6 @@ def _version(bot: Bot, update: Update) -> None: send_msg('*Version:* `{}`'.format(__version__), bot=bot) -<<<<<<< HEAD -======= -def shorten_date(_date): - """ - Trim the date so it fits on small screens - """ - new_date = re.sub('seconds?', 'sec', _date) - new_date = re.sub('minutes?', 'min', new_date) - new_date = re.sub('hours?', 'h', new_date) - new_date = re.sub('days?', 'd', new_date) - new_date = re.sub('^an?', '1', new_date) - return new_date - - -def _exec_forcesell(trade: Trade) -> None: - # Check if there is there is an open order - if trade.open_order_id: - order = exchange.get_order(trade.open_order_id, trade.pair) - - # Cancel open LIMIT_BUY orders and close trade - if order and not order['closed'] and order['type'] == 'LIMIT_BUY': - exchange.cancel_order(trade.open_order_id, trade.pair) - trade.close(order.get('rate') or trade.open_rate) - # TODO: sell amount which has been bought already - return - - # Ignore trades with an attached LIMIT_SELL order - if order and not order['closed'] and order['type'] == 'LIMIT_SELL': - return - - # Get current rate and execute sell - current_rate = exchange.get_ticker(trade.pair, False)['bid'] - from freqtrade.main import execute_sell - execute_sell(trade, current_rate) - - ->>>>>>> Add Binance exchange support def send_msg(msg: str, bot: Bot = None, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: """ Send given markdown message From c819d73cb039289fd72dfb67a92bd2adbaf6516a Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Fri, 2 Feb 2018 01:10:01 +0100 Subject: [PATCH 12/17] Fix hyperopt exchange name --- freqtrade/optimize/hyperopt.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 8b89e1985..28dece296 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -19,10 +19,10 @@ from hyperopt.mongoexp import MongoTrials from pandas import DataFrame import freqtrade.vendor.qtpylib.indicators as qtpylib +from freqtrade import OperationalException # Monkey patch config from freqtrade import main # noqa; noqa from freqtrade import exchange, misc, optimize -from freqtrade.exchange import Bittrex from freqtrade.misc import load_config from freqtrade.optimize import backtesting from freqtrade.optimize.backtesting import backtest @@ -401,7 +401,8 @@ def optimizer(params): results = backtest({'stake_amount': OPTIMIZE_CONFIG['stake_amount'], 'processed': PROCESSED, - 'stoploss': params['stoploss']}) + 'stoploss': params['stoploss'], + 'exchange_name': _CONFIG['exchange']['name']}) result_explanation = format_results(results) total_profit = results.profit_percent.sum() @@ -445,12 +446,10 @@ def format_results(results: DataFrame): def start(args): - global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES + global TOTAL_TRIES, PROCESSED, TRIALS, _CURRENT_TRIES, _CONFIG TOTAL_TRIES = args.epochs - exchange._API = Bittrex({'key': '', 'secret': ''}) - # Initialize logger logging.basicConfig( level=args.loglevel, @@ -458,18 +457,27 @@ def start(args): ) logger.info('Using config: %s ...', args.config) - config = load_config(args.config) - pairs = config['exchange']['pair_whitelist'] + _CONFIG = load_config(args.config) + pairs = _CONFIG['exchange']['pair_whitelist'] + + exchange_name = _CONFIG['exchange']['name'] + try: + exchange_class = exchange.Exchanges[exchange_name.upper()].value + except KeyError: + raise OperationalException('Exchange {} is not supported'.format( + exchange_name)) + + exchange._API = exchange_class({'key': '', 'secret': ''}) # If -i/--ticker-interval is use we override the configuration parameter # (that will override the strategy configuration) if args.ticker_interval: - config.update({'ticker_interval': args.ticker_interval}) + _CONFIG.update({'ticker_interval': args.ticker_interval}) # init the strategy to use - config.update({'strategy': args.strategy}) + _CONFIG.update({'strategy': args.strategy}) strategy = Strategy() - strategy.init(config) + strategy.init(_CONFIG) timerange = misc.parse_timerange(args.timerange) data = optimize.load_data(args.datadir, pairs=pairs, From 67b4af5ec4ea82cb02d108b65902b7bc87cac19d Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Sun, 4 Feb 2018 13:49:42 +0100 Subject: [PATCH 13/17] Improve Binance error handling and resilience - allow for non-fatal exceptions and retrying of API calls - catch more common exceptions - add human error for config.json JSON errors - update setup files --- freqtrade/exchange/binance.py | 330 +++++++++++++++++++++++++--------- freqtrade/misc.py | 11 +- setup.py | 1 + setup.sh | 4 +- 4 files changed, 259 insertions(+), 87 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 16915e0ea..fa9a05d01 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -2,6 +2,7 @@ import logging import datetime import json import http +from time import sleep from typing import List, Dict, Optional from binance.client import Client as _Binance @@ -65,32 +66,117 @@ class Binance(Exchange): return '{0}_{1}'.format(symbol_stake_currency, symbol_currency) @staticmethod - def _handle_exception(excepter) -> None: + def _handle_exception(excepter) -> Dict: """ Validates the given Binance response/exception and raises a ContentDecodingError if a non-fatal issue happened. """ - # Could to alternate exception handling for specific exceptions/errors - # See: http://python-binance.readthedocs.io/en/latest/binance.html#module-binance.exceptions + # Python exceptions: + # http://python-binance.readthedocs.io/en/latest/binance.html#module-binance.exceptions + + handle = {} + if type(excepter) == http.client.RemoteDisconnected: logger.info( - 'Got HTTP error from Binance: %s' % excepter + 'Retrying: got disconnected from Binance: %s' % excepter ) - return True + handle['retry'] = True + handle['retry_max'] = 3 + handle['fatal'] = False + return handle if type(excepter) == json.decoder.JSONDecodeError: logger.info( - 'Got JSON error from Binance: %s' % excepter + 'Retrying: got JSON error from Binance: %s' % excepter ) - return True + handle['retry'] = True + handle['retry_max'] = 3 + handle['fatal'] = False + return handle + # API errors: + # https://github.com/binance-exchange/binance-official-api-docs/blob/master/errors.md if type(excepter) == BinanceAPIException: - logger.info( - 'Got API error from Binance: %s' % excepter - ) + if excepter.code == -1000: + logger.info( + 'Retrying: General unknown API error from Binance: %s' % excepter + ) + handle['retry'] = True + handle['retry_max'] = 3 + handle['fatal'] = False + return handle - return True + if excepter.code == -1003: + logger.error( + 'Binance API Rate limiter hit: %s' % excepter + ) + # Panic: this is bad: we don't want to get banned + # TODO: automatic request throttling respecting API rate limits? + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle + + if excepter.code == -1021: + logger.error( + "Binance reports invalid timestamp, " + + "check your machine (NTP) time synchronisation: {}".format( + excepter) + ) + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle + + if excepter.code == -1015: + logger.error( + 'Binance says we have too many orders: %s' % excepter + ) + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle + + if excepter.code == -2014: + logger.error( + "Binance reports bad api key format, " + + "you're probably trying to use the API with an empty key/secret: {}".format( + excepter) + ) + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle + + if excepter.code == -2015: + logger.error( + "Binance reports invalid api key, source IP or permission, " + + "check your API key settings in config.json and on binance.com: {}".format( + excepter) + ) + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle + + if excepter.code == -2011: + logger.error( + "Binance rejected order cancellation: %s" % excepter + ) + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle + + # All other exceptions we don't know about + logger.info( + 'Got error: %s' % excepter + ) + handle['retry'] = False + handle['retry_max'] = None + handle['fatal'] = True + return handle raise type(excepter)(excepter.args) @@ -104,18 +190,28 @@ class Binance(Exchange): symbol = self._pair_to_symbol(pair) - try: - data = _API.order_limit_buy( - symbol=symbol, - quantity="{0:.8f}".format(amount), - price="{0:.8f}".format(rate)) - except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( - message=str(e), - pair=pair, - rate=Decimal(rate), - amount=Decimal(amount))) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.order_limit_buy( + symbol=symbol, + quantity="{0:.8f}".format(amount), + price="{0:.8f}".format(rate)) + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal'] or tries == max_tries: + raise OperationalException('{message} params=({pair}, {rate}, {amount})'.format( + message=str(e), + pair=pair, + rate=Decimal(rate), + amount=Decimal(amount))) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) return data['orderId'] @@ -123,19 +219,29 @@ class Binance(Exchange): symbol = self._pair_to_symbol(pair) - try: - data = _API.order_limit_sell( - symbol=symbol, - quantity="{0:.8f}".format(amount), - price="{0:.8f}".format(rate)) - except Exception as e: - Binance._handle_exception(e) - raise OperationalException( - '{message} params=({pair}, {rate}, {amount})'.format( - message=str(e), - pair=pair, - rate=rate, - amount=amount)) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.order_limit_sell( + symbol=symbol, + quantity="{0:.8f}".format(amount), + price="{0:.8f}".format(rate)) + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException( + '{message} params=({pair}, {rate}, {amount})'.format( + message=str(e), + pair=pair, + rate=rate, + amount=amount)) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) return data['orderId'] @@ -144,10 +250,11 @@ class Binance(Exchange): try: data = _API.get_asset_balance(asset=currency) except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message} params=({currency})'.format( - message=str(e), - currency=currency)) + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message} params=({currency})'.format( + message=str(e), + currency=currency)) return float(data['free'] or 0.0) @@ -156,8 +263,9 @@ class Binance(Exchange): try: data = _API.get_account() except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message}'.format(message=str(e))) + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message}'.format(message=str(e))) balances = data['balances'] @@ -180,13 +288,24 @@ class Binance(Exchange): symbol = self._pair_to_symbol(pair) - try: - data = _API.get_ticker(symbol=symbol) - except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message} params=({pair})'.format( - message=str(e), - pair=pair)) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.get_ticker(symbol=symbol) + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message} params=({pair} {refresh})'.format( + message=str(e), + pair=pair, + refresh=refresh)) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) return { 'bid': float(data['bidPrice']), @@ -211,13 +330,24 @@ class Binance(Exchange): symbol = self._pair_to_symbol(pair) - try: - data = _API.get_klines(symbol=symbol, interval=INTERVAL_ENUM) - except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message} params=({pair})'.format( - message=str(e), - pair=pair)) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.get_klines(symbol=symbol, interval=INTERVAL_ENUM) + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message} params=({pair} {tick_interval})'.format( + message=str(e), + pair=pair, + tick_interval=tick_interval)) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) tick_data = [] @@ -239,15 +369,25 @@ class Binance(Exchange): symbol = self._pair_to_symbol(pair) - try: - data = _API.get_all_orders(symbol=symbol, orderId=order_id) - except Exception as e: - Binance._handle_exception(e) - raise OperationalException( - '{message} params=({symbol},{order_id})'.format( - message=str(e), - symbol=symbol, - order_id=order_id)) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.get_all_orders(symbol=symbol, orderId=order_id) + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException( + '{message} params=({symbol},{order_id})'.format( + message=str(e), + symbol=symbol, + order_id=order_id)) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) order = {} @@ -273,13 +413,24 @@ class Binance(Exchange): symbol = self._pair_to_symbol(pair) - try: - data = _API.cancel_order(symbol=symbol, orderId=order_id) - except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message} params=({order_id})'.format( - message=str(e), - order_id=order_id)) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.cancel_order(symbol=symbol, orderId=order_id) + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message} params=({order_id}, {pair})'.format( + message=str(e), + order_id=order_id), + pair=pair) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) return data @@ -291,8 +442,9 @@ class Binance(Exchange): try: data = _API.get_all_tickers() except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message}'.format(message=str(e))) + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message}'.format(message=str(e))) markets = [] @@ -313,8 +465,9 @@ class Binance(Exchange): try: data = _API.get_ticker() except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message}'.format(message=str(e))) + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message}'.format(message=str(e))) market_summaries = [] @@ -343,11 +496,21 @@ class Binance(Exchange): def get_trade_qty(self, pair: str) -> tuple: - try: - data = _API.get_exchange_info() - except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message}'.format(message=str(e))) + api_try = True + tries = 0 + max_tries = 1 + + while api_try and tries < max_tries: + try: + tries = tries + 1 + data = _API.get_exchange_info() + except Exception as e: + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message}'.format(message=str(e))) + api_try = h['retry'] + max_tries = h['retry_max'] + sleep(0.1) symbol = self._pair_to_symbol(pair) @@ -368,8 +531,9 @@ class Binance(Exchange): try: data = _API.get_exchange_info() except Exception as e: - Binance._handle_exception(e) - raise OperationalException('{message}'.format(message=str(e))) + h = Binance._handle_exception(e) + if h['fatal']: + raise OperationalException('{message}'.format(message=str(e))) wallet_health = [] diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 0512330d9..2e2b292a2 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -5,6 +5,7 @@ import logging import time import os import re +import sys from datetime import datetime from typing import Any, Callable, Dict, List @@ -91,8 +92,14 @@ def load_config(path: str) -> Dict: :param path: path as str :return: configuration as dictionary """ - with open(path) as file: - conf = json.load(file) + try: + with open(path) as file: + conf = json.load(file) + except json.decoder.JSONDecodeError as e: + logger.fatal('Syntax configuration error: invalid JSON format in {path}: {error}'.format( + path=path, error=e)) + sys.exit(1) + if 'internals' not in conf: conf['internals'] = {} logger.info('Validating configuration ...') diff --git a/setup.py b/setup.py index e53606dea..3066dfcc8 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ setup(name='freqtrade', tests_require=['pytest', 'pytest-mock', 'pytest-cov'], install_requires=[ 'python-bittrex', + 'python-binance', 'SQLAlchemy', 'python-telegram-bot', 'arrow', diff --git a/setup.sh b/setup.sh index 7d382190c..5d06141dd 100755 --- a/setup.sh +++ b/setup.sh @@ -98,7 +98,7 @@ function config_generator () { read -p "Fiat currency: (Default: USD) " fiat_currency echo "------------------------" - echo "Bittrex config generator" + echo "Exchange config generator" echo "------------------------" echo read -p "Exchange API key: " api_key @@ -205,4 +205,4 @@ plot help ;; esac -exit 0 \ No newline at end of file +exit 0 From 691b4fae6f12f9a7ec940d9e10b90cc4c1422174 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Sun, 4 Feb 2018 15:47:09 +0100 Subject: [PATCH 14/17] Implement partial sell order timeout handling --- freqtrade/main.py | 73 +++++++++++++++++++++++++++++------- freqtrade/persistence.py | 2 +- freqtrade/tests/conftest.py | 13 +++++++ freqtrade/tests/test_main.py | 48 +++++++++++++++++++++--- 4 files changed, 115 insertions(+), 21 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 3b365f549..b9150f1dc 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -180,21 +180,66 @@ def handle_timedout_limit_sell(trade: Trade, order: Dict) -> bool: Sell timeout - cancel order and update trade :return: True if order was fully cancelled """ - if order['remaining'] == order['amount']: - # if trade is not partially completed, just cancel the trade - exchange.cancel_order(trade.open_order_id, trade.pair) - trade.close_rate = None - trade.close_profit = None - trade.close_date = None - trade.is_open = True - trade.open_order_id = None - rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( - trade.pair.replace('_', '/'))) - logger.info('Sell order timeout for %s.', trade) - return True + logger.info('Sell order timeout for %s.', trade) - # TODO: figure out how to handle partially complete sell orders - return False + # Partial filled sell order timed out + if order['remaining'] < order['amount']: + + (min_qty, max_qty, step_qty) = exchange.get_trade_qty(trade.pair) + + # Create new trade for partial filled amount and close that new trade + new_trade_amount = order['amount'] - order['remaining'] + + if min_qty: + # Remaining amount must be exchange minimum order quantity to be able to sell it + if new_trade_amount < min_qty: + logger.info('Wont cancel partial filled sell order that timed out for {}:'.format( + trade) + + 'remaining amount {} too low for new order '.format(new_trade_amount) + + '(minimum order quantity: {})'.format(new_trade_amount, min_qty)) + return False + + exchange.cancel_order(trade.open_order_id, trade.pair) + + new_trade_stake_amount = new_trade_amount * trade.open_rate + + # but give it half fee: because we share buy order with current trade + # this trade only costs sell fee + new_trade = Trade( + pair=trade.pair, + stake_amount=new_trade_stake_amount, + amount=new_trade_amount, + fee=(trade.fee/2), + open_rate=trade.open_rate, + open_date=trade.open_date, + exchange=trade.exchange, + open_order_id=None + ) + new_trade.close(order['rate']) + + # Update stake and amount leftover of current trade to still be handled + trade.amount = order['remaining'] + trade.stake_amount = trade.amount * trade.open_rate + trade.open_order_id = None + + rpc.send_msg('*Timeout:* Partially filled sell order for {} cancelled: '.format( + trade.pair.replace('_', '/')) + + '{} amount remains'.format(trade.amount)) + + return False + + # Order is not partially filled: full amount remains + # Just remove the order and the trade remains to be handled + exchange.cancel_order(trade.open_order_id, trade.pair) + trade.close_rate = None + trade.close_profit = None + trade.close_date = None + trade.is_open = True + trade.open_order_id = None + rpc.send_msg('*Timeout:* Unfilled sell order for {} cancelled'.format( + trade.pair.replace('_', '/'))) + + return True def check_handle_timedout(timeoutvalue: int) -> None: diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 79903d66f..1b8bab2bd 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -133,7 +133,7 @@ class Trade(_DECL_BASE): self.is_open = False self.open_order_id = None logger.info( - 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', + 'Marking %s as closed since found no open orders for it.', self ) diff --git a/freqtrade/tests/conftest.py b/freqtrade/tests/conftest.py index 2b1d14268..d1117e608 100644 --- a/freqtrade/tests/conftest.py +++ b/freqtrade/tests/conftest.py @@ -184,6 +184,19 @@ def limit_buy_order_old_partial(): } +@pytest.fixture +def limit_sell_order_old_partial(): + return { + 'id': 'mocked_limit_sell_old_partial', + 'type': 'LIMIT_SELL', + 'pair': 'BTC_ETH', + 'opened': str(arrow.utcnow().shift(minutes=-601).datetime), + 'rate': 0.00001099, + 'amount': 90.99181073, + 'remaining': 67.99181073, + } + + @pytest.fixture def limit_sell_order(): return { diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 1adfa8418..134f912e7 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -535,14 +535,50 @@ def test_handle_timedout_limit_sell(mocker): 'amount': 1} assert main.handle_timedout_limit_sell(trade, order) assert cancel_order.call_count == 1 - order['amount'] = 2 - assert not main.handle_timedout_limit_sell(trade, order) - # Assert cancel_order was not called (callcount remains unchanged) - assert cancel_order.call_count == 1 -def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, - mocker): +def test_check_handle_timedout_partial_sell(default_conf, ticker, limit_sell_order_old_partial, + mocker): + mocker.patch.dict('freqtrade.main._CONF', default_conf) + cancel_order_mock = MagicMock() + get_trade_qty_mock = MagicMock(return_value=(None, None, None)) + mocker.patch('freqtrade.rpc.init', MagicMock()) + rpc_mock = mocker.patch('freqtrade.main.rpc.send_msg', MagicMock()) + mocker.patch.multiple('freqtrade.main.exchange', + validate_pairs=MagicMock(), + get_ticker=ticker, + get_order=MagicMock(return_value=limit_sell_order_old_partial), + cancel_order=cancel_order_mock, + get_trade_qty=get_trade_qty_mock) + init(default_conf, create_engine('sqlite://')) + + trade_sell = Trade( + pair='BTC_ETH', + open_rate=0.00001099, + exchange='BITTREX', + open_order_id='123456789', + amount=90.99181073, + fee=0.0, + stake_amount=1, + open_date=arrow.utcnow().shift(minutes=-601).datetime, + is_open=True + ) + + Trade.session.add(trade_sell) + + # check it does cancel sell orders over the time limit + # note this is for a partially-complete sell order + check_handle_timedout(600) + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + trades = Trade.query.filter(Trade.open_order_id.is_(trade_sell.open_order_id)).all() + assert len(trades) == 1 + assert trades[0].amount == 67.99181073 + assert trades[0].stake_amount == trade_sell.open_rate * trades[0].amount + + +def test_check_handle_timedout_partial_buy(default_conf, ticker, limit_buy_order_old_partial, + mocker): mocker.patch.dict('freqtrade.main._CONF', default_conf) cancel_order_mock = MagicMock() mocker.patch('freqtrade.rpc.init', MagicMock()) From 71636e8326131ffde0023cbc49aba12f8c791fd5 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Sun, 4 Feb 2018 23:43:30 +0100 Subject: [PATCH 15/17] Don't cancel partial filled buy orders that leave unsellable amount --- freqtrade/main.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index b9150f1dc..6a1109d2c 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -165,7 +165,20 @@ def handle_timedout_limit_buy(trade: Trade, order: Dict) -> bool: # if trade is partially complete, edit the stake details for the trade # and close the order - trade.amount = order['amount'] - order['remaining'] + + new_trade_amount = order['amount'] - order['remaining'] + (min_qty, max_qty, step_qty) = exchange.get_trade_qty(trade.pair) + + if min_qty: + # Remaining amount must be exchange minimum order quantity to be able to sell it + if new_trade_amount < min_qty: + logger.info('Wont cancel partial filled buy order that timed out for {}:'.format( + trade) + + 'remaining amount {} too low for new order '.format(new_trade_amount) + + '(minimum order quantity: {})'.format(new_trade_amount, min_qty)) + return False + + trade.amount = new_trade_amount trade.stake_amount = trade.amount * trade.open_rate trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) From e99afe7597630a92bf0db40aff3148f3843fb411 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Mon, 5 Feb 2018 22:38:33 +0100 Subject: [PATCH 16/17] Include transaction fees into balance check determination of being able to trade --- freqtrade/main.py | 11 ++++++++--- freqtrade/tests/test_main.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 6a1109d2c..a5b06928a 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -435,10 +435,15 @@ def create_trade(stake_amount: float, interval: int) -> bool: stake_amount ) whitelist = copy.deepcopy(_CONF['exchange']['pair_whitelist']) - # Check if stake_amount is fulfilled - if exchange.get_balance(_CONF['stake_currency']) < stake_amount: + + # We need minimum funds of: stake amount + 2x transaction (buy+sell) fee to create a trade + min_required_funds = stake_amount + (stake_amount * (exchange.get_fee() * 2)) + + # Check if we have enough funds to be able to trade + if exchange.get_balance(_CONF['stake_currency']) < min_required_funds: raise DependencyException( - 'stake amount is not fulfilled (currency={})'.format(_CONF['stake_currency']) + 'not enough funds to create trade (balance={}, required={})'.format( + _CONF['stake_currency'], min_required_funds) ) # Remove currently opened and latest pairs from whitelist diff --git a/freqtrade/tests/test_main.py b/freqtrade/tests/test_main.py index 134f912e7..62ccd2d5c 100644 --- a/freqtrade/tests/test_main.py +++ b/freqtrade/tests/test_main.py @@ -220,7 +220,7 @@ def test_create_trade_no_stake_amount(default_conf, ticker, mocker): get_ticker=ticker, buy=MagicMock(return_value='mocked_limit_buy'), get_balance=MagicMock(return_value=default_conf['stake_amount'] * 0.5)) - with pytest.raises(DependencyException, match=r'.*stake amount.*'): + with pytest.raises(DependencyException, match=r'.*not enough funds.*'): create_trade(default_conf['stake_amount'], int(default_conf['ticker_interval'])) From 56e6c563aa1812be9bb79f048f37a7a5b5f2d681 Mon Sep 17 00:00:00 2001 From: Ramon Bastiaans Date: Mon, 5 Feb 2018 22:49:40 +0100 Subject: [PATCH 17/17] Fix log line of available funds check --- freqtrade/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index a5b06928a..c75146e99 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -438,12 +438,13 @@ def create_trade(stake_amount: float, interval: int) -> bool: # We need minimum funds of: stake amount + 2x transaction (buy+sell) fee to create a trade min_required_funds = stake_amount + (stake_amount * (exchange.get_fee() * 2)) + fund_balance = exchange.get_balance(_CONF['stake_currency']) # Check if we have enough funds to be able to trade - if exchange.get_balance(_CONF['stake_currency']) < min_required_funds: + if fund_balance < min_required_funds: raise DependencyException( 'not enough funds to create trade (balance={}, required={})'.format( - _CONF['stake_currency'], min_required_funds) + fund_balance, min_required_funds) ) # Remove currently opened and latest pairs from whitelist