From f3824d970bb511d1a230fe6de713f735b81c80c7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 May 2020 20:18:38 +0200 Subject: [PATCH 1/4] Use dict for symbol_is_pair --- freqtrade/commands/list_commands.py | 2 +- freqtrade/exchange/exchange.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index e5131f9b2..bc4bd694f 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -163,7 +163,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], 'Base': v['base'], 'Quote': v['quote'], 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v['symbol'])} + **({'Is pair': symbol_is_pair(v)} if not pairs_only else {})}) if (args.get('print_one_column', False) or diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index af745e8d0..09f700bbb 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -214,7 +214,7 @@ class Exchange: if quote_currencies: markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies} if pairs_only: - markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])} + markets = {k: v for k, v in markets.items() if symbol_is_pair(v)} if active_only: markets = {k: v for k, v in markets.items() if market_is_active(v)} return markets @@ -1210,7 +1210,7 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) -def symbol_is_pair(market_symbol: str, base_currency: str = None, +def symbol_is_pair(market_symbol: Dict[str, Any], base_currency: str = None, quote_currency: str = None) -> bool: """ Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the @@ -1218,10 +1218,12 @@ def symbol_is_pair(market_symbol: str, base_currency: str = None, it also checks that the symbol contains appropriate base and/or quote currency part before and after the separating character correspondingly. """ - symbol_parts = market_symbol.split('/') + symbol_parts = market_symbol['symbol'].split('/') return (len(symbol_parts) == 2 and - (symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and - (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0)) + (market_symbol.get('base') == base_currency + if base_currency else len(symbol_parts[0]) > 0) and + (market_symbol.get('quote') == quote_currency + if quote_currency else len(symbol_parts[1]) > 0)) def market_is_active(market: Dict) -> bool: From b22e3a67d86b27d06ab2c1b4bee0d8f0b2f0f382 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Jun 2020 20:29:48 +0200 Subject: [PATCH 2/4] rename symbol_is_pair to market_is_tradable Make it part of the exchange class, so subclasses can override this --- freqtrade/commands/list_commands.py | 4 ++-- freqtrade/exchange/__init__.py | 3 +-- freqtrade/exchange/exchange.py | 31 +++++++++++++---------------- freqtrade/exchange/ftx.py | 12 ++++++++++- freqtrade/exchange/kraken.py | 12 ++++++++++- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index bc4bd694f..503f8a4ee 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -14,7 +14,7 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) + market_is_active) from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode @@ -163,7 +163,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], 'Base': v['base'], 'Quote': v['quote'], 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v)} + **({'Is pair': exchange.market_is_tradable(v)} if not pairs_only else {})}) if (args.get('print_one_column', False) or diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a39f8f5df..bdf1f91ec 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -12,8 +12,7 @@ from freqtrade.exchange.exchange import (timeframe_to_seconds, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date) -from freqtrade.exchange.exchange import (market_is_active, - symbol_is_pair) +from freqtrade.exchange.exchange import (market_is_active) from freqtrade.exchange.kraken import Kraken from freqtrade.exchange.binance import Binance from freqtrade.exchange.bibox import Bibox diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 09f700bbb..bec8b9686 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -214,7 +214,7 @@ class Exchange: if quote_currencies: markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies} if pairs_only: - markets = {k: v for k, v in markets.items() if symbol_is_pair(v)} + markets = {k: v for k, v in markets.items() if self.symbol_is_pair(v)} if active_only: markets = {k: v for k, v in markets.items() if market_is_active(v)} return markets @@ -238,6 +238,19 @@ class Exchange: """ return self.markets.get(pair, {}).get('base', '') + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + By default, checks if it's splittable by `/` and both sides correspond to base / quote + """ + symbol_parts = market['symbol'].split('/') + return (len(symbol_parts) == 2 and + len(symbol_parts[0]) > 0 and + len(symbol_parts[1]) > 0 and + symbol_parts[0] == market.get('base') and + symbol_parts[1] == market.get('quote') + ) + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] @@ -1210,22 +1223,6 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) -def symbol_is_pair(market_symbol: Dict[str, Any], base_currency: str = None, - quote_currency: str = None) -> bool: - """ - Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the - quote currency separated by '/' character. If base_currency and/or quote_currency is passed, - it also checks that the symbol contains appropriate base and/or quote currency part before - and after the separating character correspondingly. - """ - symbol_parts = market_symbol['symbol'].split('/') - return (len(symbol_parts) == 2 and - (market_symbol.get('base') == base_currency - if base_currency else len(symbol_parts[0]) > 0) and - (market_symbol.get('quote') == quote_currency - if quote_currency else len(symbol_parts[1]) > 0)) - - def market_is_active(market: Dict) -> bool: """ Return True if the market is active. diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 75915122b..cad11bbfa 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -1,6 +1,6 @@ """ FTX exchange subclass """ import logging -from typing import Dict +from typing import Any, Dict from freqtrade.exchange import Exchange @@ -12,3 +12,13 @@ class Ftx(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1500, } + + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + Default checks + check if pair is darkpool pair. + """ + parent_check = super().market_is_tradable(market) + + return (parent_check and + market.get('spot', False) is True) diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 932d82a27..af75ef9b2 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -1,6 +1,6 @@ """ Kraken exchange subclass """ import logging -from typing import Dict +from typing import Any, Dict import ccxt @@ -21,6 +21,16 @@ class Kraken(Exchange): "trades_pagination_arg": "since", } + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + Default checks + check if pair is darkpool pair. + """ + parent_check = super().market_is_tradable(market) + + return (parent_check and + market.get('darkpool', False) is False) + @retrier def get_balances(self) -> dict: if self._config['dry_run']: From b74a3addc65514aa8f7494ca995caa4bee74c4b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Jun 2020 20:30:31 +0200 Subject: [PATCH 3/4] Update tests --- freqtrade/exchange/ftx.py | 2 +- tests/exchange/test_exchange.py | 55 +++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index cad11bbfa..e5f083fb6 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -16,7 +16,7 @@ class Ftx(Exchange): def market_is_tradable(self, market: Dict[str, Any]) -> bool: """ Check if the market symbol is tradable by Freqtrade. - Default checks + check if pair is darkpool pair. + Default checks + check if pair is spot pair (no futures trading yet). """ parent_check = super().market_is_tradable(market) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e40f691a8..b87acc27c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -15,7 +15,7 @@ from freqtrade.exceptions import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken from freqtrade.exchange.common import API_RETRY_COUNT -from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair, +from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, @@ -2117,25 +2117,42 @@ def test_timeframe_to_next_date(): assert timeframe_to_next_date("5m") > date -@pytest.mark.parametrize("market_symbol,base_currency,quote_currency,expected_result", [ - ("BTC/USDT", None, None, True), - ("USDT/BTC", None, None, True), - ("BTCUSDT", None, None, False), - ("BTC/USDT", None, "USDT", True), - ("USDT/BTC", None, "USDT", False), - ("BTCUSDT", None, "USDT", False), - ("BTC/USDT", "BTC", None, True), - ("USDT/BTC", "BTC", None, False), - ("BTCUSDT", "BTC", None, False), - ("BTC/USDT", "BTC", "USDT", True), - ("BTC/USDT", "USDT", "BTC", False), - ("BTC/USDT", "BTC", "USD", False), - ("BTCUSDT", "BTC", "USDT", False), - ("BTC/", None, None, False), - ("/USDT", None, None, False), +@pytest.mark.parametrize("market_symbol,base,quote,exchange,add_dict,expected_result", [ + ("BTC/USDT", 'BTC', 'USDT', "binance", {}, True), + ("USDT/BTC", 'USDT', 'BTC', "binance", {}, True), + ("USDT/BTC", 'BTC', 'USDT', "binance", {}, False), # Reversed currencies + ("BTCUSDT", 'BTC', 'USDT', "binance", {}, False), # No seperating / + ("BTCUSDT", None, "USDT", "binance", {}, False), # + ("USDT/BTC", "BTC", None, "binance", {}, False), + ("BTCUSDT", "BTC", None, "binance", {}, False), + ("BTC/USDT", "BTC", "USDT", "binance", {}, True), + ("BTC/USDT", "USDT", "BTC", "binance", {}, False), # reversed currencies + ("BTC/USDT", "BTC", "USD", "binance", {}, False), # Wrong quote currency + ("BTC/", "BTC", 'UNK', "binance", {}, False), + ("/USDT", 'UNK', 'USDT', "binance", {}, False), + ("BTC/EUR", 'BTC', 'EUR', "kraken", {"darkpool": False}, True), + ("EUR/BTC", 'EUR', 'BTC', "kraken", {"darkpool": False}, True), + ("EUR/BTC", 'BTC', 'EUR', "kraken", {"darkpool": False}, False), # Reversed currencies + ("BTC/EUR", 'BTC', 'USD', "kraken", {"darkpool": False}, False), # wrong quote currency + ("BTC/EUR", 'BTC', 'EUR', "kraken", {"darkpool": True}, False), # no darkpools + ("BTC/EUR.d", 'BTC', 'EUR', "kraken", {"darkpool": True}, False), # no darkpools + ("BTC/USD", 'BTC', 'USD', "ftx", {'spot': True}, True), + ("USD/BTC", 'USD', 'BTC', "ftx", {'spot': True}, True), + ("BTC/USD", 'BTC', 'USDT', "ftx", {'spot': True}, False), # Wrong quote currency + ("BTC/USD", 'USD', 'BTC', "ftx", {'spot': True}, False), # Reversed currencies + ("BTC/USD", 'BTC', 'USD', "ftx", {'spot': False}, False), # Can only trade spot markets + ("BTC-PERP", 'BTC', 'USD', "ftx", {'spot': False}, False), # Can only trade spot markets ]) -def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_result) -> None: - assert symbol_is_pair(market_symbol, base_currency, quote_currency) == expected_result +def test_market_is_tradable(mocker, default_conf, market_symbol, base, + quote, add_dict, exchange, expected_result) -> None: + ex = get_patched_exchange(mocker, default_conf, id=exchange) + market = { + 'symbol': market_symbol, + 'base': base, + 'quote': quote, + **(add_dict), + } + assert ex.market_is_tradable(market) == expected_result @pytest.mark.parametrize("market,expected_result", [ From 08049d23b48242c5c468102b4ae8cb182a9236cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Jun 2020 20:41:29 +0200 Subject: [PATCH 4/4] Use "market_is_tradable" for whitelist validation --- freqtrade/exchange/exchange.py | 2 +- freqtrade/pairlist/IPairList.py | 5 +++++ tests/pairlist/test_pairlist.py | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index bec8b9686..a2bb8627a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -214,7 +214,7 @@ class Exchange: if quote_currencies: markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies} if pairs_only: - markets = {k: v for k, v in markets.items() if self.symbol_is_pair(v)} + markets = {k: v for k, v in markets.items() if self.market_is_tradable(v)} if active_only: markets = {k: v for k, v in markets.items() if market_is_active(v)} return markets diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index f48a7dcfd..61fdc73ad 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -159,6 +159,11 @@ class IPairList(ABC): f"{self._exchange.name}. Removing it from whitelist..") continue + if not self._exchange.market_is_tradable(markets[pair]): + logger.warning(f"Pair {pair} is not tradable with Freqtrade." + "Removing it from whitelist..") + continue + if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']: logger.warning(f"Pair {pair} is not compatible with your stake currency " f"{self._config['stake_currency']}. Removing it from whitelist..") diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 421f06911..c6d1eeb38 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -400,7 +400,9 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): # BCH/BTC not available (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BTT/BTC is inactive - (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") + (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active"), + # XLTCUSDT is not a valid pair + (['ETH/BTC', 'TKN/BTC', 'XLTCUSDT'], "is not tradable with Freqtrade"), ]) def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, log_message, tickers):