adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added suppoort for regex in whitelist

This commit is contained in:
nas- 2021-01-12 01:13:58 +01:00
parent dbc25f00ac
commit 4d7ffa8c81
9 changed files with 93 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
freqtrade/commands/data_commands.py
View File

@ -10,6 +10,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
refresh_backtest_trades_data)
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode
@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None:
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
logger.info(f"About to download pairs: {config['pairs']}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
pairs_not_available: List[str] = []
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None:
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange, pairs=config['pairs'], datadir=config['datadir'],
exchange, pairs=expanded_pairs, datadir=config['datadir'],
timerange=timerange, erase=bool(config.get('erase')),
data_format=config['dataformat_trades'])
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=config['pairs'], timeframes=config['timeframes'],
pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
)
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config['pairs'], timeframes=config['timeframes'],
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format=config['dataformat_ohlcv'])

7
freqtrade/edge/edge_positioning.py
View File

@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.strategy.interface import SellType
@ -80,10 +81,12 @@ class Edge:
if config.get('fee'):
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
self.fee = self.exchange.get_fee(symbol=expand_pairlist(
self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
def calculate(self) -> bool:
pairs = self.config['exchange']['pair_whitelist']
pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
list(self.exchange.markets))
heartbeat = self.edge_config.get('process_throttle_secs')
if (self._last_updated > 0) and (

4
freqtrade/exchange/exchange.py
View File

@ -25,6 +25,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
CcxtModuleType = Any
@ -335,8 +336,9 @@ class Exchange:
if not self.markets:
logger.warning('Unable to validate pairs (assuming they are correct).')
return
extended_pairs = expand_pairlist(pairs, list(self.markets))
invalid_pairs = []
for pair in pairs:
for pair in extended_pairs:
# Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
# TODO: add a support for having coins in BTC/USDT format
if self.markets and pair not in self.markets:

14
freqtrade/plot/plotting.py
View File

@ -13,6 +13,7 @@ from freqtrade.data.history import get_timerange, load_data
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds
from freqtrade.misc import pair_to_filename
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.strategy import IStrategy
@ -29,16 +30,16 @@ except ImportError:
exit(1)
def init_plotscript(config, startup_candles: int = 0):
def init_plotscript(config, markets: List, startup_candles: int = 0):
"""
Initialize objects needed for plotting
:return: Dict with candle (OHLCV) data, trades and pairs
"""
if "pairs" in config:
pairs = config['pairs']
pairs = expand_pairlist(config['pairs'], markets)
else:
pairs = config['exchange']['pair_whitelist']
pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets)
# Set timerange to use
timerange = TimeRange.parse_timerange(config.get('timerange'))
@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
f"{row['sell_reason']}, "
f"{row['trade_duration']} min",
axis=1)
axis=1)
trade_buys = go.Scatter(
x=trades["open_date"],
y=trades["open_rate"],
@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
IStrategy.dp = DataProvider(config, exchange)
plot_elements = init_plotscript(config, strategy.startup_candle_count)
plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
timerange = plot_elements['timerange']
trades = plot_elements['trades']
pair_counter = 0
@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
But should be somewhat proportional, and therefor useful
in helping out to find a good algorithm.
"""
plot_elements = init_plotscript(config)
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
plot_elements = init_plotscript(config, list(exchange.markets))
trades = plot_elements['trades']
# Filter trades to relevant pairs
# Remove open pairs - we don't know the profit yet so can't calculate profit for these.

9
freqtrade/plugins/pairlist/IPairList.py
View File

@ -124,6 +124,15 @@ class IPairList(LoggingMixin, ABC):
"""
return self._pairlistmanager.verify_blacklist(pairlist, logmethod)
def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
"""
Proxy method to verify_whitelist for easy access for child classes.
:param pairlist: Pairlist to validate
:param logmethod: Function that'll be called, `logger.info` or `logger.warning`.
:return: pairlist - whitelisted pairs
"""
return self._pairlistmanager.verify_whitelist(pairlist, logmethod)
def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
"""
Check available markets and remove pair from whitelist if necessary

5
freqtrade/plugins/pairlist/StaticPairList.py
View File

@ -50,9 +50,10 @@ class StaticPairList(IPairList):
:return: List of pairs
"""
if self._allow_inactive:
return self._config['exchange']['pair_whitelist']
return self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info)
else:
return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
return self._whitelist_for_active_markets(
self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info))
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""

20
freqtrade/plugins/pairlistmanager.py
View File

@ -59,6 +59,11 @@ class PairListManager():
"""The expanded blacklist (including wildcard expansion)"""
return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
@property
def expanded_whitelist(self) -> List[str]:
"""The expanded whitelist (including wildcard expansion)"""
return expand_pairlist(self._whitelist, self._exchange.get_markets().keys())
@property
def name_list(self) -> List[str]:
"""Get list of loaded Pairlist Handler names"""
@ -129,6 +134,21 @@ class PairListManager():
pairlist.remove(pair)
return pairlist
def verify_whitelist(self, pairlist: List[str], logmethod) -> List[str]:
"""
Verify and remove items from pairlist - returning a filtered pairlist.
Logs a warning or info depending on `aswarning`.
Pairlist Handlers explicitly using this method shall use
`logmethod=logger.info` to avoid spamming with warning messages
:return: pairlist - blacklisted pairs
"""
try:
whitelist = self.expanded_whitelist
except ValueError as err:
logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
return []
return whitelist
def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes:
"""
Create list of pair tuples with (pair, timeframe)

63
tests/exchange/test_exchange.py
View File

@ -505,37 +505,38 @@ def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs d
Exchange(default_conf)
def test_validate_pairs_not_available(default_conf, mocker):
api_mock = MagicMock()
type(api_mock).markets = PropertyMock(return_value={
'XRP/BTC': {'inactive': True}
})
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
with pytest.raises(OperationalException, match=r'not available'):
Exchange(default_conf)
def test_validate_pairs_exception(default_conf, mocker, caplog):
caplog.set_level(logging.INFO)
api_mock = MagicMock()
mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
type(api_mock).markets = PropertyMock(return_value={})
mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
Exchange(default_conf)
mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
Exchange(default_conf)
assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
# This cannot happen anymore as expand_pairlist implicitly filters out unavaliablie pairs
# def test_validate_pairs_not_available(default_conf, mocker):
# api_mock = MagicMock()
# type(api_mock).markets = PropertyMock(return_value={
# 'XRP/BTC': {'inactive': True, 'base': 'XRP', 'quote': 'BTC'}
# })
# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock))
# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
# mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
#
# with pytest.raises(OperationalException, match=r'not available'):
# Exchange(default_conf)
#
#
# def test_validate_pairs_exception(default_conf, mocker, caplog):
# caplog.set_level(logging.INFO)
# api_mock = MagicMock()
# mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance'))
#
# type(api_mock).markets = PropertyMock(return_value={})
# mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock)
# mocker.patch('freqtrade.exchange.Exchange.validate_timeframes')
# mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency')
# mocker.patch('freqtrade.exchange.Exchange._load_async_markets')
#
# with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'):
# Exchange(default_conf)
#
# mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value={}))
# Exchange(default_conf)
# assert log_has('Unable to validate pairs (assuming they are correct).', caplog)
def test_validate_pairs_restricted(default_conf, mocker, caplog):

5
tests/test_plotting.py
View File

@ -47,14 +47,15 @@ def test_init_plotscript(default_conf, mocker, testdatadir):
default_conf['timeframe'] = "5m"
default_conf["datadir"] = testdatadir
default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
ret = init_plotscript(default_conf)
supported_markets = ["TRX/BTC", "ADA/BTC"]
ret = init_plotscript(default_conf, supported_markets)
assert "ohlcv" in ret
assert "trades" in ret
assert "pairs" in ret
assert 'timerange' in ret
default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"]
ret = init_plotscript(default_conf, 20)
ret = init_plotscript(default_conf, supported_markets, 20)
assert "ohlcv" in ret
assert "TRX/BTC" in ret["ohlcv"]
assert "ADA/BTC" in ret["ohlcv"]