adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4197 from nas-/develop

Browse Source 
Added support for regex in whitelist
This commit is contained in:
Matthias
2021-01-15 07:34:49 +01:00
committed by GitHub
parent f3de0dd3eb bf5868c96d
commit baef8b4f79
13 changed files with 167 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
freqtrade/commands/data_commands.py
View File

@@ -10,6 +10,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
refresh_backtest_trades_data)
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver
from freqtrade.state import RunMode
@@ -42,15 +43,17 @@ def start_download_data(args: Dict[str, Any]) -> None:
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
logger.info(f"About to download pairs: {config['pairs']}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
pairs_not_available: List[str] = []
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
@@ -58,20 +61,20 @@ def start_download_data(args: Dict[str, Any]) -> None:
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange, pairs=config['pairs'], datadir=config['datadir'],
exchange, pairs=expanded_pairs, datadir=config['datadir'],
timerange=timerange, erase=bool(config.get('erase')),
data_format=config['dataformat_trades'])
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=config['pairs'], timeframes=config['timeframes'],
pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
)
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=config['pairs'], timeframes=config['timeframes'],
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format=config['dataformat_ohlcv'])

7
freqtrade/edge/edge_positioning.py
View File

@@ -12,6 +12,7 @@ from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.strategy.interface import SellType
@@ -80,10 +81,12 @@ class Edge:
if config.get('fee'):
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0])
self.fee = self.exchange.get_fee(symbol=expand_pairlist(
self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
def calculate(self) -> bool:
pairs = self.config['exchange']['pair_whitelist']
pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
list(self.exchange.markets))
heartbeat = self.edge_config.get('process_throttle_secs')
if (self._last_updated > 0) and (

4
freqtrade/exchange/exchange.py
View File

@@ -25,6 +25,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
CcxtModuleType = Any
@@ -335,8 +336,9 @@ class Exchange:
if not self.markets:
logger.warning('Unable to validate pairs (assuming they are correct).')
return
extended_pairs = expand_pairlist(pairs, list(self.markets), keep_invalid=True)
invalid_pairs = []
for pair in pairs:
for pair in extended_pairs:
# Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
# TODO: add a support for having coins in BTC/USDT format
if self.markets and pair not in self.markets:

14
freqtrade/plot/plotting.py
View File

@@ -13,6 +13,7 @@ from freqtrade.data.history import get_timerange, load_data
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds
from freqtrade.misc import pair_to_filename
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.strategy import IStrategy
@@ -29,16 +30,16 @@ except ImportError:
exit(1)
def init_plotscript(config, startup_candles: int = 0):
def init_plotscript(config, markets: List, startup_candles: int = 0):
"""
Initialize objects needed for plotting
:return: Dict with candle (OHLCV) data, trades and pairs
"""
if "pairs" in config:
pairs = config['pairs']
pairs = expand_pairlist(config['pairs'], markets)
else:
pairs = config['exchange']['pair_whitelist']
pairs = expand_pairlist(config['exchange']['pair_whitelist'], markets)
# Set timerange to use
timerange = TimeRange.parse_timerange(config.get('timerange'))
@@ -177,7 +178,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, "
f"{row['sell_reason']}, "
f"{row['trade_duration']} min",
axis=1)
axis=1)
trade_buys = go.Scatter(
x=trades["open_date"],
y=trades["open_rate"],
@@ -527,7 +528,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
IStrategy.dp = DataProvider(config, exchange)
plot_elements = init_plotscript(config, strategy.startup_candle_count)
plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
timerange = plot_elements['timerange']
trades = plot_elements['trades']
pair_counter = 0
@@ -562,7 +563,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
But should be somewhat proportional, and therefor useful
in helping out to find a good algorithm.
"""
plot_elements = init_plotscript(config)
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
plot_elements = init_plotscript(config, list(exchange.markets))
trades = plot_elements['trades']
# Filter trades to relevant pairs
# Remove open pairs - we don't know the profit yet so can't calculate profit for these.

13
freqtrade/plugins/pairlist/IPairList.py
View File

@@ -124,10 +124,21 @@ class IPairList(LoggingMixin, ABC):
"""
return self._pairlistmanager.verify_blacklist(pairlist, logmethod)
def verify_whitelist(self, pairlist: List[str], logmethod,
keep_invalid: bool = False) -> List[str]:
"""
Proxy method to verify_whitelist for easy access for child classes.
:param pairlist: Pairlist to validate
:param logmethod: Function that'll be called, `logger.info` or `logger.warning`
:param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes.
:return: pairlist - whitelisted pairs
"""
return self._pairlistmanager.verify_whitelist(pairlist, logmethod, keep_invalid)
def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]:
"""
Check available markets and remove pair from whitelist if necessary
:param whitelist: the sorted list of pairs the user might want to trade
:param pairlist: the sorted list of pairs the user might want to trade
:return: the list of pairs the user wants to trade without those unavailable or
black_listed
"""

7
freqtrade/plugins/pairlist/StaticPairList.py
View File

@@ -50,9 +50,12 @@ class StaticPairList(IPairList):
:return: List of pairs
"""
if self._allow_inactive:
return self._config['exchange']['pair_whitelist']
return self.verify_whitelist(
self._config['exchange']['pair_whitelist'], logger.info, keep_invalid=True
)
else:
return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist'])
return self._whitelist_for_active_markets(
self.verify_whitelist(self._config['exchange']['pair_whitelist'], logger.info))
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""

37
freqtrade/plugins/pairlist/pairlist_helpers.py
View File

@@ -2,22 +2,41 @@ import re
from typing import List
def expand_pairlist(wildcardpl: List[str], available_pairs: List[str]) -> List[str]:
def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
keep_invalid: bool = False) -> List[str]:
"""
Expand pairlist potentially containing wildcards based on available markets.
This will implicitly filter all pairs in the wildcard-list which are not in available_pairs.
:param wildcardpl: List of Pairlists, which may contain regex
:param available_pairs: List of all available pairs (`exchange.get_markets().keys()`)
:param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes
:return expanded pairlist, with Regexes from wildcardpl applied to match all available pairs.
:raises: ValueError if a wildcard is invalid (like '*/BTC' - which should be `.*/BTC`)
"""
result = []
for pair_wc in wildcardpl:
try:
comp = re.compile(pair_wc)
result += [
pair for pair in available_pairs if re.match(comp, pair)
]
except re.error as err:
raise ValueError(f"Wildcard error in {pair_wc}, {err}")
if keep_invalid:
for pair_wc in wildcardpl:
try:
comp = re.compile(pair_wc)
result_partial = [
pair for pair in available_pairs if re.match(comp, pair)
]
# Add all matching pairs.
# If there are no matching pairs (Pair not on exchange) keep it.
result += result_partial or [pair_wc]
except re.error as err:
raise ValueError(f"Wildcard error in {pair_wc}, {err}")
for element in result:
if not re.fullmatch(r'^[A-Za-z0-9/-]+$', element):
result.remove(element)
else:
for pair_wc in wildcardpl:
try:
comp = re.compile(pair_wc)
result += [
pair for pair in available_pairs if re.match(comp, pair)
]
except re.error as err:
raise ValueError(f"Wildcard error in {pair_wc}, {err}")
return result

33
freqtrade/plugins/pairlistmanager.py
View File

@@ -59,6 +59,17 @@ class PairListManager():
"""The expanded blacklist (including wildcard expansion)"""
return expand_pairlist(self._blacklist, self._exchange.get_markets().keys())
@property
def expanded_whitelist_keep_invalid(self) -> List[str]:
"""The expanded whitelist (including wildcard expansion), maintaining invalid pairs"""
return expand_pairlist(self._whitelist, self._exchange.get_markets().keys(),
keep_invalid=True)
@property
def expanded_whitelist(self) -> List[str]:
"""The expanded whitelist (including wildcard expansion), filtering invalid pairs"""
return expand_pairlist(self._whitelist, self._exchange.get_markets().keys())
@property
def name_list(self) -> List[str]:
"""Get list of loaded Pairlist Handler names"""
@@ -129,6 +140,28 @@ class PairListManager():
pairlist.remove(pair)
return pairlist
def verify_whitelist(self, pairlist: List[str], logmethod,
keep_invalid: bool = False) -> List[str]:
"""
Verify and remove items from pairlist - returning a filtered pairlist.
Logs a warning or info depending on `aswarning`.
Pairlist Handlers explicitly using this method shall use
`logmethod=logger.info` to avoid spamming with warning messages
:param pairlist: Pairlist to validate
:param logmethod: Function that'll be called, `logger.info` or `logger.warning`
:param keep_invalid: If sets to True, drops invalid pairs silently while expanding regexes.
:return: pairlist - whitelisted pairs
"""
try:
if keep_invalid:
whitelist = self.expanded_whitelist_keep_invalid
else:
whitelist = self.expanded_whitelist
except ValueError as err:
logger.error(f"Pair whitelist contains an invalid Wildcard: {err}")
return []
return whitelist
def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes:
"""
Create list of pair tuples with (pair, timeframe)