diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index f19c5a181..8727cc3fc 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`AgeFilter`](#agefilter) +* [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) * [`PriceFilter`](#pricefilter) @@ -63,17 +64,56 @@ The `refresh_period` setting allows to define the period (in seconds), at which The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists. Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data. -`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library: +`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library: * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. ```json -"pairlists": [{ +"pairlists": [ + { "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", "refresh_period": 1800 -}], + } +], +``` + +`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. + +For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days: + +```json +"pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 86400, + "lookback_days": 7 + } +], +``` + +!!! Warning "Range look back and refresh period" + When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API. + +!!! Warning "Performance implications when using lookback range" + If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation. + +More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles: + +```json +"pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 3600, + "lookback_timeframe": "1h", + "lookback_period": 72 + } +], ``` !!! Note @@ -81,13 +121,39 @@ Filtering instances (not the first position in the list) will not apply any cach #### AgeFilter -Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`). +Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). When pairs are first listed on an exchange they can suffer huge price drops and volatility in the first few days while the pair goes through its price-discovery period. Bots can often be caught out buying before the pair has finished dropping in price. -This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. +This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`. + +#### OffsetFilter + +Offsets an incoming pairlist by a given `offset` value. + +As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split +a larger pairlist on two bot instances. + +Example to remove the first 10 pairs from the pairlist: + +```json +"pairlists": [ + { + "method": "OffsetFilter", + "offset": 10 + } +], +``` + +!!! Warning + When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` + it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the + `VolumeFilter`. + +!!! Note + An offset larger then the total length of the incoming pairlist will result in an empty pairlist. #### PerformanceFilter diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 27c620c3d..27192aa2f 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -148,13 +148,18 @@ import pandas as pd stats = load_backtest_stats(backtest_dir) strategy_stats = stats['strategy'][strategy] +dates = [] +profits = [] +for date_profit in strategy_stats['daily_profit']: + dates.append(date_profit[0]) + profits.append(date_profit[1]) + equity = 0 equity_daily = [] -for dp in strategy_stats['daily_profit']: +for daily_profit in profits: equity_daily.append(equity) - equity += float(dp) + equity += float(daily_profit) -dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end']) df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily}) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index f4c32387b..acd143708 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', - 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter', 'VolatilityFilter'] + 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', + 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', + 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index eecb63d07..1459dfd78 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path, new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else - int(arrow.utcnow().shift( - days=-new_pairs_days).float_timestamp) * 1000 + arrow.utcnow().shift( + days=-new_pairs_days).int_timestamp * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -272,7 +272,7 @@ def _download_trades_history(exchange: Exchange, if timerange.stoptype == 'date': until = timerange.stopts * 1000 else: - since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000 + since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 trades = data_handler.trades_load(pair) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 235f03269..42e86db3e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -578,7 +578,7 @@ class Exchange: 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), - 'timestamp': int(arrow.utcnow().int_timestamp * 1000), + 'timestamp': arrow.utcnow().int_timestamp * 1000, 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 7c6b7cbc3..8f818047d 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -116,6 +116,7 @@ class Backtesting: # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) + self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) def __del__(self): LoggingMixin.show_output = True diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 90976d34e..439016c14 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -75,7 +75,7 @@ class HyperoptTools(): if fn: HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json')) else: - logger.warn("Strategy not found, not exporting parameter file.") + logger.warning("Strategy not found, not exporting parameter file.") @staticmethod def has_space(config: Dict[str, Any], space: str) -> bool: diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 89cf70437..eefacbbab 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -272,7 +272,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: winning_days = sum(daily_profit > 0) draw_days = sum(daily_profit == 0) losing_days = sum(daily_profit < 0) - daily_profit_list = daily_profit.tolist() + daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()] return { 'backtest_best_day': best_rel, @@ -325,8 +325,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - results['open_timestamp'] = results['open_date'].astype(int64) // 1e6 - results['close_timestamp'] = results['close_date'].astype(int64) // 1e6 + if not results.empty: + results['open_timestamp'] = results['open_date'].view(int64) // 1e6 + results['close_timestamp'] = results['close_date'].view(int64) // 1e6 backtest_days = (max_date - min_date).days strat_stats = { diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 8f623b062..dc5cab31e 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -27,6 +27,7 @@ class AgeFilter(IPairList): super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) + self._max_days_listed = pairlistconfig.get('max_days_listed', None) if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") @@ -34,6 +35,12 @@ class AgeFilter(IPairList): raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " f"({exchange.ohlcv_candle_limit('1d')})") + if self._max_days_listed and self._max_days_listed <= self._min_days_listed: + raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") + if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'): + raise OperationalException("AgeFilter requires max_days_listed to not exceed " + "exchange max request size " + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: @@ -48,8 +55,13 @@ class AgeFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - Filtering pairs with age less than " - f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") + return ( + f"{self.name} - Filtering pairs with age less than " + f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}" + ) + (( + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" + ) if self._max_days_listed else '') def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -61,9 +73,12 @@ class AgeFilter(IPairList): if not needed_pairs: return pairlist + since_days = -( + self._max_days_listed if self._max_days_listed else self._min_days_listed + ) - 1 since_ms = int(arrow.utcnow() .floor('day') - .shift(days=-self._min_days_listed - 1) + .shift(days=since_days) .float_timestamp) * 1000 candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: @@ -86,14 +101,22 @@ class AgeFilter(IPairList): return True if daily_candles is not None: - if len(daily_candles) >= self._min_days_listed: + if ( + len(daily_candles) >= self._min_days_listed + and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed) + ): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 + self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 return True else: - self.log_once(f"Removed {pair} from whitelist, because age " - f"{len(daily_candles)} is less than {self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}", logger.info) + self.log_once(( + f"Removed {pair} from whitelist, because age " + f"{len(daily_candles)} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}" + ) + (( + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" + ) if self._max_days_listed else ''), logger.info) return False return False diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py new file mode 100644 index 000000000..573a573a6 --- /dev/null +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -0,0 +1,54 @@ +""" +Offset pair list filter +""" +import logging +from typing import Any, Dict, List + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class OffsetFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._offset = pairlistconfig.get('offset', 0) + + if self._offset < 0: + raise OperationalException("OffsetFilter requires offset to be >= 0") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Offseting pairs by {self._offset}." + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + if self._offset > len(pairlist): + self.log_once(f"Offset of {self._offset} is larger than " + + f"pair count of {len(pairlist)}", logger.warning) + pairs = pairlist[self._offset:] + self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info) + return pairs diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 5ae8e3e9f..9383e5d06 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -69,10 +69,10 @@ class VolatilityFilter(IPairList): """ needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .float_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 8eff137b0..d6b8aaaa3 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -6,9 +6,12 @@ Provides dynamic pair list based on trade volumes import logging from typing import Any, Dict, List +import arrow from cachetools.ttl import TTLCache from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList @@ -36,6 +39,35 @@ class VolumePairList(IPairList): self._min_value = self._pairlistconfig.get('min_value', 0) self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._lookback_days = self._pairlistconfig.get('lookback_days', 0) + self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d') + self._lookback_period = self._pairlistconfig.get('lookback_period', 0) + + if (self._lookback_days > 0) & (self._lookback_period > 0): + raise OperationalException( + 'Ambigous configuration: lookback_days and lookback_period both set in pairlist ' + 'config. Please set lookback_days only or lookback_period and lookback_timeframe ' + 'and restart the bot.' + ) + + # overwrite lookback timeframe and days when lookback_days is set + if self._lookback_days > 0: + self._lookback_timeframe = '1d' + self._lookback_period = self._lookback_days + + # get timeframe in minutes and seconds + self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) + self._tf_in_sec = self._tf_in_min * 60 + + # wether to use range lookback or not + self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) + + if self._use_range & (self._refresh_period < self._tf_in_sec): + raise OperationalException( + f'Refresh period of {self._refresh_period} seconds is smaller than one ' + f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period ' + f'to at least {self._tf_in_sec} and restart the bot.' + ) if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -47,6 +79,13 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + if self._lookback_period < 0: + raise OperationalException("VolumeFilter requires lookback_period to be >= 0") + if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe): + raise OperationalException("VolumeFilter requires lookback_period to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})") + @property def needstickers(self) -> bool: """ @@ -78,7 +117,6 @@ class VolumePairList(IPairList): # Item found - no refresh necessary return pairlist else: - # Use fresh pairlist # Check if pair quote currency equals to the stake currency. filtered_tickers = [ @@ -103,6 +141,60 @@ class VolumePairList(IPairList): # Use the incoming pairlist. filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + # get lookback period in ms, for exchange ohlcv fetch + if self._use_range: + since_ms = int(arrow.utcnow() + .floor('minute') + .shift(minutes=-(self._lookback_period * self._tf_in_min) + - self._tf_in_min) + .int_timestamp) * 1000 + + to_ms = int(arrow.utcnow() + .floor('minute') + .shift(minutes=-self._tf_in_min) + .int_timestamp) * 1000 + + # todo: utc date output for starting date + self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: " + f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " + f"till {format_ms_time(to_ms)}", logger.info) + needed_pairs = [ + (p, self._lookback_timeframe) for p in + [ + s['symbol'] for s in filtered_tickers + ] if p not in self._pair_cache + ] + + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv( + needed_pairs, since_ms=since_ms, cache=False + ) + for i, p in enumerate(filtered_tickers): + pair_candles = candles[ + (p['symbol'], self._lookback_timeframe) + ] if (p['symbol'], self._lookback_timeframe) in candles else None + # in case of candle data calculate typical price and quoteVolume for candle + if pair_candles is not None and not pair_candles.empty: + pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + + pair_candles['close']) / 3 + pair_candles['quoteVolume'] = ( + pair_candles['volume'] * pair_candles['typical_price'] + ) + + # ensure that a rolling sum over the lookback_period is built + # if pair_candles contains more candles than lookback_period + quoteVolume = (pair_candles['quoteVolume'] + .rolling(self._lookback_period) + .sum() + .iloc[-1]) + + # replace quoteVolume with range quoteVolume sum calculated above + filtered_tickers[i]['quoteVolume'] = quoteVolume + else: + filtered_tickers[i]['quoteVolume'] = 0 + if self._min_value > 0: filtered_tickers = [ v for v in filtered_tickers if v[self._sort_key] > self._min_value] diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 8be61166b..a6d1820de 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -62,10 +62,10 @@ class RangeStabilityFilter(IPairList): """ needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .float_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py index a8c737e04..76c8ed8f2 100644 --- a/freqtrade/rpc/api_server/web_ui.py +++ b/freqtrade/rpc/api_server/web_ui.py @@ -18,6 +18,17 @@ async def fallback(): return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html')) +@router_ui.get('/ui_version', include_in_schema=False) +async def ui_version(): + from freqtrade.commands.deploy_commands import read_ui_version + uibase = Path(__file__).parent / 'ui/installed/' + version = read_ui_version(uibase) + + return { + "version": version if version else "not_installed", + } + + @router_ui.get('/{rest_of_path:path}', include_in_schema=False) async def index_html(rest_of_path: str): """ diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 538e95f40..e0aaefe50 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -761,7 +761,7 @@ class RPC: sell_signals = 0 if has_content: - dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 + dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000 # Move open to seperate column when signal for easy plotting if 'buy' in dataframe.columns: buy_mask = (dataframe['buy'] == 1) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 171a53ca1..319a6c9c0 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -600,8 +600,8 @@ class Telegram(RPCHandler): ) total_dust_balance = 0 total_dust_currencies = 0 - curr_output = '' for curr in result['currencies']: + curr_output = '' if curr['est_stake'] > balance_dust_level: curr_output = ( f"*{curr['currency']}:*\n" diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index f3b0d8d03..99720ae6e 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -215,13 +215,18 @@ "stats = load_backtest_stats(backtest_dir)\n", "strategy_stats = stats['strategy'][strategy]\n", "\n", + "dates = []\n", + "profits = []\n", + "for date_profit in strategy_stats['daily_profit']:\n", + " dates.append(date_profit[0])\n", + " profits.append(date_profit[1])\n", + "\n", "equity = 0\n", "equity_daily = []\n", - "for dp in strategy_stats['daily_profit']:\n", + "for daily_profit in profits:\n", " equity_daily.append(equity)\n", - " equity += float(dp)\n", + " equity += float(daily_profit)\n", "\n", - "dates = pd.date_range(strategy_stats['backtest_start'], strategy_stats['backtest_end'])\n", "\n", "df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n", "\n", diff --git a/requirements.txt b/requirements.txt index b289c13ea..528dc2ce6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.0 -pandas==1.2.5 +pandas==1.3.0 ccxt==1.52.40 # Pin cryptography for now due to rust build errors with piwheels diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index ae8f6e958..b15126a33 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -79,7 +79,8 @@ def whitelist_conf_agefilter(default_conf): }, { "method": "AgeFilter", - "min_days_listed": 2 + "min_days_listed": 2, + "max_days_listed": 100 } ] return default_conf @@ -302,7 +303,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 2}, + {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": None}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, @@ -310,12 +311,24 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "ETH", []), # AgeFilter and VolumePairList (require 2 days only, all should pass age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 2}], + {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": 100}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), # AgeFilter and VolumePairList (require 10 days, all should fail age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 10}], + {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": None}], "BTC", []), + # AgeFilter and VolumePairList (all pair listed > 2, all should fail age test) + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}], + "BTC", []), + # AgeFilter and VolumePairList LTC/BTC has 6 candles - removes all + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 5}], + "BTC", []), + # AgeFilter and VolumePairList LTC/BTC has 6 candles - passes + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 10}], + "BTC", ["LTC/BTC"]), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], @@ -417,7 +430,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}, {"method": "VolatilityFilter", "lookback_days": 3, "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}], - "BTC", ['ETH/BTC', 'TKN/BTC']) + "BTC", ['ETH/BTC', 'TKN/BTC']), + # VolumePairList with no offset = unchanged pairlist + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 0}], + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with offset = 2 + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 2}], + "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with higher offset, than total pairlist + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 100}], + "USDT", []) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, pairlists, base_currency, @@ -431,7 +456,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ohlcv_data = { ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history.append(ohlcv_history), ('XRP/BTC', '1d'): ohlcv_history, ('HOT/BTC', '1d'): ohlcv_history_high_vola, } @@ -480,9 +505,13 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t for pairlist in pairlists: if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \ - len(ohlcv_history) <= pairlist['min_days_listed']: + len(ohlcv_history) < pairlist['min_days_listed']: assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' r'.* day.*', caplog) + if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \ + len(ohlcv_history) > pairlist['max_days_listed']: + assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' + r'.* day.* or more than .* day', caplog) if pairlist['method'] == 'PrecisionFilter' and whitelist_result: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) @@ -507,6 +536,105 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog) +@pytest.mark.parametrize("pairlists,base_currency,volumefilter_result", [ + # default refresh of 1800 to small for daily candle lookback + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_days": 1}], + "BTC", "default_refresh_too_short"), # OperationalException expected + # ambigous configuration with lookback days and period + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_days": 1, "lookback_period": 1}], + "BTC", "lookback_days_and_period"), # OperationalException expected + # negative lookback period + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": -1}], + "BTC", "lookback_period_negative"), # OperationalException expected + # lookback range exceedes exchange limit + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}], + "BTC", 'lookback_exceeds_exchange_request_size'), # OperationalException expected + # expecing pairs as given + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}], + "BTC", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']), + # expecting pairs from default tickers, because 1h candles are not available + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), +]) +def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, + pairlists, base_currency, volumefilter_result, caplog) -> None: + whitelist_conf['pairlists'] = pairlists + whitelist_conf['stake_currency'] = base_currency + + ohlcv_history_high_vola = ohlcv_history.copy() + ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 + + # create candles for medium overall volume with last candle high volume + ohlcv_history_medium_volume = ohlcv_history.copy() + ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5 + + # create candles for high volume with all candles high volume + ohlcv_history_high_volume = ohlcv_history.copy() + ohlcv_history_high_volume.loc[:, 'volume'] = 10 + + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history_medium_volume, + ('XRP/BTC', '1d'): ohlcv_history_high_vola, + ('HOT/BTC', '1d'): ohlcv_history_high_volume, + } + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + if volumefilter_result == 'default_refresh_too_short': + with pytest.raises(OperationalException, + match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe ' + r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ ' + r'and restart the bot\.'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + return + elif volumefilter_result == 'lookback_days_and_period': + with pytest.raises(OperationalException, + match=r'Ambigous configuration: lookback_days and lookback_period both ' + r'set in pairlist config\..*'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + elif volumefilter_result == 'lookback_period_negative': + with pytest.raises(OperationalException, + match=r'VolumeFilter requires lookback_period to be >= 0'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + elif volumefilter_result == 'lookback_exceeds_exchange_request_size': + with pytest.raises(OperationalException, + match=r'VolumeFilter requires lookback_period to not exceed ' + r'exchange max request size \([0-9]+\)'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + else: + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=shitcoinmarkets) + ) + + # remove ohlcv when looback_timeframe != 1d + # to enforce fallback to ticker data + if 'lookback_timeframe' in pairlists[0]: + if pairlists[0]['lookback_timeframe'] != '1d': + ohlcv_data = [] + + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist + + assert isinstance(whitelist, list) + assert whitelist == volumefilter_result + + def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] del whitelist_conf['stoploss'] @@ -650,6 +778,22 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) +def test_agefilter_max_days_lower_than_min_days(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'AgeFilter', 'min_days_listed': 3, + "max_days_listed": 2}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'AgeFilter max_days_listed <= min_days_listed not permitted'): + get_patched_freqtradebot(mocker, default_conf) + + def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': 99999}] @@ -695,6 +839,18 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 +def test_OffsetFilter_error(mocker, whitelist_conf) -> None: + whitelist_conf['pairlists'] = ( + [{"method": "StaticPairList"}, {"method": "OffsetFilter", "offset": -1}] + ) + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + with pytest.raises(OperationalException, + match=r'OffsetFilter requires offset to be >= 0'): + PairListManager(MagicMock, whitelist_conf) + + def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'RangeStabilityFilter', 'lookback_days': 99999}] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index b8dd112c9..89da68da7 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -105,6 +105,15 @@ def test_api_ui_fallback(botclient): assert rc.status_code == 200 +def test_api_ui_version(botclient, mocker): + ftbot, client = botclient + + mocker.patch('freqtrade.commands.deploy_commands.read_ui_version', return_value='0.1.2') + rc = client_get(client, "/ui_version") + assert rc.status_code == 200 + assert rc.json()['version'] == '0.1.2' + + def test_api_auth(): with pytest.raises(ValueError): create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234', token_type="NotATokenType")