Add a decorator which can be used to declare populate_indicators() functions for informative pairs.

This commit is contained in:
Rokas Kupstys 2021-07-17 19:19:49 +03:00
parent d84ef34740
commit 1fdb656334
11 changed files with 414 additions and 16 deletions

View File

@ -679,7 +679,89 @@ In some situations it may be confusing to deal with stops relative to current ra
```
Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation.
### *@informative()*
In most common case it is possible to easily define informative pairs by using a decorator. All decorated `populate_indicators_*` methods run in isolation,
not having access to data from other informative pairs, in the end all informative dataframes are merged and passed to main `populate_indicators()` method.
When hyperopting, please follow instructions of [optimizing an indicator parameter](hyperopt.md#optimizing-an-indicator-parameter).
??? Example "Fast and easy way to define informative pairs"
Most of the time we do not need power and flexibility offered by `merge_informative_pair()`, therefore we can use a decorator to quickly define informative pairs.
``` python
from datetime import datetime
from freqtrade.persistence import Trade
from freqtrade.strategy import IStrategy, informative
class AwesomeStrategy(IStrategy):
# This method is not required.
# def informative_pairs(self): ...
# Define informative upper timeframe for each pair. Decorators can be stacked on same
# method. Available in populate_indicators as 'rsi_30m' and 'rsi_1h'.
@informative('30m')
@informative('1h')
def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
# Define BTC/STAKE informative pair. Available in populate_indicators and other methods as
# 'btc_rsi_1h'. Current stake currency should be specified as {stake} format variable
# instead of hardcoding actual stake currency. Available in populate_indicators and other
# methods as 'btc_rsi_1h'.
@informative('1h', 'BTC/{stake}')
def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
# Define BTC/ETH informative pair. You must specify quote currency if it is different from
# stake currency. Available in populate_indicators and other methods as 'eth_btc_rsi_1h'.
@informative('1h', 'ETH/BTC')
def populate_indicators_eth_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
# Define BTC/STAKE informative pair. A custom formatter may be specified for formatting
# column names. Format string supports these format variables:
# * {asset} - full name of the asset, for example 'BTC/USDT'.
# * {base} - base currency in lower case, for example 'eth'.
# * {BASE} - same as {base}, except in upper case.
# * {quote} - quote currency in lower case, for example 'usdt'.
# * {QUOTE} - same as {quote}, except in upper case.
# * {column} - name of dataframe column.
# * {timeframe} - timeframe of informative dataframe.
# A callable `fmt(**kwargs) -> str` may be specified, to implement custom formatting.
# Available in populate_indicators and other methods as 'rsi_upper'.
@informative('1h', 'BTC/{stake}', '{name}')
def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi_upper'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
# Strategy timeframe indicators for current pair.
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
# Informative pairs are available in this method.
dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h']
return dataframe
```
See docstring of `@informative()` decorator for more information.
!!! Note
Do not use `@informative` decorator if you need to use data of one informative pair when generating another informative pair. Instead, define informative pairs
manually as described [in the DataProvider section](#complete-data-provider-sample).
!!! Warning
Methods tagged with `@informative()` decorator must always have unique names! Re-using same name (for example when copy-pasting already defined informative method)
will overwrite previously defined method and not produce any errors due to limitations of Python programming language. In such cases you will find that indicators
created in earlier-defined methods are not available in the dataframe. Carefully review method names and make sure they are unique!
!!! Warning
When using a legacy hyperopt implementation informative pairs defined with a decorator will not be executed. Please update your strategy if necessary.
## Additional data (Wallets)

View File

@ -119,7 +119,7 @@ class Edge:
)
# Download informative pairs too
res = defaultdict(list)
for p, t in self.strategy.informative_pairs():
for p, t in self.strategy.gather_informative_pairs():
res[t].append(p)
for timeframe, inf_pairs in res.items():
timerange_startup = deepcopy(self._timerange)

View File

@ -160,7 +160,7 @@ class FreqtradeBot(LoggingMixin):
# Refreshing candles
self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist),
self.strategy.informative_pairs())
self.strategy.gather_informative_pairs())
strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()

View File

@ -4,5 +4,5 @@ from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timefr
from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter,
IntParameter, RealParameter)
from freqtrade.strategy.interface import IStrategy
from freqtrade.strategy.strategy_helper import (merge_informative_pair,
from freqtrade.strategy.strategy_helper import (informative, merge_informative_pair,
stoploss_from_absolute, stoploss_from_open)

View File

@ -6,7 +6,7 @@ import logging
import warnings
from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple, Union
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import arrow
from pandas import DataFrame
@ -19,6 +19,8 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.persistence import PairLocks, Trade
from freqtrade.strategy.hyper import HyperStrategyMixin
from freqtrade.strategy.strategy_helper import (InformativeData, _create_and_merge_informative_pair,
_format_pair_name)
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
@ -134,6 +136,39 @@ class IStrategy(ABC, HyperStrategyMixin):
self._last_candle_seen_per_pair: Dict[str, datetime] = {}
super().__init__(config)
# Gather informative pairs from @informative-decorated methods.
self._ft_informative: Dict[
Tuple[str, str], Tuple[InformativeData,
Callable[[Any, DataFrame, dict], DataFrame]]] = {}
for attr_name in dir(self.__class__):
cls_method = getattr(self.__class__, attr_name)
if not callable(cls_method):
continue
ft_informative = getattr(cls_method, '_ft_informative', [])
if not isinstance(ft_informative, list):
# Type check is required because mocker would return a mock object that evaluates to
# True, confusing this code.
continue
for informative_data in ft_informative:
asset = informative_data.asset
timeframe = informative_data.timeframe
if asset:
pair = _format_pair_name(self.config, asset)
if (pair, timeframe) in self._ft_informative:
raise OperationalException(f'Informative pair {pair} {timeframe} can not '
f'be defined more than once!')
self._ft_informative[(pair, timeframe)] = (informative_data, cls_method)
elif self.dp is not None:
for pair in self.dp.current_whitelist():
if (pair, timeframe) in self._ft_informative:
raise OperationalException(f'Informative pair {pair} {timeframe} can '
f'not be defined more than once!')
self._ft_informative[(pair, timeframe)] = (informative_data, cls_method)
def _format_pair(self, pair: str) -> str:
return pair.format(stake_currency=self.config['stake_currency'],
stake=self.config['stake_currency']).upper()
@abstractmethod
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
@ -377,6 +412,14 @@ class IStrategy(ABC, HyperStrategyMixin):
# END - Intended to be overridden by strategy
###
def gather_informative_pairs(self) -> ListPairsWithTimeframes:
"""
Internal method which gathers all informative pairs (user or automatically defined).
"""
informative_pairs = self.informative_pairs()
informative_pairs += list(self._ft_informative.keys())
return list(set(informative_pairs))
def get_strategy_name(self) -> str:
"""
Returns strategy class name
@ -793,6 +836,14 @@ class IStrategy(ABC, HyperStrategyMixin):
:return: a Dataframe with all mandatory indicators for the strategies
"""
logger.debug(f"Populating indicators for pair {metadata.get('pair')}.")
# call populate_indicators_Nm() which were tagged with @informative decorator.
for (pair, timeframe), (informative_data, populate_fn) in self._ft_informative.items():
if not informative_data.asset and pair != metadata['pair']:
continue
dataframe = _create_and_merge_informative_pair(
self, dataframe, metadata, informative_data, populate_fn)
if self._populate_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)

View File

@ -1,10 +1,24 @@
import pandas as pd
from typing import Any, Callable, NamedTuple, Optional, Union
import pandas as pd
from mypy_extensions import KwArg
from pandas import DataFrame
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
class InformativeData(NamedTuple):
asset: Optional[str]
timeframe: str
fmt: Union[str, Callable[[KwArg(str)], str], None]
ffill: bool
def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame:
timeframe: str, timeframe_inf: str, ffill: bool = True,
append_timeframe: bool = True,
date_column: str = 'date') -> pd.DataFrame:
"""
Correctly merge informative samples to the original dataframe, avoiding lookahead bias.
@ -24,6 +38,8 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
:param timeframe: Timeframe of the original pair sample.
:param timeframe_inf: Timeframe of the informative pair sample.
:param ffill: Forwardfill missing values - optional but usually required
:param append_timeframe: Rename columns by appending timeframe.
:param date_column: A custom date column name.
:return: Merged dataframe
:raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe
"""
@ -32,25 +48,29 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
minutes = timeframe_to_minutes(timeframe)
if minutes == minutes_inf:
# No need to forwardshift if the timeframes are identical
informative['date_merge'] = informative["date"]
informative['date_merge'] = informative[date_column]
elif minutes < minutes_inf:
# Subtract "small" timeframe so merging is not delayed by 1 small candle
# Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073
informative['date_merge'] = (
informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm')
informative[date_column] + pd.to_timedelta(minutes_inf, 'm') -
pd.to_timedelta(minutes, 'm')
)
else:
raise ValueError("Tried to merge a faster timeframe to a slower timeframe."
"This would create new rows, and can throw off your regular indicators.")
# Rename columns to be unique
informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
date_merge = 'date_merge'
if append_timeframe:
date_merge = f'date_merge_{timeframe_inf}'
informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
# Combine the 2 dataframes
# all indicators on the informative sample MUST be calculated before this point
dataframe = pd.merge(dataframe, informative, left_on='date',
right_on=f'date_merge_{timeframe_inf}', how='left')
dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1)
right_on=date_merge, how='left')
dataframe = dataframe.drop(date_merge, axis=1)
if ffill:
dataframe = dataframe.ffill()
@ -94,3 +114,117 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float:
:return: Positive stop loss value relative to current price
"""
return 1 - (stop_rate / current_rate)
def informative(timeframe: str, asset: str = '',
fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None,
ffill: bool = True) -> Callable[[Callable[[Any, DataFrame, dict], DataFrame]],
Callable[[Any, DataFrame, dict], DataFrame]]:
"""
A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to
define informative indicators.
Example usage:
@informative('1h')
def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
:param timeframe: Informative timeframe. Must always be higher than strategy timeframe.
:param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use
current pair.
:param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not
specified, defaults to {asset}_{name}_{timeframe} if asset is specified, or {name}_{timeframe}
otherwise.
* {asset}: name of informative asset, provided in lower-case, with / replaced with _. Stake
currency is not included in this string.
* {name}: user-specified dataframe column name.
* {timeframe}: informative timeframe.
:param ffill: ffill dataframe after mering informative pair.
"""
_asset = asset
_timeframe = timeframe
_fmt = fmt
_ffill = ffill
def decorator(fn: Callable[[Any, DataFrame, dict], DataFrame]):
informative_pairs = getattr(fn, '_ft_informative', [])
informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill))
setattr(fn, '_ft_informative', informative_pairs)
return fn
return decorator
def _format_pair_name(config, pair: str) -> str:
return pair.format(stake_currency=config['stake_currency'],
stake=config['stake_currency']).upper()
def _create_and_merge_informative_pair(strategy, dataframe: DataFrame,
metadata: dict, informative_data: InformativeData,
populate_indicators: Callable[[Any, DataFrame, dict],
DataFrame]):
asset = informative_data.asset or ''
timeframe = informative_data.timeframe
fmt = informative_data.fmt
ffill = informative_data.ffill
config = strategy.config
dp = strategy.dp
if asset:
# Insert stake currency if needed.
asset = _format_pair_name(config, asset)
else:
# Not specifying an asset will define informative dataframe for current pair.
asset = metadata['pair']
if '/' in asset:
base, quote = asset.split('/')
else:
# When futures are supported this may need reevaluation.
# base, quote = asset, None
raise OperationalException('Not implemented.')
# Default format. This optimizes for the common case: informative pairs using same stake
# currency. When quote currency matches stake currency, column name will omit base currency.
# This allows easily reconfiguring strategy to use different base currency. In a rare case
# where it is desired to keep quote currency in column name at all times user should specify
# fmt='{base}_{quote}_{column}_{timeframe}' format or similar.
if not fmt:
fmt = '{column}_{timeframe}' # Informatives of current pair
if asset != metadata['pair']:
if quote == config['stake_currency']:
fmt = '{base}_' + fmt # Informatives of other pair
else:
fmt = '{base}_{quote}_' + fmt # Informatives of different quote currency
inf_metadata = {'pair': asset, 'timeframe': timeframe}
inf_dataframe = dp.get_pair_dataframe(asset, timeframe)
inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata)
formatter: Any = None
if callable(fmt):
formatter = fmt # A custom user-specified formatter function.
else:
formatter = fmt.format # A default string formatter.
fmt_args = {
'BASE': base.upper(),
'QUOTE': quote.upper(),
'base': base.lower(),
'quote': quote.lower(),
'asset': asset,
'timeframe': timeframe,
}
inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args),
inplace=True)
date_column = formatter(column='date', **fmt_args)
if date_column in dataframe.columns:
raise OperationalException(f'Duplicate column name {date_column} exists in '
f'dataframe! Ensure column names are unique!')
dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe,
ffill=ffill, append_timeframe=False,
date_column=date_column)
return dataframe

View File

@ -1218,6 +1218,7 @@ def test_api_strategies(botclient):
assert_response(rc)
assert rc.json() == {'strategies': [
'HyperoptableStrategy',
'InformativeDecoratorTest',
'StrategyTestV2',
'TestStrategyLegacyV1'
]}

View File

@ -0,0 +1,75 @@
# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
from pandas import DataFrame
from freqtrade.strategy import informative, merge_informative_pair
from freqtrade.strategy.interface import IStrategy
class InformativeDecoratorTest(IStrategy):
"""
Strategy used by tests freqtrade bot.
Please do not modify this strategy, it's intended for internal use only.
Please look at the SampleStrategy in the user_data/strategy directory
or strategy repository https://github.com/freqtrade/freqtrade-strategies
for samples and inspiration.
"""
INTERFACE_VERSION = 2
stoploss = -0.10
timeframe = '5m'
startup_candle_count: int = 20
def informative_pairs(self):
return [('BTC/USDT', '5m')]
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['buy'] = 0
return dataframe
def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['sell'] = 0
return dataframe
# Decorator stacking test.
@informative('30m')
@informative('1h')
def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = 14
return dataframe
# Simple informative test.
@informative('1h', 'BTC/{stake}')
def populate_indicators_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = 14
return dataframe
# Quote currency different from stake currency test.
@informative('1h', 'ETH/BTC')
def populate_indicators_eth_btc_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = 14
return dataframe
# Formatting test.
@informative('30m', 'BTC/{stake}', '{column}_{BASE}_{QUOTE}_{base}_{quote}_{asset}_{timeframe}')
def populate_indicators_btc_1h_2(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = 14
return dataframe
# Custom formatter test
@informative('30m', 'ETH/{stake}', fmt=lambda column, **kwargs: column + '_from_callable')
def populate_indicators_eth_30m(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = 14
return dataframe
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
# Strategy timeframe indicators for current pair.
dataframe['rsi'] = 14
# Informative pairs are available in this method.
dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h']
# Mixing manual informative pairs with decorators.
informative = self.dp.get_pair_dataframe('BTC/USDT', '5m')
informative['rsi'] = 14
dataframe = merge_informative_pair(dataframe, informative, self.timeframe, '5m', ffill=True)
return dataframe

View File

@ -607,7 +607,7 @@ def test_is_informative_pairs_callback(default_conf):
strategy = StrategyResolver.load_strategy(default_conf)
# Should return empty
# Uses fallback to base implementation
assert [] == strategy.informative_pairs()
assert [] == strategy.gather_informative_pairs()
@pytest.mark.parametrize('error', [

View File

@ -4,6 +4,7 @@ import numpy as np
import pandas as pd
import pytest
from freqtrade.data.dataprovider import DataProvider
from freqtrade.strategy import merge_informative_pair, stoploss_from_open, timeframe_to_minutes
@ -132,3 +133,57 @@ def test_stoploss_from_open():
assert stoploss == 0
else:
assert isclose(stop_price, expected_stop_price, rel_tol=0.00001)
def test_informative_decorator(mocker, default_conf):
test_data_5m = generate_test_data('5m', 40)
test_data_30m = generate_test_data('30m', 40)
test_data_1h = generate_test_data('1h', 40)
data = {
('XRP/USDT', '5m'): test_data_5m,
('XRP/USDT', '30m'): test_data_30m,
('XRP/USDT', '1h'): test_data_1h,
('LTC/USDT', '5m'): test_data_5m,
('LTC/USDT', '30m'): test_data_30m,
('LTC/USDT', '1h'): test_data_1h,
('BTC/USDT', '30m'): test_data_30m,
('BTC/USDT', '5m'): test_data_5m,
('BTC/USDT', '1h'): test_data_1h,
('ETH/USDT', '1h'): test_data_1h,
('ETH/USDT', '30m'): test_data_30m,
('ETH/BTC', '1h'): test_data_1h,
}
from .strats.informative_decorator_strategy import InformativeDecoratorTest
default_conf['stake_currency'] = 'USDT'
InformativeDecoratorTest.dp = DataProvider({}, None, None)
mocker.patch.object(InformativeDecoratorTest.dp, 'current_whitelist', return_value=[
'XRP/USDT', 'LTC/USDT'
])
strategy = InformativeDecoratorTest(config=default_conf)
assert len(strategy._ft_informative) == 8
informative_pairs = [('XRP/USDT', '1h'), ('LTC/USDT', '1h'), ('XRP/USDT', '30m'),
('LTC/USDT', '30m'), ('BTC/USDT', '1h'), ('BTC/USDT', '30m'),
('BTC/USDT', '5m'), ('ETH/BTC', '1h'), ('ETH/USDT', '30m')]
for inf_pair in informative_pairs:
assert inf_pair in strategy.gather_informative_pairs()
def test_historic_ohlcv(pair, timeframe):
return data[(pair, timeframe or strategy.timeframe)].copy()
mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv',
side_effect=test_historic_ohlcv)
analyzed = strategy.advise_all_indicators(
{p: data[(p, strategy.timeframe)] for p in ('XRP/USDT', 'LTC/USDT')})
expected_columns = [
'rsi_1h', 'rsi_30m', # Stacked informative decorators
'btc_rsi_1h', # BTC 1h informative
'rsi_BTC_USDT_btc_usdt_BTC/USDT_30m', # Column formatting
'rsi_from_callable', # Custom column formatter
'eth_btc_rsi_1h', # Quote currency not matching stake currency
'rsi', 'rsi_less', # Non-informative columns
'rsi_5m', # Manual informative dataframe
]
for _, dataframe in analyzed.items():
for col in expected_columns:
assert col in dataframe.columns

View File

@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed():
directory = Path(__file__).parent / "strats"
strategies = StrategyResolver.search_all_objects(directory, enum_failed=False)
assert isinstance(strategies, list)
assert len(strategies) == 3
assert len(strategies) == 4
assert isinstance(strategies[0], dict)
@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed():
directory = Path(__file__).parent / "strats"
strategies = StrategyResolver.search_all_objects(directory, enum_failed=True)
assert isinstance(strategies, list)
assert len(strategies) == 4
assert len(strategies) == 5
# with enum_failed=True search_all_objects() shall find 2 good strategies
# and 1 which fails to load
assert len([x for x in strategies if x['class'] is not None]) == 3
assert len([x for x in strategies if x['class'] is not None]) == 4
assert len([x for x in strategies if x['class'] is None]) == 1