adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add a decorator which can be used to declare populate_indicators() functions for informative pairs.

Browse Source
This commit is contained in:
Rokas Kupstys
2021-07-17 19:19:49 +03:00
parent d84ef34740
commit 1fdb656334
11 changed files with 414 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/edge/edge_positioning.py
View File

@@ -119,7 +119,7 @@ class Edge:
)
# Download informative pairs too
res = defaultdict(list)
for p, t in self.strategy.informative_pairs():
for p, t in self.strategy.gather_informative_pairs():
res[t].append(p)
for timeframe, inf_pairs in res.items():
timerange_startup = deepcopy(self._timerange)

2
freqtrade/freqtradebot.py
View File

@@ -160,7 +160,7 @@ class FreqtradeBot(LoggingMixin):
# Refreshing candles
self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist),
self.strategy.informative_pairs())
self.strategy.gather_informative_pairs())
strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()

2
freqtrade/strategy/__init__.py
View File

@@ -4,5 +4,5 @@ from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timefr
from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter,
IntParameter, RealParameter)
from freqtrade.strategy.interface import IStrategy
from freqtrade.strategy.strategy_helper import (merge_informative_pair,
from freqtrade.strategy.strategy_helper import (informative, merge_informative_pair,
stoploss_from_absolute, stoploss_from_open)

53
freqtrade/strategy/interface.py
View File

@@ -6,7 +6,7 @@ import logging
import warnings
from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple, Union
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import arrow
from pandas import DataFrame
@@ -19,6 +19,8 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.persistence import PairLocks, Trade
from freqtrade.strategy.hyper import HyperStrategyMixin
from freqtrade.strategy.strategy_helper import (InformativeData, _create_and_merge_informative_pair,
_format_pair_name)
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
@@ -134,6 +136,39 @@ class IStrategy(ABC, HyperStrategyMixin):
self._last_candle_seen_per_pair: Dict[str, datetime] = {}
super().__init__(config)
# Gather informative pairs from @informative-decorated methods.
self._ft_informative: Dict[
Tuple[str, str], Tuple[InformativeData,
Callable[[Any, DataFrame, dict], DataFrame]]] = {}
for attr_name in dir(self.__class__):
cls_method = getattr(self.__class__, attr_name)
if not callable(cls_method):
continue
ft_informative = getattr(cls_method, '_ft_informative', [])
if not isinstance(ft_informative, list):
# Type check is required because mocker would return a mock object that evaluates to
# True, confusing this code.
continue
for informative_data in ft_informative:
asset = informative_data.asset
timeframe = informative_data.timeframe
if asset:
pair = _format_pair_name(self.config, asset)
if (pair, timeframe) in self._ft_informative:
raise OperationalException(f'Informative pair {pair} {timeframe} can not '
f'be defined more than once!')
self._ft_informative[(pair, timeframe)] = (informative_data, cls_method)
elif self.dp is not None:
for pair in self.dp.current_whitelist():
if (pair, timeframe) in self._ft_informative:
raise OperationalException(f'Informative pair {pair} {timeframe} can '
f'not be defined more than once!')
self._ft_informative[(pair, timeframe)] = (informative_data, cls_method)
def _format_pair(self, pair: str) -> str:
return pair.format(stake_currency=self.config['stake_currency'],
stake=self.config['stake_currency']).upper()
@abstractmethod
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
@@ -377,6 +412,14 @@ class IStrategy(ABC, HyperStrategyMixin):
# END - Intended to be overridden by strategy
###
def gather_informative_pairs(self) -> ListPairsWithTimeframes:
"""
Internal method which gathers all informative pairs (user or automatically defined).
"""
informative_pairs = self.informative_pairs()
informative_pairs += list(self._ft_informative.keys())
return list(set(informative_pairs))
def get_strategy_name(self) -> str:
"""
Returns strategy class name
@@ -793,6 +836,14 @@ class IStrategy(ABC, HyperStrategyMixin):
:return: a Dataframe with all mandatory indicators for the strategies
"""
logger.debug(f"Populating indicators for pair {metadata.get('pair')}.")
# call populate_indicators_Nm() which were tagged with @informative decorator.
for (pair, timeframe), (informative_data, populate_fn) in self._ft_informative.items():
if not informative_data.asset and pair != metadata['pair']:
continue
dataframe = _create_and_merge_informative_pair(
self, dataframe, metadata, informative_data, populate_fn)
if self._populate_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)

148
freqtrade/strategy/strategy_helper.py
View File

@@ -1,10 +1,24 @@
import pandas as pd
from typing import Any, Callable, NamedTuple, Optional, Union
import pandas as pd
from mypy_extensions import KwArg
from pandas import DataFrame
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
class InformativeData(NamedTuple):
asset: Optional[str]
timeframe: str
fmt: Union[str, Callable[[KwArg(str)], str], None]
ffill: bool
def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame:
timeframe: str, timeframe_inf: str, ffill: bool = True,
append_timeframe: bool = True,
date_column: str = 'date') -> pd.DataFrame:
"""
Correctly merge informative samples to the original dataframe, avoiding lookahead bias.
@@ -24,6 +38,8 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
:param timeframe: Timeframe of the original pair sample.
:param timeframe_inf: Timeframe of the informative pair sample.
:param ffill: Forwardfill missing values - optional but usually required
:param append_timeframe: Rename columns by appending timeframe.
:param date_column: A custom date column name.
:return: Merged dataframe
:raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe
"""
@@ -32,25 +48,29 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
minutes = timeframe_to_minutes(timeframe)
if minutes == minutes_inf:
# No need to forwardshift if the timeframes are identical
informative['date_merge'] = informative["date"]
informative['date_merge'] = informative[date_column]
elif minutes < minutes_inf:
# Subtract "small" timeframe so merging is not delayed by 1 small candle
# Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073
informative['date_merge'] = (
informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm')
informative[date_column] + pd.to_timedelta(minutes_inf, 'm') -
pd.to_timedelta(minutes, 'm')
)
else:
raise ValueError("Tried to merge a faster timeframe to a slower timeframe."
"This would create new rows, and can throw off your regular indicators.")
# Rename columns to be unique
informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
date_merge = 'date_merge'
if append_timeframe:
date_merge = f'date_merge_{timeframe_inf}'
informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
# Combine the 2 dataframes
# all indicators on the informative sample MUST be calculated before this point
dataframe = pd.merge(dataframe, informative, left_on='date',
right_on=f'date_merge_{timeframe_inf}', how='left')
dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1)
right_on=date_merge, how='left')
dataframe = dataframe.drop(date_merge, axis=1)
if ffill:
dataframe = dataframe.ffill()
@@ -94,3 +114,117 @@ def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float:
:return: Positive stop loss value relative to current price
"""
return 1 - (stop_rate / current_rate)
def informative(timeframe: str, asset: str = '',
fmt: Optional[Union[str, Callable[[KwArg(str)], str]]] = None,
ffill: bool = True) -> Callable[[Callable[[Any, DataFrame, dict], DataFrame]],
Callable[[Any, DataFrame, dict], DataFrame]]:
"""
A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to
define informative indicators.
Example usage:
@informative('1h')
def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
:param timeframe: Informative timeframe. Must always be higher than strategy timeframe.
:param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use
current pair.
:param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not
specified, defaults to {asset}_{name}_{timeframe} if asset is specified, or {name}_{timeframe}
otherwise.
* {asset}: name of informative asset, provided in lower-case, with / replaced with _. Stake
currency is not included in this string.
* {name}: user-specified dataframe column name.
* {timeframe}: informative timeframe.
:param ffill: ffill dataframe after mering informative pair.
"""
_asset = asset
_timeframe = timeframe
_fmt = fmt
_ffill = ffill
def decorator(fn: Callable[[Any, DataFrame, dict], DataFrame]):
informative_pairs = getattr(fn, '_ft_informative', [])
informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill))
setattr(fn, '_ft_informative', informative_pairs)
return fn
return decorator
def _format_pair_name(config, pair: str) -> str:
return pair.format(stake_currency=config['stake_currency'],
stake=config['stake_currency']).upper()
def _create_and_merge_informative_pair(strategy, dataframe: DataFrame,
metadata: dict, informative_data: InformativeData,
populate_indicators: Callable[[Any, DataFrame, dict],
DataFrame]):
asset = informative_data.asset or ''
timeframe = informative_data.timeframe
fmt = informative_data.fmt
ffill = informative_data.ffill
config = strategy.config
dp = strategy.dp
if asset:
# Insert stake currency if needed.
asset = _format_pair_name(config, asset)
else:
# Not specifying an asset will define informative dataframe for current pair.
asset = metadata['pair']
if '/' in asset:
base, quote = asset.split('/')
else:
# When futures are supported this may need reevaluation.
# base, quote = asset, None
raise OperationalException('Not implemented.')
# Default format. This optimizes for the common case: informative pairs using same stake
# currency. When quote currency matches stake currency, column name will omit base currency.
# This allows easily reconfiguring strategy to use different base currency. In a rare case
# where it is desired to keep quote currency in column name at all times user should specify
# fmt='{base}_{quote}_{column}_{timeframe}' format or similar.
if not fmt:
fmt = '{column}_{timeframe}' # Informatives of current pair
if asset != metadata['pair']:
if quote == config['stake_currency']:
fmt = '{base}_' + fmt # Informatives of other pair
else:
fmt = '{base}_{quote}_' + fmt # Informatives of different quote currency
inf_metadata = {'pair': asset, 'timeframe': timeframe}
inf_dataframe = dp.get_pair_dataframe(asset, timeframe)
inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata)
formatter: Any = None
if callable(fmt):
formatter = fmt # A custom user-specified formatter function.
else:
formatter = fmt.format # A default string formatter.
fmt_args = {
'BASE': base.upper(),
'QUOTE': quote.upper(),
'base': base.lower(),
'quote': quote.lower(),
'asset': asset,
'timeframe': timeframe,
}
inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args),
inplace=True)
date_column = formatter(column='date', **fmt_args)
if date_column in dataframe.columns:
raise OperationalException(f'Duplicate column name {date_column} exists in '
f'dataframe! Ensure column names are unique!')
dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe,
ffill=ffill, append_timeframe=False,
date_column=date_column)
return dataframe