add producer names

This commit is contained in:
Timothy Pogue 2022-08-31 11:43:02 -06:00
parent 510cf4f305
commit 865b34cd6f
6 changed files with 46 additions and 36 deletions

View File

@ -498,6 +498,7 @@ CONF_SCHEMA = {
'items': { 'items': {
'type': 'object', 'type': 'object',
'properties': { 'properties': {
'name': {'type': 'string'},
'url': {'type': 'string', 'default': ''}, 'url': {'type': 'string', 'default': ''},
'ws_token': {'type': 'string', 'default': ''}, 'ws_token': {'type': 'string', 'default': ''},
} }

View File

@ -34,8 +34,8 @@ class DataProvider:
self, self,
config: dict, config: dict,
exchange: Optional[Exchange], exchange: Optional[Exchange],
rpc: Optional[RPCManager] = None, pairlists=None,
pairlists=None rpc: Optional[RPCManager] = None
) -> None: ) -> None:
self._config = config self._config = config
self._exchange = exchange self._exchange = exchange
@ -44,8 +44,9 @@ class DataProvider:
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None self.__slice_index: Optional[int] = None
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {} self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
self.__external_pairs_df: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} self.__producer_pairs_df: Dict[str,
self.__producer_pairs: List[str] = [] Dict[PairWithTimeframe, Tuple[DataFrame, datetime]]] = {}
self.__producer_pairs: Dict[str, List[str]] = {}
self._msg_queue: deque = deque() self._msg_queue: deque = deque()
self.__msg_cache = PeriodicCache( self.__msg_cache = PeriodicCache(
@ -84,22 +85,22 @@ class DataProvider:
dataframe, datetime.now(timezone.utc)) dataframe, datetime.now(timezone.utc))
# For multiple producers we will want to merge the pairlists instead of overwriting # For multiple producers we will want to merge the pairlists instead of overwriting
def set_producer_pairs(self, pairlist: List[str]): def set_producer_pairs(self, pairlist: List[str], producer_name: str = "default"):
""" """
Set the pairs received to later be used. Set the pairs received to later be used.
This only supports 1 Producer right now. This only supports 1 Producer right now.
:param pairlist: List of pairs :param pairlist: List of pairs
""" """
self.__producer_pairs = pairlist.copy() self.__producer_pairs[producer_name] = pairlist.copy()
def get_producer_pairs(self) -> List[str]: def get_producer_pairs(self, producer_name: str = "default") -> List[str]:
""" """
Get the pairs cached from the producer Get the pairs cached from the producer
:returns: List of pairs :returns: List of pairs
""" """
return self.__producer_pairs return self.__producer_pairs.get(producer_name, [])
def emit_df( def emit_df(
self, self,
@ -129,6 +130,7 @@ class DataProvider:
timeframe: str, timeframe: str,
dataframe: DataFrame, dataframe: DataFrame,
candle_type: CandleType, candle_type: CandleType,
producer_name: str = "default"
) -> None: ) -> None:
""" """
Add the pair data to this class from an external source. Add the pair data to this class from an external source.
@ -139,15 +141,19 @@ class DataProvider:
""" """
pair_key = (pair, timeframe, candle_type) pair_key = (pair, timeframe, candle_type)
if producer_name not in self.__producer_pairs_df:
self.__producer_pairs_df[producer_name] = {}
# For multiple leaders, if the data already exists, we'd merge # For multiple leaders, if the data already exists, we'd merge
self.__external_pairs_df[pair_key] = (dataframe, datetime.now(timezone.utc)) self.__producer_pairs_df[producer_name][pair_key] = (dataframe, datetime.now(timezone.utc))
def get_external_df( def get_external_df(
self, self,
pair: str, pair: str,
timeframe: str, timeframe: str,
candle_type: CandleType candle_type: CandleType,
) -> DataFrame: producer_name: str = "default"
) -> Tuple[DataFrame, datetime]:
""" """
Get the pair data from the external sources. Will wait if the policy is Get the pair data from the external sources. Will wait if the policy is
set to, and data is not available. set to, and data is not available.
@ -158,11 +164,15 @@ class DataProvider:
""" """
pair_key = (pair, timeframe, candle_type) pair_key = (pair, timeframe, candle_type)
if pair_key not in self.__external_pairs_df: if producer_name not in self.__producer_pairs_df:
# We don't have this data yet, return empty DataFrame and datetime (01-01-1970) # We don't have this data yet, return empty DataFrame and datetime (01-01-1970)
return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
return self.__external_pairs_df[pair_key] if pair_key not in self.__producer_pairs_df:
# We don't have this data yet, return empty DataFrame and datetime (01-01-1970)
return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
return self.__producer_pairs_df[producer_name][pair_key]
def add_pairlisthandler(self, pairlists) -> None: def add_pairlisthandler(self, pairlists) -> None:
""" """

View File

@ -30,7 +30,7 @@ from freqtrade.plugins.pairlistmanager import PairListManager
from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.plugins.protectionmanager import ProtectionManager
from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager from freqtrade.rpc import RPCManager
from freqtrade.rpc.emc import ExternalMessageConsumer from freqtrade.rpc.external_message_consumer import ExternalMessageConsumer
from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.interface import IStrategy
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.util import FtPrecise from freqtrade.util import FtPrecise
@ -85,7 +85,7 @@ class FreqtradeBot(LoggingMixin):
# Keep this at the end of this initialization method. # Keep this at the end of this initialization method.
self.rpc: RPCManager = RPCManager(self) self.rpc: RPCManager = RPCManager(self)
self.dataprovider = DataProvider(self.config, self.exchange, self.rpc, self.pairlists) self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc)
# Attach Dataprovider to strategy instance # Attach Dataprovider to strategy instance
self.strategy.dp = self.dataprovider self.strategy.dp = self.dataprovider
@ -202,8 +202,9 @@ class FreqtradeBot(LoggingMixin):
# This just means we won't broadcast dataframes if we're listening to a producer # This just means we won't broadcast dataframes if we're listening to a producer
# Doesn't necessarily NEED to be this way, as maybe we'd like to broadcast # Doesn't necessarily NEED to be this way, as maybe we'd like to broadcast
# even if we are using external dataframes in the future. # even if we are using external dataframes in the future.
self.strategy.analyze(self.active_pair_whitelist, self.strategy.analyze(self.active_pair_whitelist,
external_data=self.dataprovider.external_data_enabled) emit_df=self.dataprovider.external_data_enabled)
with self._exit_lock: with self._exit_lock:
# Check for exchange cancelations, timeouts and user requested replace # Check for exchange cancelations, timeouts and user requested replace

View File

@ -162,7 +162,7 @@ class ExternalMessageConsumer:
async with lock: async with lock:
# Handle the message # Handle the message
self.handle_producer_message(message) self.handle_producer_message(producer, message)
except (asyncio.TimeoutError, websockets.exceptions.ConnectionClosed): except (asyncio.TimeoutError, websockets.exceptions.ConnectionClosed):
# We haven't received data yet. Check the connection and continue. # We haven't received data yet. Check the connection and continue.
@ -210,10 +210,11 @@ class ExternalMessageConsumer:
# How we do things here isn't set in stone. There seems to be some interest # How we do things here isn't set in stone. There seems to be some interest
# in figuring out a better way, but we shall do this for now. # in figuring out a better way, but we shall do this for now.
def handle_producer_message(self, message: Dict[str, Any]): def handle_producer_message(self, producer: Dict[str, Any], message: Dict[str, Any]):
""" """
Handles external messages from a Producer Handles external messages from a Producer
""" """
producer_name = producer.get('name', 'default')
# Should we have a default message type? # Should we have a default message type?
message_type = message.get('type', RPCMessageType.STATUS) message_type = message.get('type', RPCMessageType.STATUS)
message_data = message.get('data') message_data = message.get('data')
@ -229,7 +230,7 @@ class ExternalMessageConsumer:
pairlist = message_data pairlist = message_data
# Add the pairlist data to the DataProvider # Add the pairlist data to the DataProvider
self._dp.set_producer_pairs(pairlist) self._dp.set_producer_pairs(pairlist, producer_name=producer_name)
# Handle analyzed dataframes # Handle analyzed dataframes
elif message_type == RPCMessageType.ANALYZED_DF: elif message_type == RPCMessageType.ANALYZED_DF:
@ -246,4 +247,5 @@ class ExternalMessageConsumer:
dataframe = remove_entry_exit_signals(dataframe) dataframe = remove_entry_exit_signals(dataframe)
# Add the dataframe to the dataprovider # Add the dataframe to the dataprovider
self._dp.add_external_df(pair, timeframe, dataframe, candle_type) self._dp.add_external_df(pair, timeframe, dataframe,
candle_type, producer_name=producer_name)

View File

@ -682,8 +682,7 @@ class IStrategy(ABC, HyperStrategyMixin):
def analyze_ticker( def analyze_ticker(
self, self,
dataframe: DataFrame, dataframe: DataFrame,
metadata: dict, metadata: dict
populate_indicators: bool = True
) -> DataFrame: ) -> DataFrame:
""" """
Parses the given candle (OHLCV) data and returns a populated DataFrame Parses the given candle (OHLCV) data and returns a populated DataFrame
@ -693,7 +692,6 @@ class IStrategy(ABC, HyperStrategyMixin):
:return: DataFrame of candle (OHLCV) data with indicator data and signals added :return: DataFrame of candle (OHLCV) data with indicator data and signals added
""" """
logger.debug("TA Analysis Launched") logger.debug("TA Analysis Launched")
if populate_indicators:
dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_indicators(dataframe, metadata)
dataframe = self.advise_entry(dataframe, metadata) dataframe = self.advise_entry(dataframe, metadata)
dataframe = self.advise_exit(dataframe, metadata) dataframe = self.advise_exit(dataframe, metadata)
@ -703,7 +701,7 @@ class IStrategy(ABC, HyperStrategyMixin):
self, self,
dataframe: DataFrame, dataframe: DataFrame,
metadata: dict, metadata: dict,
external_data: bool = False emit_df: bool = False
) -> DataFrame: ) -> DataFrame:
""" """
Parses the given candle (OHLCV) data and returns a populated DataFrame Parses the given candle (OHLCV) data and returns a populated DataFrame
@ -720,16 +718,15 @@ class IStrategy(ABC, HyperStrategyMixin):
if (not self.process_only_new_candles or if (not self.process_only_new_candles or
self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']):
populate_indicators = not external_data
# Defs that only make change on new candle data. # Defs that only make change on new candle data.
dataframe = self.analyze_ticker(dataframe, metadata, populate_indicators) dataframe = self.analyze_ticker(dataframe, metadata)
self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date']
candle_type = self.config.get('candle_type_def', CandleType.SPOT) candle_type = self.config.get('candle_type_def', CandleType.SPOT)
self.dp._set_cached_df(pair, self.timeframe, dataframe, candle_type=candle_type) self.dp._set_cached_df(pair, self.timeframe, dataframe, candle_type=candle_type)
if populate_indicators: if emit_df:
self.dp.emit_df((pair, self.timeframe, candle_type), dataframe) self.dp.emit_df((pair, self.timeframe, candle_type), dataframe)
else: else:
@ -743,7 +740,7 @@ class IStrategy(ABC, HyperStrategyMixin):
def analyze_pair( def analyze_pair(
self, self,
pair: str, pair: str,
external_data: bool = False emit_df: bool = False
) -> None: ) -> None:
""" """
Fetch data for this pair from dataprovider and analyze. Fetch data for this pair from dataprovider and analyze.
@ -764,7 +761,7 @@ class IStrategy(ABC, HyperStrategyMixin):
dataframe = strategy_safe_wrapper( dataframe = strategy_safe_wrapper(
self._analyze_ticker_internal, message="" self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair}, external_data) )(dataframe, {'pair': pair}, emit_df)
self.assert_df(dataframe, df_len, df_close, df_date) self.assert_df(dataframe, df_len, df_close, df_date)
except StrategyError as error: except StrategyError as error:
@ -778,14 +775,14 @@ class IStrategy(ABC, HyperStrategyMixin):
def analyze( def analyze(
self, self,
pairs: List[str], pairs: List[str],
external_data: bool = False emit_df: bool = False
) -> None: ) -> None:
""" """
Analyze all pairs using analyze_pair(). Analyze all pairs using analyze_pair().
:param pairs: List of pairs to analyze :param pairs: List of pairs to analyze
""" """
for pair in pairs: for pair in pairs:
self.analyze_pair(pair, external_data) self.analyze_pair(pair, emit_df)
@ staticmethod @ staticmethod
def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]: def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]:

View File

@ -2138,7 +2138,6 @@ def test_send_msg_strategy_msg_notification(default_conf, mocker) -> None:
def test_send_msg_unknown_type(default_conf, mocker) -> None: def test_send_msg_unknown_type(default_conf, mocker) -> None:
telegram, _, _ = get_telegram_testobject(mocker, default_conf) telegram, _, _ = get_telegram_testobject(mocker, default_conf)
with pytest.raises(NotImplementedError, match=r'Unknown message type: None'):
telegram.send_msg({ telegram.send_msg({
'type': None, 'type': None,
}) })