DataFrame transmission, strategy follower logic

This commit is contained in:
Timothy Pogue 2022-08-21 22:45:36 -06:00
parent 739b68f8fd
commit 6f5478cc02
13 changed files with 332 additions and 79 deletions

View File

@ -7,6 +7,7 @@ Common Interface for bot and strategy to access data.
import logging import logging
from collections import deque from collections import deque
from datetime import datetime, timezone from datetime import datetime, timezone
from threading import Event
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame from pandas import DataFrame
@ -28,13 +29,16 @@ MAX_DATAFRAME_CANDLES = 1000
class DataProvider: class DataProvider:
def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None: def __init__(self, config: dict, exchange: Optional[Exchange],
pairlists=None, replicate_controller=None) -> None:
self._config = config self._config = config
self._exchange = exchange self._exchange = exchange
self._pairlists = pairlists self._pairlists = pairlists
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None self.__slice_index: Optional[int] = None
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {} self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
self.__external_pairs_df: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__external_pairs_event: Dict[str, Event] = {}
self._msg_queue: deque = deque() self._msg_queue: deque = deque()
self.__msg_cache = PeriodicCache( self.__msg_cache = PeriodicCache(
@ -63,9 +67,58 @@ class DataProvider:
:param dataframe: analyzed dataframe :param dataframe: analyzed dataframe
:param candle_type: Any of the enum CandleType (must match trading mode!) :param candle_type: Any of the enum CandleType (must match trading mode!)
""" """
self.__cached_pairs[(pair, timeframe, candle_type)] = ( pair_key = (pair, timeframe, candle_type)
self.__cached_pairs[pair_key] = (
dataframe, datetime.now(timezone.utc)) dataframe, datetime.now(timezone.utc))
def add_external_df(
self,
pair: str,
timeframe: str,
dataframe: DataFrame,
candle_type: CandleType
) -> None:
"""
Add the DataFrame to the __external_pairs_df. If a pair event exists,
set it to release the main thread from waiting.
"""
pair_key = (pair, timeframe, candle_type)
# Delete stale data
if pair_key in self.__external_pairs_df:
del self.__external_pairs_df[pair_key]
self.__external_pairs_df[pair_key] = (dataframe, datetime.now(timezone.utc))
pair_event = self.__external_pairs_event.get(pair)
if pair_event:
logger.debug(f"Leader data for pair {pair_key} has been added")
pair_event.set()
def get_external_df(
self,
pair: str,
timeframe: str,
candle_type: CandleType
) -> DataFrame:
"""
If the pair exists in __external_pairs_df, return it. If it doesn't,
create a new threading Event in __external_pairs_event and wait on it.
"""
pair_key = (pair, timeframe, candle_type)
if pair_key not in self.__external_pairs_df:
pair_event = Event()
self.__external_pairs_event[pair] = pair_event
logger.debug(f"Waiting on Leader data for: {pair_key}")
self.__external_pairs_event[pair].wait()
if pair_key in self.__external_pairs_df:
return self.__external_pairs_df[pair_key]
# Because of the waiting mechanism, this should never return
return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
def add_pairlisthandler(self, pairlists) -> None: def add_pairlisthandler(self, pairlists) -> None:
""" """
Allow adding pairlisthandler after initialization Allow adding pairlisthandler after initialization

View File

@ -8,3 +8,4 @@ class ReplicateModeType(str, Enum):
class LeaderMessageType(str, Enum): class LeaderMessageType(str, Enum):
pairlist = "pairlist" pairlist = "pairlist"
analyzed_df = "analyzed_df"

View File

@ -23,7 +23,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
InvalidOrderException, PricingError) InvalidOrderException, PricingError)
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.misc import safe_value_fallback, safe_value_fallback2 from freqtrade.misc import dataframe_to_json, safe_value_fallback, safe_value_fallback2
from freqtrade.mixins import LoggingMixin from freqtrade.mixins import LoggingMixin
from freqtrade.persistence import Order, PairLocks, Trade, init_db from freqtrade.persistence import Order, PairLocks, Trade, init_db
from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.pairlistmanager import PairListManager
@ -77,6 +77,8 @@ class FreqtradeBot(LoggingMixin):
self.replicate_controller = None self.replicate_controller = None
self.pairlists = PairListManager(self.exchange, self.config)
# RPC runs in separate threads, can start handling external commands just after # RPC runs in separate threads, can start handling external commands just after
# initialization, even before Freqtradebot has a chance to start its throttling, # initialization, even before Freqtradebot has a chance to start its throttling,
# so anything in the Freqtradebot instance should be ready (initialized), including # so anything in the Freqtradebot instance should be ready (initialized), including
@ -84,8 +86,6 @@ class FreqtradeBot(LoggingMixin):
# Keep this at the end of this initialization method. # Keep this at the end of this initialization method.
self.rpc: RPCManager = RPCManager(self) self.rpc: RPCManager = RPCManager(self)
self.pairlists = PairListManager(self.exchange, self.config)
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)
# Attach Dataprovider to strategy instance # Attach Dataprovider to strategy instance
@ -93,6 +93,9 @@ class FreqtradeBot(LoggingMixin):
# Attach Wallets to strategy instance # Attach Wallets to strategy instance
self.strategy.wallets = self.wallets self.strategy.wallets = self.wallets
# Attach ReplicateController to the strategy
# self.strategy.replicate_controller = self.replicate_controller
# Initializing Edge only if enabled # Initializing Edge only if enabled
self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.edge = Edge(self.config, self.exchange, self.strategy) if \
self.config.get('edge', {}).get('enabled', False) else None self.config.get('edge', {}).get('enabled', False) else None
@ -194,6 +197,27 @@ class FreqtradeBot(LoggingMixin):
strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
if self.replicate_controller:
if not self.replicate_controller.is_leader():
# Run Follower mode analyzing
leader_pairs = self.pairlists._whitelist
self.strategy.analyze_external(self.active_pair_whitelist, leader_pairs)
else:
# We are leader, make sure to pass callback func to emit data
def emit_on_finish(pair, dataframe, timeframe, candle_type):
logger.debug(f"Emitting dataframe for {pair}")
return self.rpc.emit_data(
{
"data_type": LeaderMessageType.analyzed_df,
"data": {
"key": (pair, timeframe, candle_type),
"value": dataframe_to_json(dataframe)
}
}
)
self.strategy.analyze(self.active_pair_whitelist, finish_callback=emit_on_finish)
else:
self.strategy.analyze(self.active_pair_whitelist) self.strategy.analyze(self.active_pair_whitelist)
with self._exit_lock: with self._exit_lock:
@ -264,14 +288,13 @@ class FreqtradeBot(LoggingMixin):
# Or should this class be made available to the PairListManager and ran # Or should this class be made available to the PairListManager and ran
# when filter_pairlist is called? # when filter_pairlist is called?
if self.replicate_controller: if self.replicate_controller:
if self.replicate_controller.is_leader(): if self.replicate_controller.is_leader():
self.replicate_controller.send_message( self.rpc.emit_data({
{
"data_type": LeaderMessageType.pairlist, "data_type": LeaderMessageType.pairlist,
"data": _whitelist "data": _whitelist
} })
)
# Calculating Edge positioning # Calculating Edge positioning
if self.edge: if self.edge:

View File

@ -10,6 +10,7 @@ from typing import Any, Iterator, List
from typing.io import IO from typing.io import IO
from urllib.parse import urlparse from urllib.parse import urlparse
import pandas
import rapidjson import rapidjson
from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@ -249,3 +250,24 @@ def parse_db_uri_for_logging(uri: str):
return uri return uri
pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
"""
Serialize a DataFrame for transmission over the wire using JSON
:param dataframe: A pandas DataFrame
:returns: A JSON string of the pandas DataFrame
"""
return dataframe.to_json(orient='records')
def json_to_dataframe(data: str) -> pandas.DataFrame:
"""
Deserialize JSON into a DataFrame
:param data: A JSON string
:returns: A pandas DataFrame from the JSON string
"""
dataframe = pandas.read_json(data)
dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
return dataframe

View File

@ -81,11 +81,10 @@ class ExternalPairList(IPairList):
seen = set(self._leader_pairs) seen = set(self._leader_pairs)
for pair in pairlist: for pair in pairlist:
if pair in seen: if pair in seen:
logger.debug(f"Encountered already existing pair {pair}")
continue continue
self._leader_pairs.append(pair) self._leader_pairs.append(pair)
if not self._has_data.is_set(): if not self._has_data.is_set() and len(self._leader_pairs) > 0:
self._has_data.set() self._has_data.set()
def gen_pairlist(self, tickers: Dict) -> List[str]: def gen_pairlist(self, tickers: Dict) -> List[str]:

View File

@ -5,6 +5,7 @@ import asyncio
import logging import logging
import secrets import secrets
import socket import socket
import traceback
from threading import Event, Thread from threading import Event, Thread
from typing import Any, Coroutine, Dict, Union from typing import Any, Coroutine, Dict, Union
@ -17,6 +18,7 @@ from freqtrade.enums import LeaderMessageType, ReplicateModeType, RPCMessageType
from freqtrade.rpc import RPC, RPCHandler from freqtrade.rpc import RPC, RPCHandler
from freqtrade.rpc.replicate.channel import ChannelManager from freqtrade.rpc.replicate.channel import ChannelManager
from freqtrade.rpc.replicate.thread_queue import Queue as ThreadedQueue from freqtrade.rpc.replicate.thread_queue import Queue as ThreadedQueue
from freqtrade.rpc.replicate.types import MessageType
from freqtrade.rpc.replicate.utils import is_websocket_alive from freqtrade.rpc.replicate.utils import is_websocket_alive
@ -79,11 +81,11 @@ class ReplicateController(RPCHandler):
self.mode = ReplicateModeType[self.replicate_config.get('mode', 'leader').lower()] self.mode = ReplicateModeType[self.replicate_config.get('mode', 'leader').lower()]
self.leaders_list = self.replicate_config.get('leaders', []) self.leaders_list = self.replicate_config.get('leaders', [])
self.push_throttle_secs = self.replicate_config.get('push_throttle_secs', 1) self.push_throttle_secs = self.replicate_config.get('push_throttle_secs', 0.1)
self.reply_timeout = self.replicate_config.get('follower_reply_timeout', 10) self.reply_timeout = self.replicate_config.get('follower_reply_timeout', 10)
self.ping_timeout = self.replicate_config.get('follower_ping_timeout', 2) self.ping_timeout = self.replicate_config.get('follower_ping_timeout', 2)
self.sleep_time = self.replicate_config.get('follower_sleep_time', 1) self.sleep_time = self.replicate_config.get('follower_sleep_time', 5)
if self.mode == ReplicateModeType.follower and len(self.leaders_list) == 0: if self.mode == ReplicateModeType.follower and len(self.leaders_list) == 0:
raise ValueError("You must specify at least 1 leader in follower mode.") raise ValueError("You must specify at least 1 leader in follower mode.")
@ -143,6 +145,8 @@ class ReplicateController(RPCHandler):
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass
except Exception:
pass
finally: finally:
self._loop.stop() self._loop.stop()
@ -170,22 +174,19 @@ class ReplicateController(RPCHandler):
self._thread.join() self._thread.join()
def send_msg(self, msg: Dict[str, Any]) -> None: def send_msg(self, msg: MessageType) -> None:
""" """
Support RPC calls Support RPC calls
""" """
if msg["type"] == RPCMessageType.EMIT_DATA: if msg["type"] == RPCMessageType.EMIT_DATA:
self.send_message( message = msg.get("message")
{ if message:
"data_type": msg.get("data_type"), self.send_message(message)
"data": msg.get("data") else:
} logger.error(f"Message is empty! {msg}")
)
def send_message(self, msg: Dict[str, Any]) -> None: def send_message(self, msg: MessageType) -> None:
""" Push message through """ """ Broadcast message over all channels if there are any """
# We should probably do some type of schema validation here
if self.channel_manager.has_channels(): if self.channel_manager.has_channels():
self._send_message(msg) self._send_message(msg)
@ -193,12 +194,11 @@ class ReplicateController(RPCHandler):
logger.debug("No listening followers, skipping...") logger.debug("No listening followers, skipping...")
pass pass
def _send_message(self, msg: Dict[Any, Any]): def _send_message(self, msg: MessageType):
""" """
Add data to the internal queue to be broadcasted. This func will block Add data to the internal queue to be broadcasted. This func will block
if the queue is full. This is meant to be called in the main thread. if the queue is full. This is meant to be called in the main thread.
""" """
if self._queue: if self._queue:
queue = self._queue.sync_q queue = self._queue.sync_q
queue.put(msg) # This will block if the queue is full queue.put(msg) # This will block if the queue is full
@ -226,7 +226,6 @@ class ReplicateController(RPCHandler):
This starts all of the leader coros and registers the endpoint on This starts all of the leader coros and registers the endpoint on
the ApiServer the ApiServer
""" """
logger.info("Running rpc.replicate in Leader mode") logger.info("Running rpc.replicate in Leader mode")
logger.info("-" * 15) logger.info("-" * 15)
logger.info(f"API_KEY: {self.secret_api_key}") logger.info(f"API_KEY: {self.secret_api_key}")
@ -253,16 +252,17 @@ class ReplicateController(RPCHandler):
# Get data from queue # Get data from queue
data = await async_queue.get() data = await async_queue.get()
logger.info(f"Found data - broadcasting: {data}")
# Broadcast it to everyone # Broadcast it to everyone
await self.channel_manager.broadcast(data) await self.channel_manager.broadcast(data)
# Sleep # Sleep
await asyncio.sleep(self.push_throttle_secs) await asyncio.sleep(self.push_throttle_secs)
except asyncio.CancelledError: except asyncio.CancelledError:
# Silently stop # Silently stop
pass pass
except Exception as e:
logger.exception(e)
async def get_api_token( async def get_api_token(
self, self,
@ -285,7 +285,6 @@ class ReplicateController(RPCHandler):
:param path: The endpoint path :param path: The endpoint path
""" """
if not self.api_server: if not self.api_server:
raise RuntimeError("The leader needs the ApiServer to be active") raise RuntimeError("The leader needs the ApiServer to be active")
@ -312,10 +311,13 @@ class ReplicateController(RPCHandler):
# we may not have to send initial data at all. Further testing # we may not have to send initial data at all. Further testing
# required. # required.
await self.send_initial_data(channel)
# Keep connection open until explicitly closed, and sleep # Keep connection open until explicitly closed, and sleep
try: try:
while not channel.is_closed(): while not channel.is_closed():
await channel.recv() request = await channel.recv()
logger.info(f"Follower request - {request}")
except WebSocketDisconnect: except WebSocketDisconnect:
# Handle client disconnects # Handle client disconnects
@ -332,6 +334,17 @@ class ReplicateController(RPCHandler):
logger.error(f"Failed to serve - {websocket.client}") logger.error(f"Failed to serve - {websocket.client}")
await self.channel_manager.on_disconnect(websocket) await self.channel_manager.on_disconnect(websocket)
async def send_initial_data(self, channel):
logger.info("Sending initial data through channel")
# We first send pairlist data
initial_data = {
"data_type": LeaderMessageType.pairlist,
"data": self.freqtrade.pairlists.whitelist
}
await channel.send(initial_data)
# -------------------------------FOLLOWER LOGIC---------------------------- # -------------------------------FOLLOWER LOGIC----------------------------
async def follower_loop(self): async def follower_loop(self):
@ -340,18 +353,27 @@ class ReplicateController(RPCHandler):
This starts all of the follower connection coros This starts all of the follower connection coros
""" """
logger.info("Starting rpc.replicate in Follower mode") logger.info("Starting rpc.replicate in Follower mode")
try: responses = await self._connect_to_leaders()
results = await self._connect_to_leaders()
except Exception as e: # Eventually add the ability to send requests to the Leader
logger.error("Exception occurred in Follower loop: ") # await self._send_requests()
logger.exception(e)
finally: for result in responses:
for result in results:
if isinstance(result, Exception): if isinstance(result, Exception):
logger.debug(f"Exception in Follower loop: {result}") logger.debug(f"Exception in Follower loop: {result}")
traceback_message = ''.join(traceback.format_tb(result.__traceback__))
logger.error(traceback_message)
async def _handle_leader_message(self, message: MessageType):
"""
Handle message received from a Leader
"""
type = message.get("data_type")
data = message.get("data")
self._rpc._handle_emitted_data(type, data)
async def _connect_to_leaders(self): async def _connect_to_leaders(self):
""" """
@ -375,7 +397,6 @@ class ReplicateController(RPCHandler):
""" """
try: try:
url, token = leader["url"], leader["token"] url, token = leader["url"], leader["token"]
websocket_url = f"{url}?token={token}" websocket_url = f"{url}?token={token}"
logger.info(f"Attempting to connect to Leader at: {url}") logger.info(f"Attempting to connect to Leader at: {url}")
@ -384,6 +405,7 @@ class ReplicateController(RPCHandler):
try: try:
async with websockets.connect(websocket_url) as ws: async with websockets.connect(websocket_url) as ws:
channel = await self.channel_manager.on_connect(ws) channel = await self.channel_manager.on_connect(ws)
logger.info(f"Connection to Leader at {url} successful")
while True: while True:
try: try:
data = await asyncio.wait_for( data = await asyncio.wait_for(
@ -420,13 +442,3 @@ class ReplicateController(RPCHandler):
except asyncio.CancelledError: except asyncio.CancelledError:
pass pass
async def _handle_leader_message(self, message: Dict[str, Any]):
type = message.get('data_type')
data = message.get('data')
logger.info(f"Received message from Leader: {type} - {data}")
if type == LeaderMessageType.pairlist:
# Add the data to the ExternalPairlist
self.freqtrade.pairlists._pairlist_handlers[0].add_pairlist_data(data)

View File

@ -2,7 +2,7 @@ import logging
from typing import Type from typing import Type
from freqtrade.rpc.replicate.proxy import WebSocketProxy from freqtrade.rpc.replicate.proxy import WebSocketProxy
from freqtrade.rpc.replicate.serializer import JSONWebSocketSerializer, WebSocketSerializer from freqtrade.rpc.replicate.serializer import MsgPackWebSocketSerializer, WebSocketSerializer
from freqtrade.rpc.replicate.types import WebSocketType from freqtrade.rpc.replicate.types import WebSocketType
@ -17,7 +17,7 @@ class WebSocketChannel:
def __init__( def __init__(
self, self,
websocket: WebSocketType, websocket: WebSocketType,
serializer_cls: Type[WebSocketSerializer] = JSONWebSocketSerializer serializer_cls: Type[WebSocketSerializer] = MsgPackWebSocketSerializer
): ):
# The WebSocket object # The WebSocket object
self._websocket = WebSocketProxy(websocket) self._websocket = WebSocketProxy(websocket)
@ -34,6 +34,7 @@ class WebSocketChannel:
""" """
Send data on the wrapped websocket Send data on the wrapped websocket
""" """
# logger.info(f"Serialized Send - {self._wrapped_ws._serialize(data)}")
await self._wrapped_ws.send(data) await self._wrapped_ws.send(data)
async def recv(self): async def recv(self):
@ -116,6 +117,17 @@ class ChannelManager:
# Handle cannot send after close cases # Handle cannot send after close cases
await self.on_disconnect(websocket) await self.on_disconnect(websocket)
async def send_direct(self, channel, data):
"""
Send data directly through direct_channel only
:param direct_channel: The WebSocketChannel object to send data through
:param data: The data to send
"""
# We iterate over the channels to get reference to the websocket object
# so we can disconnect incase of failure
await channel.send(data)
def has_channels(self): def has_channels(self):
""" """
Flag for more than 0 channels Flag for more than 0 channels

View File

@ -1,9 +1,16 @@
import json import json
import logging
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
import msgpack
import orjson
from freqtrade.rpc.replicate.proxy import WebSocketProxy from freqtrade.rpc.replicate.proxy import WebSocketProxy
logger = logging.getLogger(__name__)
class WebSocketSerializer(ABC): class WebSocketSerializer(ABC):
def __init__(self, websocket: WebSocketProxy): def __init__(self, websocket: WebSocketProxy):
self._websocket: WebSocketProxy = websocket self._websocket: WebSocketProxy = websocket
@ -34,9 +41,25 @@ class WebSocketSerializer(ABC):
class JSONWebSocketSerializer(WebSocketSerializer): class JSONWebSocketSerializer(WebSocketSerializer):
def _serialize(self, data): def _serialize(self, data):
# json expects string not bytes
return json.dumps(data) return json.dumps(data)
def _deserialize(self, data): def _deserialize(self, data):
# The WebSocketSerializer gives bytes not string
return json.loads(data) return json.loads(data)
class ORJSONWebSocketSerializer(WebSocketSerializer):
ORJSON_OPTIONS = orjson.OPT_NAIVE_UTC | orjson.OPT_SERIALIZE_NUMPY
def _serialize(self, data):
return orjson.dumps(data, option=self.ORJSON_OPTIONS)
def _deserialize(self, data):
return orjson.loads(data, option=self.ORJSON_OPTIONS)
class MsgPackWebSocketSerializer(WebSocketSerializer):
def _serialize(self, data):
return msgpack.packb(data, use_bin_type=True)
def _deserialize(self, data):
return msgpack.unpackb(data, raw=False)

View File

@ -1,7 +1,8 @@
from typing import TypeVar from typing import Any, Dict, TypeVar
from fastapi import WebSocket as FastAPIWebSocket from fastapi import WebSocket as FastAPIWebSocket
from websockets import WebSocketClientProtocol as WebSocket from websockets import WebSocketClientProtocol as WebSocket
WebSocketType = TypeVar("WebSocketType", FastAPIWebSocket, WebSocket) WebSocketType = TypeVar("WebSocketType", FastAPIWebSocket, WebSocket)
MessageType = Dict[str, Any]

View File

@ -19,12 +19,12 @@ from freqtrade.configuration.timerange import TimeRange
from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT
from freqtrade.data.history import load_data from freqtrade.data.history import load_data
from freqtrade.data.metrics import calculate_max_drawdown from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State, from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, LeaderMessageType,
TradingMode) SignalDirection, State, TradingMode)
from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler from freqtrade.loggers import bufferHandler
from freqtrade.misc import decimals_per_coin, shorten_date from freqtrade.misc import decimals_per_coin, json_to_dataframe, shorten_date
from freqtrade.persistence import PairLocks, Trade from freqtrade.persistence import PairLocks, Trade
from freqtrade.persistence.models import PairLock from freqtrade.persistence.models import PairLock
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@ -1089,3 +1089,36 @@ class RPC:
'last_process_loc': last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT), 'last_process_loc': last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT),
'last_process_ts': int(last_p.timestamp()), 'last_process_ts': int(last_p.timestamp()),
} }
def _handle_emitted_data(self, type, data):
"""
Handles the emitted data from the Leaders
:param type: The data_type of the data
:param data: The data
"""
logger.debug(f"Handling emitted data of type ({type})")
if type == LeaderMessageType.pairlist:
pairlist = data
logger.debug(pairlist)
# Add the pairlist data to the ExternalPairList object
external_pairlist = self._freqtrade.pairlists._pairlist_handlers[0]
external_pairlist.add_pairlist_data(pairlist)
elif type == LeaderMessageType.analyzed_df:
# Convert the dataframe back from json
key, value = data["key"], data["value"]
pair, timeframe, candle_type = key
dataframe = json_to_dataframe(value)
dataprovider = self._freqtrade.dataprovider
logger.debug(f"Received analyzed dataframe for {pair}")
logger.debug(dataframe.tail())
# Add the dataframe to the dataprovider
dataprovider.add_external_df(pair, timeframe, dataframe, candle_type)

View File

@ -20,6 +20,7 @@ class RPCManager:
def __init__(self, freqtrade) -> None: def __init__(self, freqtrade) -> None:
""" Initializes all enabled rpc modules """ """ Initializes all enabled rpc modules """
self.registered_modules: List[RPCHandler] = [] self.registered_modules: List[RPCHandler] = []
self._freqtrade = freqtrade
self._rpc = RPC(freqtrade) self._rpc = RPC(freqtrade)
config = freqtrade.config config = freqtrade.config
# Enable telegram # Enable telegram
@ -82,6 +83,7 @@ class RPCManager:
'status': 'stopping bot' 'status': 'stopping bot'
} }
""" """
if msg.get("type") != RPCMessageType.EMIT_DATA:
logger.info('Sending rpc message: %s', msg) logger.info('Sending rpc message: %s', msg)
if 'pair' in msg: if 'pair' in msg:
msg.update({ msg.update({
@ -141,3 +143,12 @@ class RPCManager:
'type': RPCMessageType.STARTUP, 'type': RPCMessageType.STARTUP,
'status': f'Using Protections: \n{prots}' 'status': f'Using Protections: \n{prots}'
}) })
def emit_data(self, data: Dict[str, Any]):
"""
Send a message via RPC with type RPCMessageType.EMIT_DATA
"""
self.send_msg({
"type": RPCMessageType.EMIT_DATA,
"message": data
})

View File

@ -5,7 +5,7 @@ This module defines the interface to apply for strategies
import logging import logging
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple, Union from typing import Callable, Dict, List, Optional, Tuple, Union
import arrow import arrow
from pandas import DataFrame from pandas import DataFrame
@ -18,6 +18,7 @@ from freqtrade.enums.runmode import RunMode
from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exceptions import OperationalException, StrategyError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date, timeframe_to_seconds
from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence import Order, PairLocks, Trade
from freqtrade.rpc.replicate import ReplicateController
from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.hyper import HyperStrategyMixin
from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators, from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators,
_create_and_merge_informative_pair, _create_and_merge_informative_pair,
@ -110,6 +111,7 @@ class IStrategy(ABC, HyperStrategyMixin):
# the dataprovider (dp) (access to other candles, historic data, ...) # the dataprovider (dp) (access to other candles, historic data, ...)
# and wallets - access to the current balance. # and wallets - access to the current balance.
dp: DataProvider dp: DataProvider
replicate_controller: Optional[ReplicateController]
wallets: Optional[Wallets] = None wallets: Optional[Wallets] = None
# Filled from configuration # Filled from configuration
stake_currency: str stake_currency: str
@ -123,6 +125,7 @@ class IStrategy(ABC, HyperStrategyMixin):
self.config = config self.config = config
# Dict to determine if analysis is necessary # Dict to determine if analysis is necessary
self._last_candle_seen_per_pair: Dict[str, datetime] = {} self._last_candle_seen_per_pair: Dict[str, datetime] = {}
self._last_candle_seen_external: Dict[str, datetime] = {}
super().__init__(config) super().__init__(config)
# Gather informative pairs from @informative-decorated methods. # Gather informative pairs from @informative-decorated methods.
@ -678,7 +681,12 @@ class IStrategy(ABC, HyperStrategyMixin):
lock_time = timeframe_to_next_date(self.timeframe, candle_date) lock_time = timeframe_to_next_date(self.timeframe, candle_date)
return PairLocks.is_pair_locked(pair, lock_time, side=side) return PairLocks.is_pair_locked(pair, lock_time, side=side)
def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def analyze_ticker(
self,
dataframe: DataFrame,
metadata: dict,
populate_indicators: bool = True
) -> DataFrame:
""" """
Parses the given candle (OHLCV) data and returns a populated DataFrame Parses the given candle (OHLCV) data and returns a populated DataFrame
add several TA indicators and entry order signal to it add several TA indicators and entry order signal to it
@ -687,12 +695,19 @@ class IStrategy(ABC, HyperStrategyMixin):
:return: DataFrame of candle (OHLCV) data with indicator data and signals added :return: DataFrame of candle (OHLCV) data with indicator data and signals added
""" """
logger.debug("TA Analysis Launched") logger.debug("TA Analysis Launched")
if populate_indicators:
dataframe = self.advise_indicators(dataframe, metadata) dataframe = self.advise_indicators(dataframe, metadata)
dataframe = self.advise_entry(dataframe, metadata) dataframe = self.advise_entry(dataframe, metadata)
dataframe = self.advise_exit(dataframe, metadata) dataframe = self.advise_exit(dataframe, metadata)
return dataframe return dataframe
def _analyze_ticker_internal(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def _analyze_ticker_internal(
self,
dataframe: DataFrame,
metadata: dict,
external_data: bool = False,
finish_callback: Optional[Callable] = None,
) -> DataFrame:
""" """
Parses the given candle (OHLCV) data and returns a populated DataFrame Parses the given candle (OHLCV) data and returns a populated DataFrame
add several TA indicators and buy signal to it add several TA indicators and buy signal to it
@ -707,12 +722,19 @@ class IStrategy(ABC, HyperStrategyMixin):
# always run if process_only_new_candles is set to false # always run if process_only_new_candles is set to false
if (not self.process_only_new_candles or if (not self.process_only_new_candles or
self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']): self._last_candle_seen_per_pair.get(pair, None) != dataframe.iloc[-1]['date']):
populate_indicators = not external_data
# Defs that only make change on new candle data. # Defs that only make change on new candle data.
dataframe = self.analyze_ticker(dataframe, metadata) dataframe = self.analyze_ticker(dataframe, metadata, populate_indicators)
self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date']
self.dp._set_cached_df(
pair, self.timeframe, dataframe, candle_type = self.config.get('candle_type_def', CandleType.SPOT)
candle_type=self.config.get('candle_type_def', CandleType.SPOT)) self.dp._set_cached_df(pair, self.timeframe, dataframe, candle_type=candle_type)
if finish_callback:
finish_callback(pair, dataframe, self.timeframe, candle_type)
else: else:
logger.debug("Skipping TA Analysis for already analyzed candle") logger.debug("Skipping TA Analysis for already analyzed candle")
dataframe[SignalType.ENTER_LONG.value] = 0 dataframe[SignalType.ENTER_LONG.value] = 0
@ -726,16 +748,25 @@ class IStrategy(ABC, HyperStrategyMixin):
return dataframe return dataframe
def analyze_pair(self, pair: str) -> None: def analyze_pair(
self,
pair: str,
external_data: bool = False,
finish_callback: Optional[Callable] = None,
) -> None:
""" """
Fetch data for this pair from dataprovider and analyze. Fetch data for this pair from dataprovider and analyze.
Stores the dataframe into the dataprovider. Stores the dataframe into the dataprovider.
The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`. The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`.
:param pair: Pair to analyze. :param pair: Pair to analyze.
""" """
dataframe = self.dp.ohlcv( candle_type = self.config.get('candle_type_def', CandleType.SPOT)
pair, self.timeframe, candle_type=self.config.get('candle_type_def', CandleType.SPOT)
) if not external_data:
dataframe = self.dp.ohlcv(pair, self.timeframe, candle_type)
else:
dataframe, last_analyzed = self.dp.get_external_df(pair, self.timeframe, candle_type)
if not isinstance(dataframe, DataFrame) or dataframe.empty: if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty candle (OHLCV) data for pair %s', pair) logger.warning('Empty candle (OHLCV) data for pair %s', pair)
return return
@ -745,7 +776,7 @@ class IStrategy(ABC, HyperStrategyMixin):
dataframe = strategy_safe_wrapper( dataframe = strategy_safe_wrapper(
self._analyze_ticker_internal, message="" self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair}) )(dataframe, {'pair': pair}, external_data, finish_callback)
self.assert_df(dataframe, df_len, df_close, df_date) self.assert_df(dataframe, df_len, df_close, df_date)
except StrategyError as error: except StrategyError as error:
@ -756,15 +787,43 @@ class IStrategy(ABC, HyperStrategyMixin):
logger.warning('Empty dataframe for pair %s', pair) logger.warning('Empty dataframe for pair %s', pair)
return return
def analyze(self, pairs: List[str]) -> None: def analyze(
self,
pairs: List[str],
finish_callback: Optional[Callable] = None
) -> None:
""" """
Analyze all pairs using analyze_pair(). Analyze all pairs using analyze_pair().
:param pairs: List of pairs to analyze :param pairs: List of pairs to analyze
""" """
for pair in pairs: for pair in pairs:
self.analyze_pair(pair, finish_callback=finish_callback)
def analyze_external(self, pairs: List[str], leader_pairs: List[str]) -> None:
"""
Analyze the pre-populated dataframes from the Leader
:param pairs: The active pair whitelist
:param leader_pairs: The list of pairs from the Leaders
"""
# Get the extra pairs not listed in Leader pairs, and process
# them normally.
# List order is not preserved when doing this!
# We use ^ instead of - for symmetric difference
# What do we do with these?
extra_pairs = list(set(pairs) ^ set(leader_pairs))
# These would be the pairs that we have trades in, which means
# we would have to analyze them normally
for pair in leader_pairs:
# Analyze the pairs, but get the dataframe from the external data
self.analyze_pair(pair, external_data=True)
for pair in extra_pairs:
self.analyze_pair(pair) self.analyze_pair(pair)
@staticmethod @ staticmethod
def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]: def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]:
""" keep some data for dataframes """ """ keep some data for dataframes """
return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1] return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1]
@ -1185,6 +1244,9 @@ class IStrategy(ABC, HyperStrategyMixin):
dataframe = _create_and_merge_informative_pair( dataframe = _create_and_merge_informative_pair(
self, dataframe, metadata, inf_data, populate_fn) self, dataframe, metadata, inf_data, populate_fn)
# If in follower mode, get analyzed dataframe from leader df's in dp
# otherise run populate_indicators
return self.populate_indicators(dataframe, metadata) return self.populate_indicators(dataframe, metadata)
def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame: def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

View File

@ -3,3 +3,4 @@
# Required for follower # Required for follower
websockets websockets
msgpack