adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove data waiting, remove explicit analyzing of external df

Browse Source
This commit is contained in:
Timothy Pogue
2022-08-31 10:40:26 -06:00
parent 115a901773
commit 510cf4f305
8 changed files with 182 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
freqtrade/rpc/api_server/api_ws.py
View File

@@ -1,22 +1,48 @@
import logging
from typing import Any, Dict
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
from freqtrade.rpc.api_server.deps import get_channel_manager, get_rpc_optional
from freqtrade.enums import RPCMessageType, RPCRequestType
from freqtrade.rpc.api_server.deps import get_channel_manager
from freqtrade.rpc.api_server.ws.channel import WebSocketChannel
from freqtrade.rpc.api_server.ws.utils import is_websocket_alive
# from typing import Any, Dict
logger = logging.getLogger(__name__)
# Private router, protected by API Key authentication
router = APIRouter()
# We are passed a Channel object, we can only do sync functions on that channel object
def _process_consumer_request(request: Dict[str, Any], channel: WebSocketChannel):
type, data = request.get('type'), request.get('data')
# If the request is empty, do nothing
if not data:
return
# If we have a request of type SUBSCRIBE, set the topics in this channel
if type == RPCRequestType.SUBSCRIBE:
if isinstance(data, list):
logger.error(f"Improper request from channel: {channel} - {request}")
return
# If all topics passed are a valid RPCMessageType, set subscriptions on channel
if all([any(x.value == topic for x in RPCMessageType) for topic in data]):
logger.debug(f"{channel} subscribed to topics: {data}")
channel.set_subscriptions(data)
@router.websocket("/message/ws")
async def message_endpoint(
ws: WebSocket,
channel_manager=Depends(get_channel_manager),
rpc=Depends(get_rpc_optional)
channel_manager=Depends(get_channel_manager)
):
try:
if is_websocket_alive(ws):
@@ -32,9 +58,8 @@ async def message_endpoint(
request = await channel.recv()
# Process the request here. Should this be a method of RPC?
if rpc:
logger.info(f"Request: {request}")
rpc._process_consumer_request(request, channel)
logger.info(f"Request: {request}")
_process_consumer_request(request, channel)
except WebSocketDisconnect:
# Handle client disconnects

153
freqtrade/rpc/emc.py
View File

@@ -12,9 +12,9 @@ from typing import Any, Dict
import websockets
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import RPCMessageType, RPCRequestType
from freqtrade.misc import json_to_dataframe, remove_entry_exit_signals
from freqtrade.rpc import RPC
from freqtrade.rpc.api_server.ws.channel import WebSocketChannel
@@ -29,11 +29,11 @@ class ExternalMessageConsumer:
def __init__(
self,
rpc: RPC,
config: Dict[str, Any],
dataprovider: DataProvider
):
self._rpc = rpc
self._config = config
self._dp = dataprovider
self._running = False
self._thread = None
@@ -99,12 +99,12 @@ class ExternalMessageConsumer:
"""
The main task coroutine
"""
rpc_lock = asyncio.Lock()
lock = asyncio.Lock()
try:
# Create a connection to each producer
self._sub_tasks = [
self._loop.create_task(self._handle_producer_connection(producer, rpc_lock))
self._loop.create_task(self._handle_producer_connection(producer, lock))
for producer in self.producers
]
@@ -115,73 +115,90 @@ class ExternalMessageConsumer:
# Stop the loop once we are done
self._loop.stop()
async def _handle_producer_connection(self, producer, lock):
async def _handle_producer_connection(self, producer: Dict[str, Any], lock: asyncio.Lock):
"""
Main connection loop for the consumer
:param producer: Dictionary containing producer info: {'url': '', 'ws_token': ''}
:param lock: An asyncio Lock
"""
try:
while True:
try:
url, token = producer['url'], producer['ws_token']
ws_url = f"{url}?token={token}"
async with websockets.connect(ws_url) as ws:
logger.info("Connection successful")
channel = WebSocketChannel(ws)
# Tell the producer we only want these topics
# Should always be the first thing we send
await channel.send(
self.compose_consumer_request(RPCRequestType.SUBSCRIBE, self.topics)
)
# Now receive data, if none is within the time limit, ping
while True:
try:
message = await asyncio.wait_for(
channel.recv(),
timeout=5
)
async with lock:
# Handle the data here
# We use a lock because it will call RPC methods
self.handle_producer_message(message)
except (asyncio.TimeoutError, websockets.exceptions.ConnectionClosed):
# We haven't received data yet. Check the connection and continue.
try:
# ping
ping = await channel.ping()
await asyncio.wait_for(ping, timeout=self.ping_timeout)
logger.debug(f"Connection to {url} still alive...")
continue
except Exception:
logger.info(
f"Ping error {url} - retrying in {self.sleep_time}s")
await asyncio.sleep(self.sleep_time)
break
except Exception as e:
logger.exception(e)
continue
except (
socket.gaierror,
ConnectionRefusedError,
websockets.exceptions.InvalidStatusCode
) as e:
logger.error(f"Connection Refused - {e} retrying in {self.sleep_time}s")
await asyncio.sleep(self.sleep_time)
continue
await self._create_connection(producer, lock)
except asyncio.CancelledError:
# Exit silently
pass
def compose_consumer_request(self, type_: str, data: Any) -> Dict[str, Any]:
async def _create_connection(self, producer: Dict[str, Any], lock: asyncio.Lock):
"""
Actually creates and handles the websocket connection, pinging on timeout
and handling connection errors.
:param producer: Dictionary containing producer info: {'url': '', 'ws_token': ''}
:param lock: An asyncio Lock
"""
while self._running:
try:
url, token = producer['url'], producer['ws_token']
ws_url = f"{url}?token={token}"
# This will raise InvalidURI if the url is bad
async with websockets.connect(ws_url) as ws:
logger.info("Connection successful")
channel = WebSocketChannel(ws)
# Tell the producer we only want these topics
# Should always be the first thing we send
await channel.send(
self.compose_consumer_request(RPCRequestType.SUBSCRIBE, self.topics)
)
# Now receive data, if none is within the time limit, ping
while True:
try:
message = await asyncio.wait_for(
channel.recv(),
timeout=self.reply_timeout
)
async with lock:
# Handle the message
self.handle_producer_message(message)
except (asyncio.TimeoutError, websockets.exceptions.ConnectionClosed):
# We haven't received data yet. Check the connection and continue.
try:
# ping
ping = await channel.ping()
await asyncio.wait_for(ping, timeout=self.ping_timeout)
logger.debug(f"Connection to {url} still alive...")
continue
except Exception:
logger.info(
f"Ping error {url} - retrying in {self.sleep_time}s")
await asyncio.sleep(self.sleep_time)
break
except Exception as e:
logger.exception(e)
continue
except (
socket.gaierror,
ConnectionRefusedError,
websockets.exceptions.InvalidStatusCode
) as e:
logger.error(f"Connection Refused - {e} retrying in {self.sleep_time}s")
await asyncio.sleep(self.sleep_time)
continue
# Catch invalid ws_url, and break the loop
except websockets.exceptions.InvalidURI as e:
logger.error(f"{ws_url} is an invalid WebSocket URL - {e}")
break
def compose_consumer_request(self, type_: RPCRequestType, data: Any) -> Dict[str, Any]:
"""
Create a request for sending to a producer
@@ -211,9 +228,8 @@ class ExternalMessageConsumer:
if message_type == RPCMessageType.WHITELIST:
pairlist = message_data
# Add the pairlist data to the ExternalPairlist plugin
external_pairlist = self._rpc._freqtrade.pairlists._pairlist_handlers[0]
external_pairlist.add_pairlist_data(pairlist)
# Add the pairlist data to the DataProvider
self._dp.set_producer_pairs(pairlist)
# Handle analyzed dataframes
elif message_type == RPCMessageType.ANALYZED_DF:
@@ -230,5 +246,4 @@ class ExternalMessageConsumer:
dataframe = remove_entry_exit_signals(dataframe)
# Add the dataframe to the dataprovider
dataprovider = self._rpc._freqtrade.dataprovider
dataprovider.add_external_df(pair, timeframe, dataframe, candle_type)
self._dp.add_external_df(pair, timeframe, dataframe, candle_type)

14
freqtrade/rpc/rpc.py
View File

@@ -19,8 +19,8 @@ from freqtrade.configuration.timerange import TimeRange
from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT
from freqtrade.data.history import load_data
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, RPCMessageType, RPCRequestType,
SignalDirection, State, TradingMode)
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State,
TradingMode)
from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
@@ -1089,13 +1089,3 @@ class RPC:
'last_process_loc': last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT),
'last_process_ts': int(last_p.timestamp()),
}
# We are passed a Channel object, we can only do sync functions on that channel object
def _process_consumer_request(self, request, channel):
# Should we ensure that request is Dict[str, Any]?
type, data = request.get('type'), request.get('data')
if type == RPCRequestType.SUBSCRIBE:
if all([any(x.value == topic for x in RPCMessageType) for topic in data]):
logger.debug(f"{channel} subscribed to topics: {data}")
channel.set_subscriptions(data)