Renamed to external signals, controller class refactored
This commit is contained in:
Timothy Pogue
5
freqtrade/rpc/external_signal/__init__.py
Normal file
5
freqtrade/rpc/external_signal/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
# flake8: noqa: F401
|
||||
from freqtrade.rpc.external_signal.controller import ExternalSignalController
|
||||
|
||||
|
||||
__all__ = ('ExternalSignalController')
|
||||
135
freqtrade/rpc/external_signal/channel.py
Normal file
135
freqtrade/rpc/external_signal/channel.py
Normal file
@@ -0,0 +1,135 @@
|
||||
import logging
|
||||
from typing import Type
|
||||
|
||||
from freqtrade.rpc.external_signal.proxy import WebSocketProxy
|
||||
from freqtrade.rpc.external_signal.serializer import MsgPackWebSocketSerializer, WebSocketSerializer
|
||||
from freqtrade.rpc.external_signal.types import WebSocketType
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WebSocketChannel:
|
||||
"""
|
||||
Object to help facilitate managing a websocket connection
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
websocket: WebSocketType,
|
||||
serializer_cls: Type[WebSocketSerializer] = MsgPackWebSocketSerializer
|
||||
):
|
||||
# The WebSocket object
|
||||
self._websocket = WebSocketProxy(websocket)
|
||||
# The Serializing class for the WebSocket object
|
||||
self._serializer_cls = serializer_cls
|
||||
|
||||
# Internal event to signify a closed websocket
|
||||
self._closed = False
|
||||
|
||||
# Wrap the WebSocket in the Serializing class
|
||||
self._wrapped_ws = self._serializer_cls(self._websocket)
|
||||
|
||||
async def send(self, data):
|
||||
"""
|
||||
Send data on the wrapped websocket
|
||||
"""
|
||||
# logger.info(f"Serialized Send - {self._wrapped_ws._serialize(data)}")
|
||||
await self._wrapped_ws.send(data)
|
||||
|
||||
async def recv(self):
|
||||
"""
|
||||
Receive data on the wrapped websocket
|
||||
"""
|
||||
return await self._wrapped_ws.recv()
|
||||
|
||||
async def ping(self):
|
||||
"""
|
||||
Ping the websocket
|
||||
"""
|
||||
return await self._websocket.ping()
|
||||
|
||||
async def close(self):
|
||||
"""
|
||||
Close the WebSocketChannel
|
||||
"""
|
||||
|
||||
self._closed = True
|
||||
|
||||
def is_closed(self):
|
||||
return self._closed
|
||||
|
||||
|
||||
class ChannelManager:
|
||||
def __init__(self):
|
||||
self.channels = dict()
|
||||
|
||||
async def on_connect(self, websocket: WebSocketType):
|
||||
"""
|
||||
Wrap websocket connection into Channel and add to list
|
||||
|
||||
:param websocket: The WebSocket object to attach to the Channel
|
||||
"""
|
||||
if hasattr(websocket, "accept"):
|
||||
try:
|
||||
await websocket.accept()
|
||||
except RuntimeError:
|
||||
# The connection was closed before we could accept it
|
||||
return
|
||||
|
||||
ws_channel = WebSocketChannel(websocket)
|
||||
self.channels[websocket] = ws_channel
|
||||
|
||||
return ws_channel
|
||||
|
||||
async def on_disconnect(self, websocket: WebSocketType):
|
||||
"""
|
||||
Call close on the channel if it's not, and remove from channel list
|
||||
|
||||
:param websocket: The WebSocket objet attached to the Channel
|
||||
"""
|
||||
if websocket in self.channels.keys():
|
||||
channel = self.channels[websocket]
|
||||
|
||||
logger.debug(f"Disconnecting channel - {channel}")
|
||||
|
||||
if not channel.is_closed():
|
||||
await channel.close()
|
||||
del self.channels[websocket]
|
||||
|
||||
async def disconnect_all(self):
|
||||
"""
|
||||
Disconnect all Channels
|
||||
"""
|
||||
for websocket in self.channels.keys():
|
||||
await self.on_disconnect(websocket)
|
||||
|
||||
async def broadcast(self, data):
|
||||
"""
|
||||
Broadcast data on all Channels
|
||||
|
||||
:param data: The data to send
|
||||
"""
|
||||
for websocket, channel in self.channels.items():
|
||||
try:
|
||||
await channel.send(data)
|
||||
except RuntimeError:
|
||||
# Handle cannot send after close cases
|
||||
await self.on_disconnect(websocket)
|
||||
|
||||
async def send_direct(self, channel, data):
|
||||
"""
|
||||
Send data directly through direct_channel only
|
||||
|
||||
:param direct_channel: The WebSocketChannel object to send data through
|
||||
:param data: The data to send
|
||||
"""
|
||||
# We iterate over the channels to get reference to the websocket object
|
||||
# so we can disconnect incase of failure
|
||||
await channel.send(data)
|
||||
|
||||
def has_channels(self):
|
||||
"""
|
||||
Flag for more than 0 channels
|
||||
"""
|
||||
return len(self.channels) > 0
|
||||
440
freqtrade/rpc/external_signal/controller.py
Normal file
440
freqtrade/rpc/external_signal/controller.py
Normal file
@@ -0,0 +1,440 @@
|
||||
"""
|
||||
This module manages replicate mode communication
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import secrets
|
||||
import socket
|
||||
from threading import Thread
|
||||
from typing import Any, Coroutine, Dict, Union
|
||||
|
||||
import websockets
|
||||
from fastapi import Depends
|
||||
from fastapi import WebSocket as FastAPIWebSocket
|
||||
from fastapi import WebSocketDisconnect, status
|
||||
|
||||
from freqtrade.enums import ExternalSignalModeType, LeaderMessageType, RPCMessageType
|
||||
from freqtrade.rpc import RPC, RPCHandler
|
||||
from freqtrade.rpc.external_signal.channel import ChannelManager
|
||||
from freqtrade.rpc.external_signal.thread_queue import Queue as ThreadedQueue
|
||||
from freqtrade.rpc.external_signal.types import MessageType
|
||||
from freqtrade.rpc.external_signal.utils import is_websocket_alive
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ExternalSignalController(RPCHandler):
|
||||
""" This class handles all websocket communication """
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
rpc: RPC,
|
||||
config: Dict[str, Any],
|
||||
api_server: Union[Any, None] = None
|
||||
) -> None:
|
||||
"""
|
||||
Init the ExternalSignalController class, and init the super class RPCHandler
|
||||
:param rpc: instance of RPC Helper class
|
||||
:param config: Configuration object
|
||||
:param api_server: The ApiServer object
|
||||
:return: None
|
||||
"""
|
||||
super().__init__(rpc, config)
|
||||
|
||||
self.freqtrade = rpc._freqtrade
|
||||
self.api_server = api_server
|
||||
|
||||
if not self.api_server:
|
||||
raise RuntimeError("The API server must be enabled for external signals to work")
|
||||
|
||||
self._loop = None
|
||||
self._running = False
|
||||
self._thread = None
|
||||
self._queue = None
|
||||
|
||||
self._main_task = None
|
||||
self._sub_tasks = None
|
||||
|
||||
self.channel_manager = ChannelManager()
|
||||
|
||||
self.external_signal_config = config.get('external_signal', {})
|
||||
|
||||
# What the config should look like
|
||||
# "external_signal": {
|
||||
# "enabled": true,
|
||||
# "mode": "follower",
|
||||
# "leaders": [
|
||||
# {
|
||||
# "url": "ws://localhost:8080/signals/ws",
|
||||
# "api_token": "test"
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
|
||||
# "external_signal": {
|
||||
# "enabled": true,
|
||||
# "mode": "leader",
|
||||
# "api_token": "test"
|
||||
# }
|
||||
|
||||
self.mode = ExternalSignalModeType[
|
||||
self.external_signal_config.get('mode', 'leader').lower()
|
||||
]
|
||||
|
||||
self.leaders_list = self.external_signal_config.get('leaders', [])
|
||||
self.push_throttle_secs = self.external_signal_config.get('push_throttle_secs', 0.1)
|
||||
|
||||
self.reply_timeout = self.external_signal_config.get('follower_reply_timeout', 10)
|
||||
self.ping_timeout = self.external_signal_config.get('follower_ping_timeout', 2)
|
||||
self.sleep_time = self.external_signal_config.get('follower_sleep_time', 5)
|
||||
|
||||
if self.mode == ExternalSignalModeType.follower and len(self.leaders_list) == 0:
|
||||
raise ValueError("You must specify at least 1 leader in follower mode.")
|
||||
|
||||
# This is only used by the leader, the followers use the tokens specified
|
||||
# in each of the leaders
|
||||
# If you do not specify an API key in the config, one will be randomly
|
||||
# generated and logged on startup
|
||||
default_api_key = secrets.token_urlsafe(16)
|
||||
self.secret_api_key = self.external_signal_config.get('api_token', default_api_key)
|
||||
|
||||
self.start_threaded_loop()
|
||||
self.start()
|
||||
|
||||
def is_leader(self):
|
||||
"""
|
||||
Leader flag
|
||||
"""
|
||||
return self.enabled() and self.mode == ExternalSignalModeType.leader
|
||||
|
||||
def enabled(self):
|
||||
"""
|
||||
Enabled flag
|
||||
"""
|
||||
return self.external_signal_config.get('enabled', False)
|
||||
|
||||
def start_threaded_loop(self):
|
||||
"""
|
||||
Start the main internal loop in another thread to run coroutines
|
||||
"""
|
||||
self._loop = asyncio.new_event_loop()
|
||||
|
||||
if not self._thread:
|
||||
self._thread = Thread(target=self._loop.run_forever)
|
||||
self._thread.start()
|
||||
self._running = True
|
||||
else:
|
||||
raise RuntimeError("A loop is already running")
|
||||
|
||||
def submit_coroutine(self, coroutine: Coroutine):
|
||||
"""
|
||||
Submit a coroutine to the threaded loop
|
||||
"""
|
||||
if not self._running:
|
||||
raise RuntimeError("Cannot schedule new futures after shutdown")
|
||||
|
||||
if not self._loop or not self._loop.is_running():
|
||||
raise RuntimeError("Loop must be started before any function can"
|
||||
" be submitted")
|
||||
|
||||
return asyncio.run_coroutine_threadsafe(coroutine, self._loop)
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
Start the controller main loop
|
||||
"""
|
||||
self._main_task = self.submit_coroutine(self.main())
|
||||
|
||||
async def shutdown(self):
|
||||
"""
|
||||
Shutdown all tasks and close up
|
||||
"""
|
||||
logger.info("Stopping rpc.externalsignalcontroller")
|
||||
|
||||
# Flip running flag
|
||||
self._running = False
|
||||
|
||||
# Cancel sub tasks
|
||||
for task in self._sub_tasks:
|
||||
task.cancel()
|
||||
|
||||
# Then disconnect all channels
|
||||
await self.channel_manager.disconnect_all()
|
||||
|
||||
def cleanup(self) -> None:
|
||||
"""
|
||||
Cleanup pending module resources.
|
||||
"""
|
||||
if self._thread:
|
||||
if self._loop.is_running():
|
||||
self._main_task.cancel()
|
||||
self._thread.join()
|
||||
|
||||
async def main(self):
|
||||
"""
|
||||
Main coro
|
||||
|
||||
Start the loop based on what mode we're in
|
||||
"""
|
||||
try:
|
||||
if self.mode == ExternalSignalModeType.leader:
|
||||
logger.info("Starting rpc.externalsignalcontroller in Leader mode")
|
||||
|
||||
await self.run_leader_mode()
|
||||
elif self.mode == ExternalSignalModeType.follower:
|
||||
logger.info("Starting rpc.externalsignalcontroller in Follower mode")
|
||||
|
||||
await self.run_follower_mode()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
# We're cancelled
|
||||
await self.shutdown()
|
||||
except Exception as e:
|
||||
# Log the error
|
||||
logger.error(f"Exception occurred in main task: {e}")
|
||||
logger.exception(e)
|
||||
finally:
|
||||
# This coroutine is the last thing to be ended, so it should stop the loop
|
||||
self._loop.stop()
|
||||
|
||||
def log_api_token(self):
|
||||
"""
|
||||
Log the API token
|
||||
"""
|
||||
logger.info("-" * 15)
|
||||
logger.info(f"API_KEY: {self.secret_api_key}")
|
||||
logger.info("-" * 15)
|
||||
|
||||
def send_msg(self, msg: MessageType) -> None:
|
||||
"""
|
||||
Support RPC calls
|
||||
"""
|
||||
if msg["type"] == RPCMessageType.EMIT_DATA:
|
||||
message = msg.get("message")
|
||||
if message:
|
||||
self.send_message(message)
|
||||
else:
|
||||
logger.error(f"Message is empty! {msg}")
|
||||
|
||||
def send_message(self, msg: MessageType) -> None:
|
||||
"""
|
||||
Broadcast message over all channels if there are any
|
||||
"""
|
||||
|
||||
if self.channel_manager.has_channels():
|
||||
self._send_message(msg)
|
||||
else:
|
||||
logger.debug("No listening followers, skipping...")
|
||||
pass
|
||||
|
||||
def _send_message(self, msg: MessageType):
|
||||
"""
|
||||
Add data to the internal queue to be broadcasted. This func will block
|
||||
if the queue is full. This is meant to be called in the main thread.
|
||||
"""
|
||||
if self._queue:
|
||||
queue = self._queue.sync_q
|
||||
queue.put(msg) # This will block if the queue is full
|
||||
else:
|
||||
logger.warning("Can not send data, leader loop has not started yet!")
|
||||
|
||||
async def send_initial_data(self, channel):
|
||||
logger.info("Sending initial data through channel")
|
||||
|
||||
# We first send pairlist data
|
||||
# We should move this to a func in the RPC object
|
||||
initial_data = {
|
||||
"data_type": LeaderMessageType.pairlist,
|
||||
"data": self.freqtrade.pairlists.whitelist
|
||||
}
|
||||
|
||||
await channel.send(initial_data)
|
||||
|
||||
async def _handle_leader_message(self, message: MessageType):
|
||||
"""
|
||||
Handle message received from a Leader
|
||||
"""
|
||||
type = message.get("data_type")
|
||||
data = message.get("data")
|
||||
|
||||
self._rpc._handle_emitted_data(type, data)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
async def run_leader_mode(self):
|
||||
"""
|
||||
Main leader coroutine
|
||||
|
||||
This starts all of the leader coros and registers the endpoint on
|
||||
the ApiServer
|
||||
"""
|
||||
self.register_leader_endpoint()
|
||||
self.log_api_token()
|
||||
|
||||
self._sub_tasks = [
|
||||
self._loop.create_task(self._broadcast_queue_data())
|
||||
]
|
||||
|
||||
return await asyncio.gather(*self._sub_tasks)
|
||||
|
||||
async def run_follower_mode(self):
|
||||
"""
|
||||
Main follower coroutine
|
||||
|
||||
This starts all of the follower connection coros
|
||||
"""
|
||||
|
||||
rpc_lock = asyncio.Lock()
|
||||
|
||||
self._sub_tasks = [
|
||||
self._loop.create_task(self._handle_leader_connection(leader, rpc_lock))
|
||||
for leader in self.leaders_list
|
||||
]
|
||||
|
||||
return await asyncio.gather(*self._sub_tasks)
|
||||
|
||||
async def _broadcast_queue_data(self):
|
||||
"""
|
||||
Loop over queue data and broadcast it
|
||||
"""
|
||||
# Instantiate the queue in this coroutine so it's attached to our loop
|
||||
self._queue = ThreadedQueue()
|
||||
async_queue = self._queue.async_q
|
||||
|
||||
try:
|
||||
while self._running:
|
||||
# Get data from queue
|
||||
data = await async_queue.get()
|
||||
|
||||
# Broadcast it to everyone
|
||||
await self.channel_manager.broadcast(data)
|
||||
|
||||
# Sleep
|
||||
await asyncio.sleep(self.push_throttle_secs)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
# Silently stop
|
||||
pass
|
||||
|
||||
async def get_api_token(
|
||||
self,
|
||||
websocket: FastAPIWebSocket,
|
||||
token: Union[str, None] = None
|
||||
):
|
||||
"""
|
||||
Extract the API key from query param. Must match the
|
||||
set secret_api_key or the websocket connection will be closed.
|
||||
"""
|
||||
if token == self.secret_api_key:
|
||||
return token
|
||||
else:
|
||||
logger.info("Denying websocket request...")
|
||||
await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
|
||||
|
||||
def register_leader_endpoint(self, path: str = "/signals/ws"):
|
||||
"""
|
||||
Attach and start the main leader loop to the ApiServer
|
||||
|
||||
:param path: The endpoint path
|
||||
"""
|
||||
if not self.api_server:
|
||||
raise RuntimeError("The leader needs the ApiServer to be active")
|
||||
|
||||
# The endpoint function for running the main leader loop
|
||||
@self.api_server.app.websocket(path)
|
||||
async def leader_endpoint(
|
||||
websocket: FastAPIWebSocket,
|
||||
api_key: str = Depends(self.get_api_token)
|
||||
):
|
||||
await self.leader_endpoint_loop(websocket)
|
||||
|
||||
async def leader_endpoint_loop(self, websocket: FastAPIWebSocket):
|
||||
"""
|
||||
The WebSocket endpoint served by the ApiServer. This handles connections,
|
||||
and adding them to the channel manager.
|
||||
"""
|
||||
try:
|
||||
if is_websocket_alive(websocket):
|
||||
logger.info(f"Follower connected - {websocket.client}")
|
||||
channel = await self.channel_manager.on_connect(websocket)
|
||||
|
||||
# Send initial data here
|
||||
# Data is being broadcasted right away as soon as startup,
|
||||
# we may not have to send initial data at all. Further testing
|
||||
# required.
|
||||
|
||||
await self.send_initial_data(channel)
|
||||
|
||||
# Keep connection open until explicitly closed, and sleep
|
||||
try:
|
||||
while not channel.is_closed():
|
||||
request = await channel.recv()
|
||||
logger.info(f"Follower request - {request}")
|
||||
|
||||
except WebSocketDisconnect:
|
||||
# Handle client disconnects
|
||||
logger.info(f"Follower disconnected - {websocket.client}")
|
||||
await self.channel_manager.on_disconnect(websocket)
|
||||
except Exception as e:
|
||||
logger.info(f"Follower connection failed - {websocket.client}")
|
||||
logger.exception(e)
|
||||
# Handle cases like -
|
||||
# RuntimeError('Cannot call "send" once a closed message has been sent')
|
||||
await self.channel_manager.on_disconnect(websocket)
|
||||
|
||||
except Exception:
|
||||
logger.error(f"Failed to serve - {websocket.client}")
|
||||
await self.channel_manager.on_disconnect(websocket)
|
||||
|
||||
async def _handle_leader_connection(self, leader, lock):
|
||||
"""
|
||||
Given a leader, connect and wait on data. If connection is lost,
|
||||
it will attempt to reconnect.
|
||||
"""
|
||||
try:
|
||||
url, token = leader["url"], leader["api_token"]
|
||||
websocket_url = f"{url}?token={token}"
|
||||
|
||||
logger.info(f"Attempting to connect to Leader at: {url}")
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(websocket_url) as ws:
|
||||
channel = await self.channel_manager.on_connect(ws)
|
||||
logger.info(f"Connection to Leader at {url} successful")
|
||||
while True:
|
||||
try:
|
||||
data = await asyncio.wait_for(
|
||||
channel.recv(),
|
||||
timeout=self.reply_timeout
|
||||
)
|
||||
except (asyncio.TimeoutError, websockets.exceptions.ConnectionClosed):
|
||||
# We haven't received data yet. Check the connection and continue.
|
||||
try:
|
||||
# ping
|
||||
ping = await channel.ping()
|
||||
await asyncio.wait_for(ping, timeout=self.ping_timeout)
|
||||
logger.debug(f"Connection to {url} still alive...")
|
||||
continue
|
||||
except Exception:
|
||||
logger.info(
|
||||
f"Ping error {url} - retrying in {self.sleep_time}s")
|
||||
asyncio.sleep(self.sleep_time)
|
||||
break
|
||||
|
||||
async with lock:
|
||||
# Acquire lock so only 1 coro handling at a time
|
||||
# as we might call the RPC module in the main thread
|
||||
await self._handle_leader_message(data)
|
||||
|
||||
except socket.gaierror:
|
||||
logger.info(f"Socket error - retrying connection in {self.sleep_time}s")
|
||||
await asyncio.sleep(self.sleep_time)
|
||||
continue
|
||||
except ConnectionRefusedError:
|
||||
logger.info(f"Connection Refused - retrying connection in {self.sleep_time}s")
|
||||
await asyncio.sleep(self.sleep_time)
|
||||
continue
|
||||
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
61
freqtrade/rpc/external_signal/proxy.py
Normal file
61
freqtrade/rpc/external_signal/proxy.py
Normal file
@@ -0,0 +1,61 @@
|
||||
from typing import Union
|
||||
|
||||
from fastapi import WebSocket as FastAPIWebSocket
|
||||
from websockets import WebSocketClientProtocol as WebSocket
|
||||
|
||||
from freqtrade.rpc.external_signal.types import WebSocketType
|
||||
|
||||
|
||||
class WebSocketProxy:
|
||||
"""
|
||||
WebSocketProxy object to bring the FastAPIWebSocket and websockets.WebSocketClientProtocol
|
||||
under the same API
|
||||
"""
|
||||
|
||||
def __init__(self, websocket: WebSocketType):
|
||||
self._websocket: Union[FastAPIWebSocket, WebSocket] = websocket
|
||||
|
||||
async def send(self, data):
|
||||
"""
|
||||
Send data on the wrapped websocket
|
||||
"""
|
||||
if isinstance(data, str):
|
||||
data = data.encode()
|
||||
|
||||
if hasattr(self._websocket, "send_bytes"):
|
||||
await self._websocket.send_bytes(data)
|
||||
else:
|
||||
await self._websocket.send(data)
|
||||
|
||||
async def recv(self):
|
||||
"""
|
||||
Receive data on the wrapped websocket
|
||||
"""
|
||||
if hasattr(self._websocket, "receive_bytes"):
|
||||
return await self._websocket.receive_bytes()
|
||||
else:
|
||||
return await self._websocket.recv()
|
||||
|
||||
async def ping(self):
|
||||
"""
|
||||
Ping the websocket, not supported by FastAPI WebSockets
|
||||
"""
|
||||
if hasattr(self._websocket, "ping"):
|
||||
return await self._websocket.ping()
|
||||
return False
|
||||
|
||||
async def close(self, code: int = 1000):
|
||||
"""
|
||||
Close the websocket connection, only supported by FastAPI WebSockets
|
||||
"""
|
||||
if hasattr(self._websocket, "close"):
|
||||
return await self._websocket.close(code)
|
||||
pass
|
||||
|
||||
async def accept(self):
|
||||
"""
|
||||
Accept the WebSocket connection, only support by FastAPI WebSockets
|
||||
"""
|
||||
if hasattr(self._websocket, "accept"):
|
||||
return await self._websocket.accept()
|
||||
pass
|
||||
65
freqtrade/rpc/external_signal/serializer.py
Normal file
65
freqtrade/rpc/external_signal/serializer.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import json
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import msgpack
|
||||
import orjson
|
||||
|
||||
from freqtrade.rpc.external_signal.proxy import WebSocketProxy
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WebSocketSerializer(ABC):
|
||||
def __init__(self, websocket: WebSocketProxy):
|
||||
self._websocket: WebSocketProxy = websocket
|
||||
|
||||
@abstractmethod
|
||||
def _serialize(self, data):
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def _deserialize(self, data):
|
||||
raise NotImplementedError()
|
||||
|
||||
async def send(self, data: bytes):
|
||||
await self._websocket.send(self._serialize(data))
|
||||
|
||||
async def recv(self) -> bytes:
|
||||
data = await self._websocket.recv()
|
||||
|
||||
return self._deserialize(data)
|
||||
|
||||
async def close(self, code: int = 1000):
|
||||
await self._websocket.close(code)
|
||||
|
||||
# Going to explore using MsgPack as the serialization,
|
||||
# as that might be the best method for sending pandas
|
||||
# dataframes over the wire
|
||||
|
||||
|
||||
class JSONWebSocketSerializer(WebSocketSerializer):
|
||||
def _serialize(self, data):
|
||||
return json.dumps(data)
|
||||
|
||||
def _deserialize(self, data):
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
class ORJSONWebSocketSerializer(WebSocketSerializer):
|
||||
ORJSON_OPTIONS = orjson.OPT_NAIVE_UTC | orjson.OPT_SERIALIZE_NUMPY
|
||||
|
||||
def _serialize(self, data):
|
||||
return orjson.dumps(data, option=self.ORJSON_OPTIONS)
|
||||
|
||||
def _deserialize(self, data):
|
||||
return orjson.loads(data, option=self.ORJSON_OPTIONS)
|
||||
|
||||
|
||||
class MsgPackWebSocketSerializer(WebSocketSerializer):
|
||||
def _serialize(self, data):
|
||||
return msgpack.packb(data, use_bin_type=True)
|
||||
|
||||
def _deserialize(self, data):
|
||||
return msgpack.unpackb(data, raw=False)
|
||||
650
freqtrade/rpc/external_signal/thread_queue.py
Normal file
650
freqtrade/rpc/external_signal/thread_queue.py
Normal file
@@ -0,0 +1,650 @@
|
||||
import asyncio
|
||||
import sys
|
||||
import threading
|
||||
from asyncio import QueueEmpty as AsyncQueueEmpty
|
||||
from asyncio import QueueFull as AsyncQueueFull
|
||||
from collections import deque
|
||||
from heapq import heappop, heappush
|
||||
from queue import Empty as SyncQueueEmpty
|
||||
from queue import Full as SyncQueueFull
|
||||
from typing import Any, Callable, Deque, Generic, List, Optional, Set, TypeVar
|
||||
|
||||
from typing_extensions import Protocol
|
||||
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__all__ = (
|
||||
"Queue",
|
||||
"PriorityQueue",
|
||||
"LifoQueue",
|
||||
"SyncQueue",
|
||||
"AsyncQueue",
|
||||
"BaseQueue",
|
||||
)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
OptFloat = Optional[float]
|
||||
|
||||
|
||||
class BaseQueue(Protocol[T]):
|
||||
@property
|
||||
def maxsize(self) -> int:
|
||||
...
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
...
|
||||
|
||||
def task_done(self) -> None:
|
||||
...
|
||||
|
||||
def qsize(self) -> int:
|
||||
...
|
||||
|
||||
@property
|
||||
def unfinished_tasks(self) -> int:
|
||||
...
|
||||
|
||||
def empty(self) -> bool:
|
||||
...
|
||||
|
||||
def full(self) -> bool:
|
||||
...
|
||||
|
||||
def put_nowait(self, item: T) -> None:
|
||||
...
|
||||
|
||||
def get_nowait(self) -> T:
|
||||
...
|
||||
|
||||
|
||||
class SyncQueue(BaseQueue[T], Protocol[T]):
|
||||
@property
|
||||
def maxsize(self) -> int:
|
||||
...
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
...
|
||||
|
||||
def task_done(self) -> None:
|
||||
...
|
||||
|
||||
def qsize(self) -> int:
|
||||
...
|
||||
|
||||
@property
|
||||
def unfinished_tasks(self) -> int:
|
||||
...
|
||||
|
||||
def empty(self) -> bool:
|
||||
...
|
||||
|
||||
def full(self) -> bool:
|
||||
...
|
||||
|
||||
def put_nowait(self, item: T) -> None:
|
||||
...
|
||||
|
||||
def get_nowait(self) -> T:
|
||||
...
|
||||
|
||||
def put(self, item: T, block: bool = True, timeout: OptFloat = None) -> None:
|
||||
...
|
||||
|
||||
def get(self, block: bool = True, timeout: OptFloat = None) -> T:
|
||||
...
|
||||
|
||||
def join(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
class AsyncQueue(BaseQueue[T], Protocol[T]):
|
||||
async def put(self, item: T) -> None:
|
||||
...
|
||||
|
||||
async def get(self) -> T:
|
||||
...
|
||||
|
||||
async def join(self) -> None:
|
||||
...
|
||||
|
||||
|
||||
class Queue(Generic[T]):
|
||||
def __init__(self, maxsize: int = 0) -> None:
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._maxsize = maxsize
|
||||
|
||||
self._init(maxsize)
|
||||
|
||||
self._unfinished_tasks = 0
|
||||
|
||||
self._sync_mutex = threading.Lock()
|
||||
self._sync_not_empty = threading.Condition(self._sync_mutex)
|
||||
self._sync_not_full = threading.Condition(self._sync_mutex)
|
||||
self._all_tasks_done = threading.Condition(self._sync_mutex)
|
||||
|
||||
self._async_mutex = asyncio.Lock()
|
||||
if sys.version_info[:3] == (3, 10, 0):
|
||||
# Workaround for Python 3.10 bug, see #358:
|
||||
getattr(self._async_mutex, "_get_loop", lambda: None)()
|
||||
self._async_not_empty = asyncio.Condition(self._async_mutex)
|
||||
self._async_not_full = asyncio.Condition(self._async_mutex)
|
||||
self._finished = asyncio.Event()
|
||||
self._finished.set()
|
||||
|
||||
self._closing = False
|
||||
self._pending = set() # type: Set[asyncio.Future[Any]]
|
||||
|
||||
def checked_call_soon_threadsafe(
|
||||
callback: Callable[..., None], *args: Any
|
||||
) -> None:
|
||||
try:
|
||||
self._loop.call_soon_threadsafe(callback, *args)
|
||||
except RuntimeError:
|
||||
# swallowing agreed in #2
|
||||
pass
|
||||
|
||||
self._call_soon_threadsafe = checked_call_soon_threadsafe
|
||||
|
||||
def checked_call_soon(callback: Callable[..., None], *args: Any) -> None:
|
||||
if not self._loop.is_closed():
|
||||
self._loop.call_soon(callback, *args)
|
||||
|
||||
self._call_soon = checked_call_soon
|
||||
|
||||
self._sync_queue = _SyncQueueProxy(self)
|
||||
self._async_queue = _AsyncQueueProxy(self)
|
||||
|
||||
def close(self) -> None:
|
||||
with self._sync_mutex:
|
||||
self._closing = True
|
||||
for fut in self._pending:
|
||||
fut.cancel()
|
||||
self._finished.set() # unblocks all async_q.join()
|
||||
self._all_tasks_done.notify_all() # unblocks all sync_q.join()
|
||||
|
||||
async def wait_closed(self) -> None:
|
||||
# should be called from loop after close().
|
||||
# Nobody should put/get at this point,
|
||||
# so lock acquiring is not required
|
||||
if not self._closing:
|
||||
raise RuntimeError("Waiting for non-closed queue")
|
||||
# give execution chances for the task-done callbacks
|
||||
# of async tasks created inside
|
||||
# _notify_async_not_empty, _notify_async_not_full
|
||||
# methods.
|
||||
await asyncio.sleep(0)
|
||||
if not self._pending:
|
||||
return
|
||||
await asyncio.wait(self._pending)
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self._closing and not self._pending
|
||||
|
||||
@property
|
||||
def maxsize(self) -> int:
|
||||
return self._maxsize
|
||||
|
||||
@property
|
||||
def sync_q(self) -> "_SyncQueueProxy[T]":
|
||||
return self._sync_queue
|
||||
|
||||
@property
|
||||
def async_q(self) -> "_AsyncQueueProxy[T]":
|
||||
return self._async_queue
|
||||
|
||||
# Override these methods to implement other queue organizations
|
||||
# (e.g. stack or priority queue).
|
||||
# These will only be called with appropriate locks held
|
||||
|
||||
def _init(self, maxsize: int) -> None:
|
||||
self._queue = deque() # type: Deque[T]
|
||||
|
||||
def _qsize(self) -> int:
|
||||
return len(self._queue)
|
||||
|
||||
# Put a new item in the queue
|
||||
def _put(self, item: T) -> None:
|
||||
self._queue.append(item)
|
||||
|
||||
# Get an item from the queue
|
||||
def _get(self) -> T:
|
||||
return self._queue.popleft()
|
||||
|
||||
def _put_internal(self, item: T) -> None:
|
||||
self._put(item)
|
||||
self._unfinished_tasks += 1
|
||||
self._finished.clear()
|
||||
|
||||
def _notify_sync_not_empty(self) -> None:
|
||||
def f() -> None:
|
||||
with self._sync_mutex:
|
||||
self._sync_not_empty.notify()
|
||||
|
||||
self._loop.run_in_executor(None, f)
|
||||
|
||||
def _notify_sync_not_full(self) -> None:
|
||||
def f() -> None:
|
||||
with self._sync_mutex:
|
||||
self._sync_not_full.notify()
|
||||
|
||||
fut = asyncio.ensure_future(self._loop.run_in_executor(None, f))
|
||||
fut.add_done_callback(self._pending.discard)
|
||||
self._pending.add(fut)
|
||||
|
||||
def _notify_async_not_empty(self, *, threadsafe: bool) -> None:
|
||||
async def f() -> None:
|
||||
async with self._async_mutex:
|
||||
self._async_not_empty.notify()
|
||||
|
||||
def task_maker() -> None:
|
||||
task = self._loop.create_task(f())
|
||||
task.add_done_callback(self._pending.discard)
|
||||
self._pending.add(task)
|
||||
|
||||
if threadsafe:
|
||||
self._call_soon_threadsafe(task_maker)
|
||||
else:
|
||||
self._call_soon(task_maker)
|
||||
|
||||
def _notify_async_not_full(self, *, threadsafe: bool) -> None:
|
||||
async def f() -> None:
|
||||
async with self._async_mutex:
|
||||
self._async_not_full.notify()
|
||||
|
||||
def task_maker() -> None:
|
||||
task = self._loop.create_task(f())
|
||||
task.add_done_callback(self._pending.discard)
|
||||
self._pending.add(task)
|
||||
|
||||
if threadsafe:
|
||||
self._call_soon_threadsafe(task_maker)
|
||||
else:
|
||||
self._call_soon(task_maker)
|
||||
|
||||
def _check_closing(self) -> None:
|
||||
if self._closing:
|
||||
raise RuntimeError("Operation on the closed queue is forbidden")
|
||||
|
||||
|
||||
class _SyncQueueProxy(SyncQueue[T]):
|
||||
"""Create a queue object with a given maximum size.
|
||||
|
||||
If maxsize is <= 0, the queue size is infinite.
|
||||
"""
|
||||
|
||||
def __init__(self, parent: Queue[T]):
|
||||
self._parent = parent
|
||||
|
||||
@property
|
||||
def maxsize(self) -> int:
|
||||
return self._parent._maxsize
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self._parent.closed
|
||||
|
||||
def task_done(self) -> None:
|
||||
"""Indicate that a formerly enqueued task is complete.
|
||||
|
||||
Used by Queue consumer threads. For each get() used to fetch a task,
|
||||
a subsequent call to task_done() tells the queue that the processing
|
||||
on the task is complete.
|
||||
|
||||
If a join() is currently blocking, it will resume when all items
|
||||
have been processed (meaning that a task_done() call was received
|
||||
for every item that had been put() into the queue).
|
||||
|
||||
Raises a ValueError if called more times than there were items
|
||||
placed in the queue.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._all_tasks_done:
|
||||
unfinished = self._parent._unfinished_tasks - 1
|
||||
if unfinished <= 0:
|
||||
if unfinished < 0:
|
||||
raise ValueError("task_done() called too many times")
|
||||
self._parent._all_tasks_done.notify_all()
|
||||
self._parent._loop.call_soon_threadsafe(self._parent._finished.set)
|
||||
self._parent._unfinished_tasks = unfinished
|
||||
|
||||
def join(self) -> None:
|
||||
"""Blocks until all items in the Queue have been gotten and processed.
|
||||
|
||||
The count of unfinished tasks goes up whenever an item is added to the
|
||||
queue. The count goes down whenever a consumer thread calls task_done()
|
||||
to indicate the item was retrieved and all work on it is complete.
|
||||
|
||||
When the count of unfinished tasks drops to zero, join() unblocks.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._all_tasks_done:
|
||||
while self._parent._unfinished_tasks:
|
||||
self._parent._all_tasks_done.wait()
|
||||
self._parent._check_closing()
|
||||
|
||||
def qsize(self) -> int:
|
||||
"""Return the approximate size of the queue (not reliable!)."""
|
||||
return self._parent._qsize()
|
||||
|
||||
@property
|
||||
def unfinished_tasks(self) -> int:
|
||||
"""Return the number of unfinished tasks."""
|
||||
return self._parent._unfinished_tasks
|
||||
|
||||
def empty(self) -> bool:
|
||||
"""Return True if the queue is empty, False otherwise (not reliable!).
|
||||
|
||||
This method is likely to be removed at some point. Use qsize() == 0
|
||||
as a direct substitute, but be aware that either approach risks a race
|
||||
condition where a queue can grow before the result of empty() or
|
||||
qsize() can be used.
|
||||
|
||||
To create code that needs to wait for all queued tasks to be
|
||||
completed, the preferred technique is to use the join() method.
|
||||
"""
|
||||
return not self._parent._qsize()
|
||||
|
||||
def full(self) -> bool:
|
||||
"""Return True if the queue is full, False otherwise (not reliable!).
|
||||
|
||||
This method is likely to be removed at some point. Use qsize() >= n
|
||||
as a direct substitute, but be aware that either approach risks a race
|
||||
condition where a queue can shrink before the result of full() or
|
||||
qsize() can be used.
|
||||
"""
|
||||
return 0 < self._parent._maxsize <= self._parent._qsize()
|
||||
|
||||
def put(self, item: T, block: bool = True, timeout: OptFloat = None) -> None:
|
||||
"""Put an item into the queue.
|
||||
|
||||
If optional args 'block' is true and 'timeout' is None (the default),
|
||||
block if necessary until a free slot is available. If 'timeout' is
|
||||
a non-negative number, it blocks at most 'timeout' seconds and raises
|
||||
the Full exception if no free slot was available within that time.
|
||||
Otherwise ('block' is false), put an item on the queue if a free slot
|
||||
is immediately available, else raise the Full exception ('timeout'
|
||||
is ignored in that case).
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._sync_not_full:
|
||||
if self._parent._maxsize > 0:
|
||||
if not block:
|
||||
if self._parent._qsize() >= self._parent._maxsize:
|
||||
raise SyncQueueFull
|
||||
elif timeout is None:
|
||||
while self._parent._qsize() >= self._parent._maxsize:
|
||||
self._parent._sync_not_full.wait()
|
||||
elif timeout < 0:
|
||||
raise ValueError("'timeout' must be a non-negative number")
|
||||
else:
|
||||
time = self._parent._loop.time
|
||||
endtime = time() + timeout
|
||||
while self._parent._qsize() >= self._parent._maxsize:
|
||||
remaining = endtime - time()
|
||||
if remaining <= 0.0:
|
||||
raise SyncQueueFull
|
||||
self._parent._sync_not_full.wait(remaining)
|
||||
self._parent._put_internal(item)
|
||||
self._parent._sync_not_empty.notify()
|
||||
self._parent._notify_async_not_empty(threadsafe=True)
|
||||
|
||||
def get(self, block: bool = True, timeout: OptFloat = None) -> T:
|
||||
"""Remove and return an item from the queue.
|
||||
|
||||
If optional args 'block' is true and 'timeout' is None (the default),
|
||||
block if necessary until an item is available. If 'timeout' is
|
||||
a non-negative number, it blocks at most 'timeout' seconds and raises
|
||||
the Empty exception if no item was available within that time.
|
||||
Otherwise ('block' is false), return an item if one is immediately
|
||||
available, else raise the Empty exception ('timeout' is ignored
|
||||
in that case).
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._sync_not_empty:
|
||||
if not block:
|
||||
if not self._parent._qsize():
|
||||
raise SyncQueueEmpty
|
||||
elif timeout is None:
|
||||
while not self._parent._qsize():
|
||||
self._parent._sync_not_empty.wait()
|
||||
elif timeout < 0:
|
||||
raise ValueError("'timeout' must be a non-negative number")
|
||||
else:
|
||||
time = self._parent._loop.time
|
||||
endtime = time() + timeout
|
||||
while not self._parent._qsize():
|
||||
remaining = endtime - time()
|
||||
if remaining <= 0.0:
|
||||
raise SyncQueueEmpty
|
||||
self._parent._sync_not_empty.wait(remaining)
|
||||
item = self._parent._get()
|
||||
self._parent._sync_not_full.notify()
|
||||
self._parent._notify_async_not_full(threadsafe=True)
|
||||
return item
|
||||
|
||||
def put_nowait(self, item: T) -> None:
|
||||
"""Put an item into the queue without blocking.
|
||||
|
||||
Only enqueue the item if a free slot is immediately available.
|
||||
Otherwise raise the Full exception.
|
||||
"""
|
||||
return self.put(item, block=False)
|
||||
|
||||
def get_nowait(self) -> T:
|
||||
"""Remove and return an item from the queue without blocking.
|
||||
|
||||
Only get an item if one is immediately available. Otherwise
|
||||
raise the Empty exception.
|
||||
"""
|
||||
return self.get(block=False)
|
||||
|
||||
|
||||
class _AsyncQueueProxy(AsyncQueue[T]):
|
||||
"""Create a queue object with a given maximum size.
|
||||
|
||||
If maxsize is <= 0, the queue size is infinite.
|
||||
"""
|
||||
|
||||
def __init__(self, parent: Queue[T]):
|
||||
self._parent = parent
|
||||
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self._parent.closed
|
||||
|
||||
def qsize(self) -> int:
|
||||
"""Number of items in the queue."""
|
||||
return self._parent._qsize()
|
||||
|
||||
@property
|
||||
def unfinished_tasks(self) -> int:
|
||||
"""Return the number of unfinished tasks."""
|
||||
return self._parent._unfinished_tasks
|
||||
|
||||
@property
|
||||
def maxsize(self) -> int:
|
||||
"""Number of items allowed in the queue."""
|
||||
return self._parent._maxsize
|
||||
|
||||
def empty(self) -> bool:
|
||||
"""Return True if the queue is empty, False otherwise."""
|
||||
return self.qsize() == 0
|
||||
|
||||
def full(self) -> bool:
|
||||
"""Return True if there are maxsize items in the queue.
|
||||
|
||||
Note: if the Queue was initialized with maxsize=0 (the default),
|
||||
then full() is never True.
|
||||
"""
|
||||
if self._parent._maxsize <= 0:
|
||||
return False
|
||||
else:
|
||||
return self.qsize() >= self._parent._maxsize
|
||||
|
||||
async def put(self, item: T) -> None:
|
||||
"""Put an item into the queue.
|
||||
|
||||
Put an item into the queue. If the queue is full, wait until a free
|
||||
slot is available before adding item.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
async with self._parent._async_not_full:
|
||||
self._parent._sync_mutex.acquire()
|
||||
locked = True
|
||||
try:
|
||||
if self._parent._maxsize > 0:
|
||||
do_wait = True
|
||||
while do_wait:
|
||||
do_wait = self._parent._qsize() >= self._parent._maxsize
|
||||
if do_wait:
|
||||
locked = False
|
||||
self._parent._sync_mutex.release()
|
||||
await self._parent._async_not_full.wait()
|
||||
self._parent._sync_mutex.acquire()
|
||||
locked = True
|
||||
|
||||
self._parent._put_internal(item)
|
||||
self._parent._async_not_empty.notify()
|
||||
self._parent._notify_sync_not_empty()
|
||||
finally:
|
||||
if locked:
|
||||
self._parent._sync_mutex.release()
|
||||
|
||||
def put_nowait(self, item: T) -> None:
|
||||
"""Put an item into the queue without blocking.
|
||||
|
||||
If no free slot is immediately available, raise QueueFull.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._sync_mutex:
|
||||
if self._parent._maxsize > 0:
|
||||
if self._parent._qsize() >= self._parent._maxsize:
|
||||
raise AsyncQueueFull
|
||||
|
||||
self._parent._put_internal(item)
|
||||
self._parent._notify_async_not_empty(threadsafe=False)
|
||||
self._parent._notify_sync_not_empty()
|
||||
|
||||
async def get(self) -> T:
|
||||
"""Remove and return an item from the queue.
|
||||
|
||||
If queue is empty, wait until an item is available.
|
||||
|
||||
This method is a coroutine.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
async with self._parent._async_not_empty:
|
||||
self._parent._sync_mutex.acquire()
|
||||
locked = True
|
||||
try:
|
||||
do_wait = True
|
||||
while do_wait:
|
||||
do_wait = self._parent._qsize() == 0
|
||||
|
||||
if do_wait:
|
||||
locked = False
|
||||
self._parent._sync_mutex.release()
|
||||
await self._parent._async_not_empty.wait()
|
||||
self._parent._sync_mutex.acquire()
|
||||
locked = True
|
||||
|
||||
item = self._parent._get()
|
||||
self._parent._async_not_full.notify()
|
||||
self._parent._notify_sync_not_full()
|
||||
return item
|
||||
finally:
|
||||
if locked:
|
||||
self._parent._sync_mutex.release()
|
||||
|
||||
def get_nowait(self) -> T:
|
||||
"""Remove and return an item from the queue.
|
||||
|
||||
Return an item if one is immediately available, else raise QueueEmpty.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._sync_mutex:
|
||||
if self._parent._qsize() == 0:
|
||||
raise AsyncQueueEmpty
|
||||
|
||||
item = self._parent._get()
|
||||
self._parent._notify_async_not_full(threadsafe=False)
|
||||
self._parent._notify_sync_not_full()
|
||||
return item
|
||||
|
||||
def task_done(self) -> None:
|
||||
"""Indicate that a formerly enqueued task is complete.
|
||||
|
||||
Used by queue consumers. For each get() used to fetch a task,
|
||||
a subsequent call to task_done() tells the queue that the processing
|
||||
on the task is complete.
|
||||
|
||||
If a join() is currently blocking, it will resume when all items have
|
||||
been processed (meaning that a task_done() call was received for every
|
||||
item that had been put() into the queue).
|
||||
|
||||
Raises ValueError if called more times than there were items placed in
|
||||
the queue.
|
||||
"""
|
||||
self._parent._check_closing()
|
||||
with self._parent._all_tasks_done:
|
||||
if self._parent._unfinished_tasks <= 0:
|
||||
raise ValueError("task_done() called too many times")
|
||||
self._parent._unfinished_tasks -= 1
|
||||
if self._parent._unfinished_tasks == 0:
|
||||
self._parent._finished.set()
|
||||
self._parent._all_tasks_done.notify_all()
|
||||
|
||||
async def join(self) -> None:
|
||||
"""Block until all items in the queue have been gotten and processed.
|
||||
|
||||
The count of unfinished tasks goes up whenever an item is added to the
|
||||
queue. The count goes down whenever a consumer calls task_done() to
|
||||
indicate that the item was retrieved and all work on it is complete.
|
||||
When the count of unfinished tasks drops to zero, join() unblocks.
|
||||
"""
|
||||
while True:
|
||||
with self._parent._sync_mutex:
|
||||
self._parent._check_closing()
|
||||
if self._parent._unfinished_tasks == 0:
|
||||
break
|
||||
await self._parent._finished.wait()
|
||||
|
||||
|
||||
class PriorityQueue(Queue[T]):
|
||||
"""Variant of Queue that retrieves open entries in priority order
|
||||
(lowest first).
|
||||
|
||||
Entries are typically tuples of the form: (priority number, data).
|
||||
|
||||
"""
|
||||
|
||||
def _init(self, maxsize: int) -> None:
|
||||
self._heap_queue = [] # type: List[T]
|
||||
|
||||
def _qsize(self) -> int:
|
||||
return len(self._heap_queue)
|
||||
|
||||
def _put(self, item: T) -> None:
|
||||
heappush(self._heap_queue, item)
|
||||
|
||||
def _get(self) -> T:
|
||||
return heappop(self._heap_queue)
|
||||
|
||||
|
||||
class LifoQueue(Queue[T]):
|
||||
"""Variant of Queue that retrieves most recently added entries first."""
|
||||
|
||||
def _qsize(self) -> int:
|
||||
return len(self._queue)
|
||||
|
||||
def _put(self, item: T) -> None:
|
||||
self._queue.append(item)
|
||||
|
||||
def _get(self) -> T:
|
||||
return self._queue.pop()
|
||||
8
freqtrade/rpc/external_signal/types.py
Normal file
8
freqtrade/rpc/external_signal/types.py
Normal file
@@ -0,0 +1,8 @@
|
||||
from typing import Any, Dict, TypeVar
|
||||
|
||||
from fastapi import WebSocket as FastAPIWebSocket
|
||||
from websockets import WebSocketClientProtocol as WebSocket
|
||||
|
||||
|
||||
WebSocketType = TypeVar("WebSocketType", FastAPIWebSocket, WebSocket)
|
||||
MessageType = Dict[str, Any]
|
||||
10
freqtrade/rpc/external_signal/utils.py
Normal file
10
freqtrade/rpc/external_signal/utils.py
Normal file
@@ -0,0 +1,10 @@
|
||||
from starlette.websockets import WebSocket, WebSocketState
|
||||
|
||||
|
||||
async def is_websocket_alive(ws: WebSocket) -> bool:
|
||||
if (
|
||||
ws.application_state == WebSocketState.CONNECTED and
|
||||
ws.client_state == WebSocketState.CONNECTED
|
||||
):
|
||||
return True
|
||||
return False
|
||||
Reference in New Issue
Block a user