adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring, minor improvements, data provider improvements

Browse Source
This commit is contained in:
Timothy Pogue
2022-08-26 23:40:13 -06:00
parent a998d6d773
commit 2b5f067877
14 changed files with 218 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/rpc/api_server/webserver.py
View File

@@ -74,6 +74,7 @@ class ApiServer(RPCHandler):
default_response_class=FTJSONResponse,
)
self.configure_app(self.app, self._config)
self.start_api()
def add_rpc_handler(self, rpc: RPC):
"""

40
freqtrade/rpc/external_signal/channel.py
View File

@@ -1,4 +1,5 @@
import logging
from threading import RLock
from typing import Type
from freqtrade.rpc.external_signal.proxy import WebSocketProxy
@@ -63,6 +64,7 @@ class WebSocketChannel:
class ChannelManager:
def __init__(self):
self.channels = dict()
self._lock = RLock() # Re-entrant Lock
async def on_connect(self, websocket: WebSocketType):
"""
@@ -78,7 +80,9 @@ class ChannelManager:
return
ws_channel = WebSocketChannel(websocket)
self.channels[websocket] = ws_channel
with self._lock:
self.channels[websocket] = ws_channel
return ws_channel
@@ -88,21 +92,26 @@ class ChannelManager:
:param websocket: The WebSocket objet attached to the Channel
"""
if websocket in self.channels.keys():
channel = self.channels[websocket]
with self._lock:
channel = self.channels.get(websocket)
if channel:
logger.debug(f"Disconnecting channel - {channel}")
logger.debug(f"Disconnecting channel - {channel}")
if not channel.is_closed():
await channel.close()
if not channel.is_closed():
await channel.close()
del self.channels[websocket]
del self.channels[websocket]
async def disconnect_all(self):
"""
Disconnect all Channels
"""
for websocket in self.channels.keys():
await self.on_disconnect(websocket)
with self._lock:
for websocket, channel in self.channels.items():
if not channel.is_closed():
await channel.close()
self.channels = dict()
async def broadcast(self, data):
"""
@@ -110,12 +119,13 @@ class ChannelManager:
:param data: The data to send
"""
for websocket, channel in self.channels.items():
try:
await channel.send(data)
except RuntimeError:
# Handle cannot send after close cases
await self.on_disconnect(websocket)
with self._lock:
for websocket, channel in self.channels.items():
try:
await channel.send(data)
except RuntimeError:
# Handle cannot send after close cases
await self.on_disconnect(websocket)
async def send_direct(self, channel, data):
"""

34
freqtrade/rpc/external_signal/controller.py
View File

@@ -6,7 +6,7 @@ import logging
import secrets
import socket
from threading import Thread
from typing import Any, Coroutine, Dict, Union
from typing import Any, Callable, Coroutine, Dict, Union
import websockets
from fastapi import Depends
@@ -56,8 +56,13 @@ class ExternalSignalController(RPCHandler):
self._main_task = None
self._sub_tasks = None
self.channel_manager = ChannelManager()
self._message_handlers = {
LeaderMessageType.pairlist: self._rpc._handle_pairlist_message,
LeaderMessageType.analyzed_df: self._rpc._handle_analyzed_df_message,
LeaderMessageType.default: self._rpc._handle_default_message
}
self.channel_manager = ChannelManager()
self.external_signal_config = config.get('external_signal', {})
# What the config should look like
@@ -89,6 +94,8 @@ class ExternalSignalController(RPCHandler):
self.ping_timeout = self.external_signal_config.get('follower_ping_timeout', 2)
self.sleep_time = self.external_signal_config.get('follower_sleep_time', 5)
# Validate external_signal_config here?
if self.mode == ExternalSignalModeType.follower and len(self.leaders_list) == 0:
raise ValueError("You must specify at least 1 leader in follower mode.")
@@ -99,7 +106,6 @@ class ExternalSignalController(RPCHandler):
default_api_key = secrets.token_urlsafe(16)
self.secret_api_key = self.external_signal_config.get('api_token', default_api_key)
self.start_threaded_loop()
self.start()
def is_leader(self):
@@ -114,6 +120,12 @@ class ExternalSignalController(RPCHandler):
"""
return self.external_signal_config.get('enabled', False)
def num_leaders(self):
"""
The number of leaders we should be connected to
"""
return len(self.leaders_list)
def start_threaded_loop(self):
"""
Start the main internal loop in another thread to run coroutines
@@ -144,6 +156,7 @@ class ExternalSignalController(RPCHandler):
"""
Start the controller main loop
"""
self.start_threaded_loop()
self._main_task = self.submit_coroutine(self.main())
async def shutdown(self):
@@ -242,23 +255,20 @@ class ExternalSignalController(RPCHandler):
async def send_initial_data(self, channel):
logger.info("Sending initial data through channel")
# We first send pairlist data
# We should move this to a func in the RPC object
initial_data = {
"data_type": LeaderMessageType.pairlist,
"data": self.freqtrade.pairlists.whitelist
}
data = self._rpc._initial_leader_data()
await channel.send(initial_data)
for message in data:
await channel.send(message)
async def _handle_leader_message(self, message: MessageType):
"""
Handle message received from a Leader
"""
type = message.get("data_type")
type = message.get("data_type", LeaderMessageType.default)
data = message.get("data")
self._rpc._handle_emitted_data(type, data)
handler: Callable = self._message_handlers[type]
handler(type, data)
# ----------------------------------------------------------------------

12
freqtrade/rpc/external_signal/utils.py
View File

@@ -1,5 +1,8 @@
from pandas import DataFrame
from starlette.websockets import WebSocket, WebSocketState
from freqtrade.enums.signaltype import SignalTagType, SignalType
async def is_websocket_alive(ws: WebSocket) -> bool:
if (
@@ -8,3 +11,12 @@ async def is_websocket_alive(ws: WebSocket) -> bool:
):
return True
return False
def remove_entry_exit_signals(dataframe: DataFrame):
dataframe[SignalType.ENTER_LONG.value] = 0
dataframe[SignalType.EXIT_LONG.value] = 0
dataframe[SignalType.ENTER_SHORT.value] = 0
dataframe[SignalType.EXIT_SHORT.value] = 0
dataframe[SignalTagType.ENTER_TAG.value] = None
dataframe[SignalTagType.EXIT_TAG.value] = None

72
freqtrade/rpc/rpc.py
View File

@@ -24,7 +24,8 @@ from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, LeaderMessage
from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
from freqtrade.misc import decimals_per_coin, json_to_dataframe, shorten_date
from freqtrade.misc import (decimals_per_coin, json_to_dataframe, remove_entry_exit_signals,
shorten_date)
from freqtrade.persistence import PairLocks, Trade
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -1090,41 +1091,64 @@ class RPC:
'last_process_ts': int(last_p.timestamp()),
}
def _handle_emitted_data(self, type, data):
# ------------------------------ EXTERNAL SIGNALS -----------------------
def _initial_leader_data(self):
# We create a list of Messages to send to the follower on connect
data = []
# Send Pairlist data
data.append({
"data_type": LeaderMessageType.pairlist,
"data": self._freqtrade.pairlists._whitelist
})
return data
def _handle_pairlist_message(self, type, data):
"""
Handles the emitted data from the Leaders
Handles the emitted pairlists from the Leaders
:param type: The data_type of the data
:param data: The data
"""
logger.debug(f"Handling emitted data of type ({type})")
pairlist = data
if type == LeaderMessageType.pairlist:
pairlist = data
logger.debug(f"Handling Pairlist message: {pairlist}")
logger.debug(pairlist)
external_pairlist = self._freqtrade.pairlists._pairlist_handlers[0]
external_pairlist.add_pairlist_data(pairlist)
# Add the pairlist data to the ExternalPairList object
external_pairlist = self._freqtrade.pairlists._pairlist_handlers[0]
external_pairlist.add_pairlist_data(pairlist)
def _handle_analyzed_df_message(self, type, data):
"""
Handles the analyzed dataframes from the Leaders
elif type == LeaderMessageType.analyzed_df:
:param type: The data_type of the data
:param data: The data
"""
key, value = data["key"], data["value"]
pair, timeframe, candle_type = key
# Convert the dataframe back from json
key, value = data["key"], data["value"]
# Skip any pairs that we don't have in the pairlist?
# leader_pairlist = self._freqtrade.pairlists._whitelist
# if pair not in leader_pairlist:
# return
pair, timeframe, candle_type = key
dataframe = json_to_dataframe(value)
# Skip any pairs that we don't have in the pairlist?
# leader_pairlist = self._freqtrade.pairlists._whitelist
# if pair not in leader_pairlist:
# return
if self._config.get('external_signal', {}).get('remove_signals_analyzed_df', False):
dataframe = remove_entry_exit_signals(dataframe)
dataframe = json_to_dataframe(value)
logger.debug(f"Handling analyzed dataframe for {pair}")
logger.debug(dataframe.tail())
logger.debug(f"Received analyzed dataframe for {pair}")
logger.debug(dataframe.tail())
# Add the dataframe to the dataprovider
dataprovider = self._freqtrade.dataprovider
dataprovider.add_external_df(pair, timeframe, dataframe, candle_type)
# Add the dataframe to the dataprovider
dataprovider = self._freqtrade.dataprovider
dataprovider.add_external_df(pair, timeframe, dataframe, candle_type)
def _handle_default_message(self, type, data):
"""
Default leader message handler, just logs it. We should never have to
run this unless the leader sends us some weird message.
"""
logger.debug(f"Received message from Leader of type {type}: {data}")

13
freqtrade/rpc/rpc_manager.py
View File

@@ -45,25 +45,20 @@ class RPCManager:
if config.get('api_server', {}).get('enabled', False):
logger.info('Enabling rpc.api_server')
from freqtrade.rpc.api_server import ApiServer
# Pass replicate_rpc as param or defer starting api_server
# until we register the replicate rpc enpoint?
apiserver = ApiServer(config)
apiserver.add_rpc_handler(self._rpc)
self.registered_modules.append(apiserver)
# Enable Replicate mode
# Enable External Signals mode
# For this to be enabled, the API server must also be enabled
if config.get('external_signal', {}).get('enabled', False):
logger.info('Enabling RPC.ExternalSignalController')
from freqtrade.rpc.external_signal import ExternalSignalController
external_signal_rpc = ExternalSignalController(self._rpc, config, apiserver)
self.registered_modules.append(external_signal_rpc)
external_signals = ExternalSignalController(self._rpc, config, apiserver)
self.registered_modules.append(external_signals)
# Attach the controller to FreqTrade
freqtrade.external_signal_controller = external_signal_rpc
apiserver.start_api()
freqtrade.external_signal_controller = external_signals
def cleanup(self) -> None:
""" Stops all enabled rpc modules """