From 07f806a314ec6e963a33b79782e8b658782b235d Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 4 Sep 2022 10:22:10 -0600 Subject: [PATCH] minor improvements, fixes, old config+constant removal --- .gitignore | 2 -- config_examples/config_full.example.json | 5 +++- freqtrade/constants.py | 3 +- freqtrade/data/dataprovider.py | 33 +++++++++++++++------- freqtrade/freqtradebot.py | 4 --- freqtrade/rpc/api_server/api_auth.py | 1 + freqtrade/rpc/api_server/api_ws.py | 15 +++++++--- freqtrade/rpc/api_server/webserver.py | 5 ++-- freqtrade/rpc/external_message_consumer.py | 6 ++-- freqtrade/strategy/interface.py | 15 ++++------ 10 files changed, 51 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index 6a47a7f81..e400c01f5 100644 --- a/.gitignore +++ b/.gitignore @@ -113,5 +113,3 @@ target/ !config_examples/config_full.example.json !config_examples/config_kraken.example.json !config_examples/config_freqai.example.json -!config_examples/config_leader.example.json -!config_examples/config_follower.example.json diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index 2ebad4924..99d695406 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -175,12 +175,15 @@ "password": "SuperSecurePassword", "ws_token": "a_secret_ws_token" }, + // The ExternalMessageConsumer config should only be enabled on an instance + // that listens to outside data from another instance. This should not be enabled + // in your producer of data. "external_message_consumer": { "enabled": false, "producers": [ { "name": "default", - "url": "ws://some.freqtrade.bot/api/v1/message/ws", + "url": "ws://localhost:8081/api/v1/message/ws", "ws_token": "a_secret_ws_token" } ], diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 352e48148..bc00d7cfc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -33,8 +33,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', - 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter', - 'ExternalPairList'] + 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] BACKTEST_BREAKDOWNS = ['day', 'week', 'month'] diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index af0f2a70a..32152f2f5 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -48,8 +48,11 @@ class DataProvider: self.__producer_pairs: Dict[str, List[str]] = {} self._msg_queue: deque = deque() + self._default_candle_type = self._config.get('candle_type_def', CandleType.SPOT) + self._default_timeframe = self._config.get('timeframe', '1h') + self.__msg_cache = PeriodicCache( - maxsize=1000, ttl=timeframe_to_seconds(self._config.get('timeframe', '1h'))) + maxsize=1000, ttl=timeframe_to_seconds(self._default_timeframe)) self._num_sources = len( self._config.get('external_message_consumer', {}).get('producers', []) @@ -84,7 +87,7 @@ class DataProvider: dataframe, datetime.now(timezone.utc)) # For multiple producers we will want to merge the pairlists instead of overwriting - def set_producer_pairs(self, pairlist: List[str], producer_name: str = "default"): + def _set_producer_pairs(self, pairlist: List[str], producer_name: str = "default"): """ Set the pairs received to later be used. This only supports 1 Producer right now. @@ -101,7 +104,7 @@ class DataProvider: """ return self.__producer_pairs.get(producer_name, []) - def emit_df( + def _emit_df( self, pair_key: PairWithTimeframe, dataframe: DataFrame @@ -123,12 +126,12 @@ class DataProvider: } ) - def add_external_df( + def _add_external_df( self, pair: str, - timeframe: str, dataframe: DataFrame, - candle_type: CandleType, + timeframe: Optional[str] = None, + candle_type: Optional[CandleType] = None, producer_name: str = "default" ) -> None: """ @@ -138,18 +141,22 @@ class DataProvider: :param timeframe: Timeframe to get data for :param candle_type: Any of the enum CandleType (must match trading mode!) """ - pair_key = (pair, timeframe, candle_type) + _timeframe = self._default_timeframe if not timeframe else timeframe + _candle_type = self._default_candle_type if not candle_type else candle_type + + pair_key = (pair, _timeframe, _candle_type) if producer_name not in self.__producer_pairs_df: self.__producer_pairs_df[producer_name] = {} self.__producer_pairs_df[producer_name][pair_key] = (dataframe, datetime.now(timezone.utc)) + logger.debug(f"External DataFrame for {pair_key} from {producer_name} added.") def get_external_df( self, pair: str, - timeframe: str, - candle_type: CandleType, + timeframe: Optional[str] = None, + candle_type: Optional[CandleType] = None, producer_name: str = "default" ) -> Tuple[DataFrame, datetime]: """ @@ -160,16 +167,22 @@ class DataProvider: :param timeframe: Timeframe to get data for :param candle_type: Any of the enum CandleType (must match trading mode!) """ - pair_key = (pair, timeframe, candle_type) + _timeframe = self._default_timeframe if not timeframe else timeframe + _candle_type = self._default_candle_type if not candle_type else candle_type + pair_key = (pair, _timeframe, _candle_type) + + # If we have no data from this Producer yet if producer_name not in self.__producer_pairs_df: # We don't have this data yet, return empty DataFrame and datetime (01-01-1970) return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) + # If we do have data from that Producer, but no data on this pair_key if pair_key not in self.__producer_pairs_df[producer_name]: # We don't have this data yet, return empty DataFrame and datetime (01-01-1970) return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) + # We have it, return this data return self.__producer_pairs_df[producer_name][pair_key] def add_pairlisthandler(self, pairlists) -> None: diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 888994ffb..ce01fc872 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -199,10 +199,6 @@ class FreqtradeBot(LoggingMixin): strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() - # This just means we won't broadcast dataframes if we're listening to a producer - # Doesn't necessarily NEED to be this way, as maybe we'd like to broadcast - # even if we are using external dataframes in the future. - self.strategy.analyze(self.active_pair_whitelist) with self._exit_lock: diff --git a/freqtrade/rpc/api_server/api_auth.py b/freqtrade/rpc/api_server/api_auth.py index fd90918e1..6655dbf86 100644 --- a/freqtrade/rpc/api_server/api_auth.py +++ b/freqtrade/rpc/api_server/api_auth.py @@ -50,6 +50,7 @@ def get_user_from_token(token, secret_key: str, token_type: str = "access"): # This should be reimplemented to better realign with the existing tools provided # by FastAPI regarding API Tokens +# https://github.com/tiangolo/fastapi/blob/master/fastapi/security/api_key.py async def get_ws_token( ws: WebSocket, token: Union[str, None] = None, diff --git a/freqtrade/rpc/api_server/api_ws.py b/freqtrade/rpc/api_server/api_ws.py index 95cfd031a..d11d1acfe 100644 --- a/freqtrade/rpc/api_server/api_ws.py +++ b/freqtrade/rpc/api_server/api_ws.py @@ -2,23 +2,30 @@ import logging from typing import Any, Dict from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect +# fastapi does not make this available through it, so import directly from starlette +from starlette.websockets import WebSocketState from freqtrade.enums import RPCMessageType, RPCRequestType from freqtrade.rpc.api_server.deps import get_channel_manager, get_rpc from freqtrade.rpc.api_server.ws.channel import WebSocketChannel -from freqtrade.rpc.api_server.ws.utils import is_websocket_alive from freqtrade.rpc.rpc import RPC -# from typing import Any, Dict - - logger = logging.getLogger(__name__) # Private router, protected by API Key authentication router = APIRouter() +async def is_websocket_alive(ws: WebSocket) -> bool: + if ( + ws.application_state == WebSocketState.CONNECTED and + ws.client_state == WebSocketState.CONNECTED + ): + return True + return False + + async def _process_consumer_request( request: Dict[str, Any], channel: WebSocketChannel, diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 150c83890..ad93e77a7 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -205,7 +205,7 @@ class ApiServer(RPCHandler): # For testing, shouldn't happen when stable except Exception as e: - logger.info(f"Exception happened in background task: {e}") + logger.exception(f"Exception happened in background task: {e}") def start_api(self): """ @@ -244,8 +244,7 @@ class ApiServer(RPCHandler): if self._standalone: self._server.run() else: - if self._config.get('api_server', {}).get('enable_message_ws', False): - self.start_message_queue() + self.start_message_queue() self._server.run_in_thread() except Exception: logger.exception("Api server failed to start.") diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index d3b82cadf..7f2ac01fb 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -289,7 +289,7 @@ class ExternalMessageConsumer: return # Add the pairlist data to the DataProvider - self._dp.set_producer_pairs(message_data, producer_name=producer_name) + self._dp._set_producer_pairs(message_data, producer_name=producer_name) logger.debug(f"Consumed message from {producer_name} of type RPCMessageType.WHITELIST") @@ -309,8 +309,8 @@ class ExternalMessageConsumer: dataframe = remove_entry_exit_signals(dataframe) # Add the dataframe to the dataprovider - self._dp.add_external_df(pair, timeframe, dataframe, - candle_type, producer_name=producer_name) + self._dp._add_external_df(pair, dataframe, timeframe, + candle_type, producer_name=producer_name) logger.debug( f"Consumed message from {producer_name} of type RPCMessageType.ANALYZED_DF") diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7fcae870a..73948a37a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -700,8 +700,7 @@ class IStrategy(ABC, HyperStrategyMixin): def _analyze_ticker_internal( self, dataframe: DataFrame, - metadata: dict, - emit_df: bool = False + metadata: dict ) -> DataFrame: """ Parses the given candle (OHLCV) data and returns a populated DataFrame @@ -725,7 +724,7 @@ class IStrategy(ABC, HyperStrategyMixin): candle_type = self.config.get('candle_type_def', CandleType.SPOT) self.dp._set_cached_df(pair, self.timeframe, dataframe, candle_type=candle_type) - self.dp.emit_df((pair, self.timeframe, candle_type), dataframe) + self.dp._emit_df((pair, self.timeframe, candle_type), dataframe) else: logger.debug("Skipping TA Analysis for already analyzed candle") @@ -737,8 +736,7 @@ class IStrategy(ABC, HyperStrategyMixin): def analyze_pair( self, - pair: str, - emit_df: bool = False + pair: str ) -> None: """ Fetch data for this pair from dataprovider and analyze. @@ -759,7 +757,7 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe = strategy_safe_wrapper( self._analyze_ticker_internal, message="" - )(dataframe, {'pair': pair}, emit_df) + )(dataframe, {'pair': pair}) self.assert_df(dataframe, df_len, df_close, df_date) except StrategyError as error: @@ -772,15 +770,14 @@ class IStrategy(ABC, HyperStrategyMixin): def analyze( self, - pairs: List[str], - emit_df: bool = False + pairs: List[str] ) -> None: """ Analyze all pairs using analyze_pair(). :param pairs: List of pairs to analyze """ for pair in pairs: - self.analyze_pair(pair, emit_df) + self.analyze_pair(pair) @ staticmethod def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]: