adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge develop into track-current-candle

Browse Source
This commit is contained in:
robcaulk
2022-11-03 21:09:08 +01:00
parent db942321ad c2130ed3dd
commit 444a068481
12 changed files with 121 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
freqtrade/freqai/data_kitchen.py
View File

@@ -1316,14 +1316,16 @@ class FreqaiDataKitchen:
append_df = pd.read_hdf(self.backtesting_results_path)
return append_df
def check_if_backtest_prediction_exists(
self
def check_if_backtest_prediction_is_valid(
self,
length_backtesting_dataframe: int
) -> bool:
"""
Check if a backtesting prediction already exists
:param dk: FreqaiDataKitchen
Check if a backtesting prediction already exists and if the predictions
to append has the same size of backtesting dataframe slice
:param length_backtesting_dataframe: Length of backtesting dataframe slice
:return:
:boolean: whether the prediction file exists or not.
:boolean: whether the prediction file is valid.
"""
path_to_predictionfile = Path(self.full_path /
self.backtest_predictions_folder /
@@ -1331,13 +1333,21 @@ class FreqaiDataKitchen:
self.backtesting_results_path = path_to_predictionfile
file_exists = path_to_predictionfile.is_file()
if file_exists:
logger.info(f"Found backtesting prediction file at {path_to_predictionfile}")
append_df = self.get_backtesting_prediction()
if len(append_df) == length_backtesting_dataframe:
logger.info(f"Found backtesting prediction file at {path_to_predictionfile}")
return True
else:
logger.info("A new backtesting prediction file is required. "
"(Number of predictions is different from dataframe length).")
return False
else:
logger.info(
f"Could not find backtesting prediction file at {path_to_predictionfile}"
)
return file_exists
return False
def remove_special_chars_from_feature_names(self, dataframe: pd.DataFrame) -> pd.DataFrame:
"""

2
freqtrade/freqai/freqai_interface.py
View File

@@ -277,7 +277,7 @@ class IFreqaiModel(ABC):
dk.set_new_model_names(pair, trained_timestamp)
if dk.check_if_backtest_prediction_exists():
if dk.check_if_backtest_prediction_is_valid(len(dataframe_backtest)):
self.dd.load_metadata(dk)
dk.find_features(dataframe_train)
self.check_if_feature_list_matches_strategy(dk)

1
freqtrade/plugins/pairlist/IPairList.py
View File

@@ -36,7 +36,6 @@ class IPairList(LoggingMixin, ABC):
self._pairlistconfig = pairlistconfig
self._pairlist_pos = pairlist_pos
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._last_refresh = 0
LoggingMixin.__init__(self, logger, self.refresh_period)
@property

17
freqtrade/plugins/pairlist/ShuffleFilter.py
View File

@@ -3,16 +3,20 @@ Shuffle pair list filter
"""
import logging
import random
from typing import Any, Dict, List
from typing import Any, Dict, List, Literal
from freqtrade.constants import Config
from freqtrade.enums import RunMode
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList
from freqtrade.util.periodic_cache import PeriodicCache
logger = logging.getLogger(__name__)
ShuffleValues = Literal['candle', 'iteration']
class ShuffleFilter(IPairList):
@@ -31,6 +35,9 @@ class ShuffleFilter(IPairList):
logger.info(f"Backtesting mode detected, applying seed value: {self._seed}")
self._random = random.Random(self._seed)
self._shuffle_freq: ShuffleValues = pairlistconfig.get('shuffle_frequency', 'candle')
self.__pairlist_cache = PeriodicCache(
maxsize=1000, ttl=timeframe_to_seconds(self._config['timeframe']))
@property
def needstickers(self) -> bool:
@@ -45,7 +52,7 @@ class ShuffleFilter(IPairList):
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Shuffling pairs" +
return (f"{self.name} - Shuffling pairs every {self._shuffle_freq}" +
(f", seed = {self._seed}." if self._seed is not None else "."))
def filter_pairlist(self, pairlist: List[str], tickers: Tickers) -> List[str]:
@@ -56,7 +63,13 @@ class ShuffleFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: new whitelist
"""
pairlist_bef = tuple(pairlist)
pairlist_new = self.__pairlist_cache.get(pairlist_bef)
if pairlist_new and self._shuffle_freq == 'candle':
# Use cached pairlist.
return pairlist_new
# Shuffle is done inplace
self._random.shuffle(pairlist)
self.__pairlist_cache[pairlist_bef] = pairlist
return pairlist

10
freqtrade/rpc/api_server/api_ws.py
View File

@@ -127,13 +127,6 @@ async def message_endpoint(
except Exception as e:
logger.info(f"Consumer connection failed - {channel}: {e}")
logger.debug(e, exc_info=e)
finally:
await channel_manager.on_disconnect(ws)
else:
if channel:
await channel_manager.on_disconnect(ws)
await ws.close()
except RuntimeError:
# WebSocket was closed
@@ -144,4 +137,5 @@ async def message_endpoint(
# Log tracebacks to keep track of what errors are happening
logger.exception(e)
finally:
await channel_manager.on_disconnect(ws)
if channel:
await channel_manager.on_disconnect(ws)

4
freqtrade/rpc/api_server/webserver.py
View File

@@ -197,6 +197,7 @@ class ApiServer(RPCHandler):
# Get data from queue
message: WSMessageSchemaType = await async_queue.get()
logger.debug(f"Found message of type: {message.get('type')}")
async_queue.task_done()
# Broadcast it
await self._ws_channel_manager.broadcast(message)
except asyncio.CancelledError:
@@ -210,6 +211,9 @@ class ApiServer(RPCHandler):
# Disconnect channels and stop the loop on cancel
await self._ws_channel_manager.disconnect_all()
self._ws_loop.stop()
# Avoid adding more items to the queue if they aren't
# going to get broadcasted.
self._ws_queue = None
def start_api(self):
"""

37
freqtrade/rpc/api_server/ws/channel.py
View File

@@ -1,5 +1,6 @@
import asyncio
import logging
import time
from threading import RLock
from typing import Any, Dict, List, Optional, Type, Union
from uuid import uuid4
@@ -46,7 +47,7 @@ class WebSocketChannel:
self._relay_task = asyncio.create_task(self.relay())
# Internal event to signify a closed websocket
self._closed = False
self._closed = asyncio.Event()
# Wrap the WebSocket in the Serializing class
self._wrapped_ws = self._serializer_cls(self._websocket)
@@ -73,15 +74,26 @@ class WebSocketChannel:
Add the data to the queue to be sent.
:returns: True if data added to queue, False otherwise
"""
# This block only runs if the queue is full, it will wait
# until self.drain_timeout for the relay to drain the outgoing queue
# We can't use asyncio.wait_for here because the queue may have been created with a
# different eventloop
start = time.time()
while self.queue.full():
await asyncio.sleep(1)
if (time.time() - start) > self.drain_timeout:
return False
# If for some reason the queue is still full, just return False
try:
await asyncio.wait_for(
self.queue.put(data),
timeout=self.drain_timeout
)
return True
except asyncio.TimeoutError:
self.queue.put_nowait(data)
except asyncio.QueueFull:
return False
# If we got here everything is ok
return True
async def recv(self):
"""
Receive data on the wrapped websocket
@@ -99,14 +111,19 @@ class WebSocketChannel:
Close the WebSocketChannel
"""
self._closed = True
try:
await self.raw_websocket.close()
except Exception:
pass
self._closed.set()
self._relay_task.cancel()
def is_closed(self) -> bool:
"""
Closed flag
"""
return self._closed
return self._closed.is_set()
def set_subscriptions(self, subscriptions: List[str] = []) -> None:
"""
@@ -129,7 +146,7 @@ class WebSocketChannel:
Relay messages from the channel's queue and send them out. This is started
as a task.
"""
while True:
while not self._closed.is_set():
message = await self.queue.get()
try:
await self._send(message)

8
freqtrade/rpc/external_message_consumer.py
View File

@@ -264,10 +264,10 @@ class ExternalMessageConsumer:
# We haven't received data yet. Check the connection and continue.
try:
# ping
ping = await channel.ping()
pong = await channel.ping()
latency = (await asyncio.wait_for(pong, timeout=self.ping_timeout) * 1000)
await asyncio.wait_for(ping, timeout=self.ping_timeout)
logger.debug(f"Connection to {channel} still alive...")
logger.info(f"Connection to {channel} still alive, latency: {latency}ms")
continue
except (websockets.exceptions.ConnectionClosed):
@@ -276,7 +276,7 @@ class ExternalMessageConsumer:
await asyncio.sleep(self.sleep_time)
break
except Exception as e:
logger.warning(f"Ping error {channel} - retrying in {self.sleep_time}s")
logger.warning(f"Ping error {channel} - {e} - retrying in {self.sleep_time}s")
logger.debug(e, exc_info=e)
await asyncio.sleep(self.sleep_time)