Merge pull request #7842 from wizrds/feat/refactor-emc
Change to broadcasting single candles in Producer/Consumer
This commit is contained in:
Matthias
GitHub
@@ -2,13 +2,13 @@ from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from pandas import DataFrame
|
||||
from pandas import DataFrame, Timestamp
|
||||
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.enums import CandleType, RunMode
|
||||
from freqtrade.exceptions import ExchangeError, OperationalException
|
||||
from freqtrade.plugins.pairlistmanager import PairListManager
|
||||
from tests.conftest import get_patched_exchange
|
||||
from tests.conftest import generate_test_data, get_patched_exchange
|
||||
|
||||
|
||||
@pytest.mark.parametrize('candle_type', [
|
||||
@@ -161,9 +161,9 @@ def test_producer_pairs(mocker, default_conf, ohlcv_history):
|
||||
assert dataprovider.get_producer_pairs("bad") == []
|
||||
|
||||
|
||||
def test_get_producer_df(mocker, default_conf, ohlcv_history):
|
||||
def test_get_producer_df(mocker, default_conf):
|
||||
dataprovider = DataProvider(default_conf, None)
|
||||
|
||||
ohlcv_history = generate_test_data('5m', 150)
|
||||
pair = 'BTC/USDT'
|
||||
timeframe = default_conf['timeframe']
|
||||
candle_type = CandleType.SPOT
|
||||
@@ -412,3 +412,80 @@ def test_dp_send_msg(default_conf):
|
||||
dp = DataProvider(default_conf, None)
|
||||
dp.send_msg(msg, always_send=True)
|
||||
assert msg not in dp._msg_queue
|
||||
|
||||
|
||||
def test_dp__add_external_df(default_conf_usdt):
|
||||
timeframe = '1h'
|
||||
default_conf_usdt["timeframe"] = timeframe
|
||||
dp = DataProvider(default_conf_usdt, None)
|
||||
df = generate_test_data(timeframe, 24, '2022-01-01 00:00:00+00:00')
|
||||
last_analyzed = datetime.now(timezone.utc)
|
||||
|
||||
res = dp._add_external_df('ETH/USDT', df, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is False
|
||||
# Why 1000 ??
|
||||
assert res[1] == 1000
|
||||
|
||||
# Hard add dataframe
|
||||
dp._replace_external_df('ETH/USDT', df, last_analyzed, timeframe, CandleType.SPOT)
|
||||
# BTC is not stored yet
|
||||
res = dp._add_external_df('BTC/USDT', df, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is False
|
||||
df_res, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT)
|
||||
assert len(df_res) == 24
|
||||
|
||||
# Add the same dataframe again - dataframe size shall not change.
|
||||
res = dp._add_external_df('ETH/USDT', df, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is True
|
||||
assert res[1] == 0
|
||||
df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT)
|
||||
assert len(df) == 24
|
||||
|
||||
# Add a new day.
|
||||
df2 = generate_test_data(timeframe, 24, '2022-01-02 00:00:00+00:00')
|
||||
|
||||
res = dp._add_external_df('ETH/USDT', df2, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is True
|
||||
assert res[1] == 0
|
||||
df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT)
|
||||
assert len(df) == 48
|
||||
|
||||
# Add a dataframe with a 12 hour offset - so 12 candles are overlapping, and 12 valid.
|
||||
df3 = generate_test_data(timeframe, 24, '2022-01-02 12:00:00+00:00')
|
||||
|
||||
res = dp._add_external_df('ETH/USDT', df3, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is True
|
||||
assert res[1] == 0
|
||||
df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT)
|
||||
# New length = 48 + 12 (since we have a 12 hour offset).
|
||||
assert len(df) == 60
|
||||
assert df.iloc[-1]['date'] == df3.iloc[-1]['date']
|
||||
assert df.iloc[-1]['date'] == Timestamp('2022-01-03 11:00:00+00:00')
|
||||
|
||||
# Generate 1 new candle
|
||||
df4 = generate_test_data(timeframe, 1, '2022-01-03 12:00:00+00:00')
|
||||
res = dp._add_external_df('ETH/USDT', df4, last_analyzed, timeframe, CandleType.SPOT)
|
||||
# assert res[0] is True
|
||||
# assert res[1] == 0
|
||||
df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT)
|
||||
# New length = 61 + 1
|
||||
assert len(df) == 61
|
||||
assert df.iloc[-2]['date'] == Timestamp('2022-01-03 11:00:00+00:00')
|
||||
assert df.iloc[-1]['date'] == Timestamp('2022-01-03 12:00:00+00:00')
|
||||
|
||||
# Gap in the data ...
|
||||
df4 = generate_test_data(timeframe, 1, '2022-01-05 00:00:00+00:00')
|
||||
res = dp._add_external_df('ETH/USDT', df4, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is False
|
||||
# 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00
|
||||
assert res[1] == 36
|
||||
df, _ = dp.get_producer_df('ETH/USDT', timeframe, CandleType.SPOT)
|
||||
# New length = 61 + 1
|
||||
assert len(df) == 61
|
||||
|
||||
# Empty dataframe
|
||||
df4 = generate_test_data(timeframe, 0, '2022-01-05 00:00:00+00:00')
|
||||
res = dp._add_external_df('ETH/USDT', df4, last_analyzed, timeframe, CandleType.SPOT)
|
||||
assert res[0] is False
|
||||
# 36 hours - from 2022-01-03 12:00:00+00:00 to 2022-01-05 00:00:00+00:00
|
||||
assert res[1] == 0
|
||||
|
||||
@@ -83,6 +83,7 @@ def test_emc_init(patched_emc):
|
||||
def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
|
||||
test_producer = {"name": "test", "url": "ws://test", "ws_token": "test"}
|
||||
producer_name = test_producer['name']
|
||||
invalid_msg = r"Invalid message .+"
|
||||
|
||||
caplog.set_level(logging.DEBUG)
|
||||
|
||||
@@ -94,7 +95,7 @@ def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
|
||||
assert log_has(
|
||||
f"Consumed message from `{producer_name}` of type `RPCMessageType.WHITELIST`", caplog)
|
||||
|
||||
# Test handle analyzed_df message
|
||||
# Test handle analyzed_df single candle message
|
||||
df_message = {
|
||||
"type": "analyzed_df",
|
||||
"data": {
|
||||
@@ -106,8 +107,7 @@ def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
|
||||
patched_emc.handle_producer_message(test_producer, df_message)
|
||||
|
||||
assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
|
||||
assert log_has(
|
||||
f"Consumed message from `{producer_name}` of type `RPCMessageType.ANALYZED_DF`", caplog)
|
||||
assert log_has_re(r"Holes in data or no existing df,.+", caplog)
|
||||
|
||||
# Test unhandled message
|
||||
unhandled_message = {"type": "status", "data": "RUNNING"}
|
||||
@@ -120,7 +120,8 @@ def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
|
||||
malformed_message = {"type": "whitelist", "data": {"pair": "BTC/USDT"}}
|
||||
patched_emc.handle_producer_message(test_producer, malformed_message)
|
||||
|
||||
assert log_has_re(r"Invalid message .+", caplog)
|
||||
assert log_has_re(invalid_msg, caplog)
|
||||
caplog.clear()
|
||||
|
||||
malformed_message = {
|
||||
"type": "analyzed_df",
|
||||
@@ -133,13 +134,30 @@ def test_emc_handle_producer_message(patched_emc, caplog, ohlcv_history):
|
||||
patched_emc.handle_producer_message(test_producer, malformed_message)
|
||||
|
||||
assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
|
||||
assert log_has_re(r"Invalid message .+", caplog)
|
||||
assert log_has_re(invalid_msg, caplog)
|
||||
caplog.clear()
|
||||
|
||||
# Empty dataframe
|
||||
malformed_message = {
|
||||
"type": "analyzed_df",
|
||||
"data": {
|
||||
"key": ("BTC/USDT", "5m", "spot"),
|
||||
"df": ohlcv_history.loc[ohlcv_history['open'] < 0],
|
||||
"la": datetime.now(timezone.utc)
|
||||
}
|
||||
}
|
||||
patched_emc.handle_producer_message(test_producer, malformed_message)
|
||||
|
||||
assert log_has(f"Received message of type `analyzed_df` from `{producer_name}`", caplog)
|
||||
assert not log_has_re(invalid_msg, caplog)
|
||||
assert log_has_re(r"Received Empty Dataframe for.+", caplog)
|
||||
|
||||
caplog.clear()
|
||||
malformed_message = {"some": "stuff"}
|
||||
patched_emc.handle_producer_message(test_producer, malformed_message)
|
||||
|
||||
assert log_has_re(r"Invalid message .+", caplog)
|
||||
assert log_has_re(invalid_msg, caplog)
|
||||
caplog.clear()
|
||||
|
||||
caplog.clear()
|
||||
malformed_message = {"type": "whitelist", "data": None}
|
||||
@@ -183,7 +201,7 @@ async def test_emc_create_connection_success(default_conf, caplog, mocker):
|
||||
async with websockets.serve(eat, _TEST_WS_HOST, _TEST_WS_PORT):
|
||||
await emc._create_connection(test_producer, lock)
|
||||
|
||||
assert log_has_re(r"Producer connection success.+", caplog)
|
||||
assert log_has_re(r"Connected to channel.+", caplog)
|
||||
finally:
|
||||
emc.shutdown()
|
||||
|
||||
@@ -212,7 +230,8 @@ async def test_emc_create_connection_invalid_url(default_conf, caplog, mocker, h
|
||||
|
||||
dp = DataProvider(default_conf, None, None, None)
|
||||
# Handle start explicitly to avoid messing with threading in tests
|
||||
mocker.patch("freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start",)
|
||||
mocker.patch("freqtrade.rpc.external_message_consumer.ExternalMessageConsumer.start")
|
||||
mocker.patch("freqtrade.rpc.api_server.ws.channel.create_channel")
|
||||
emc = ExternalMessageConsumer(default_conf, dp)
|
||||
|
||||
try:
|
||||
@@ -390,7 +409,9 @@ async def test_emc_receive_messages_timeout(default_conf, caplog, mocker):
|
||||
try:
|
||||
change_running(emc)
|
||||
loop.call_soon(functools.partial(change_running, emc=emc))
|
||||
await emc._receive_messages(TestChannel(), test_producer, lock)
|
||||
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await emc._receive_messages(TestChannel(), test_producer, lock)
|
||||
|
||||
assert log_has_re(r"Ping error.+", caplog)
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user