diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 49d33d46f..308f0b32d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -262,7 +262,10 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str: :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - return dataframe.to_json(orient='split') + # https://github.com/pandas-dev/pandas/issues/24889 + # https://github.com/pandas-dev/pandas/issues/40443 + # We need to convert to a dict to avoid mem leak + return dataframe.to_dict(orient='tight') def json_to_dataframe(data: str) -> pandas.DataFrame: @@ -271,7 +274,7 @@ def json_to_dataframe(data: str) -> pandas.DataFrame: :param data: A JSON string :returns: A pandas DataFrame from the JSON string """ - dataframe = pandas.read_json(data, orient='split') + dataframe = pandas.DataFrame.from_dict(data, orient='tight') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) diff --git a/freqtrade/rpc/api_server/ws/serializer.py b/freqtrade/rpc/api_server/ws/serializer.py index 6c402a100..8d06746f7 100644 --- a/freqtrade/rpc/api_server/ws/serializer.py +++ b/freqtrade/rpc/api_server/ws/serializer.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import orjson import rapidjson -from pandas import DataFrame +from pandas import DataFrame, Timestamp from freqtrade.misc import dataframe_to_json, json_to_dataframe from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy @@ -52,6 +52,11 @@ def _json_default(z): '__type__': 'dataframe', '__value__': dataframe_to_json(z) } + # Pandas returns a Timestamp object, we need to + # convert it to a timestamp int (with ms) for orjson + # to handle it + if isinstance(z, Timestamp): + return z.timestamp() * 1e3 raise TypeError diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..ff437e63e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -101,7 +101,7 @@ def json_deserialize(message): :param message: The message to deserialize """ def json_to_dataframe(data: str) -> pandas.DataFrame: - dataframe = pandas.read_json(data, orient='split') + dataframe = pandas.DataFrame.from_dict(data, orient='tight') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)