adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change df serialization to avoid mem leak

Browse Source
This commit is contained in:
Timothy Pogue 2022-11-17 11:59:03 -07:00
parent 0a7f4fd3cc
commit 875e9ab447
3 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
freqtrade/misc.py
View File

@ -262,7 +262,10 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
:param dataframe: A pandas DataFrame :param dataframe: A pandas DataFrame
:returns: A JSON string of the pandas DataFrame :returns: A JSON string of the pandas DataFrame
""" """
return dataframe.to_json(orient='split') # https://github.com/pandas-dev/pandas/issues/24889
# https://github.com/pandas-dev/pandas/issues/40443
# We need to convert to a dict to avoid mem leak
return dataframe.to_dict(orient='tight')
def json_to_dataframe(data: str) -> pandas.DataFrame: def json_to_dataframe(data: str) -> pandas.DataFrame:
@ -271,7 +274,7 @@ def json_to_dataframe(data: str) -> pandas.DataFrame:
:param data: A JSON string :param data: A JSON string
:returns: A pandas DataFrame from the JSON string :returns: A pandas DataFrame from the JSON string
""" """
dataframe = pandas.read_json(data, orient='split') dataframe = pandas.DataFrame.from_dict(data, orient='tight')
if 'date' in dataframe.columns: if 'date' in dataframe.columns:
dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)

7
freqtrade/rpc/api_server/ws/serializer.py
View File

@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
import orjson import orjson
import rapidjson import rapidjson
from pandas import DataFrame from pandas import DataFrame, Timestamp
from freqtrade.misc import dataframe_to_json, json_to_dataframe from freqtrade.misc import dataframe_to_json, json_to_dataframe
from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy
@ -52,6 +52,11 @@ def _json_default(z):
'__type__': 'dataframe', '__type__': 'dataframe',
'__value__': dataframe_to_json(z) '__value__': dataframe_to_json(z)
} }
# Pandas returns a Timestamp object, we need to
# convert it to a timestamp int (with ms) for orjson
# to handle it
if isinstance(z, Timestamp):
return z.timestamp() * 1e3
raise TypeError raise TypeError

2
scripts/ws_client.py
View File

@ -101,7 +101,7 @@ def json_deserialize(message):
:param message: The message to deserialize :param message: The message to deserialize
""" """
def json_to_dataframe(data: str) -> pandas.DataFrame: def json_to_dataframe(data: str) -> pandas.DataFrame:
dataframe = pandas.read_json(data, orient='split') dataframe = pandas.DataFrame.from_dict(data, orient='tight')
if 'date' in dataframe.columns: if 'date' in dataframe.columns:
dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)