change df serialization to avoid mem leak

freqtrade/misc.py

@@ -262,7 +262,10 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
     :param dataframe: A pandas DataFrame
     :returns: A JSON string of the pandas DataFrame
     """
-    return dataframe.to_json(orient='split')
+    # https://github.com/pandas-dev/pandas/issues/24889
+    # https://github.com/pandas-dev/pandas/issues/40443
+    # We need to convert to a dict to avoid mem leak
+    return dataframe.to_dict(orient='tight')
 
 
 def json_to_dataframe(data: str) -> pandas.DataFrame:
@@ -271,7 +274,7 @@ def json_to_dataframe(data: str) -> pandas.DataFrame:
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pandas.read_json(data, orient='split')
+    dataframe = pandas.DataFrame.from_dict(data, orient='tight')
     if 'date' in dataframe.columns:
         dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
 

freqtrade/rpc/api_server/ws/serializer.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 
 import orjson
 import rapidjson
-from pandas import DataFrame
+from pandas import DataFrame, Timestamp
 
 from freqtrade.misc import dataframe_to_json, json_to_dataframe
 from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy
@@ -52,6 +52,11 @@ def _json_default(z):
             '__type__': 'dataframe',
             '__value__': dataframe_to_json(z)
         }
+    # Pandas returns a Timestamp object, we need to
+    # convert it to a timestamp int (with ms) for orjson
+    # to handle it
+    if isinstance(z, Timestamp):
+        return z.timestamp() * 1e3
     raise TypeError
 
 

scripts/ws_client.py

@@ -101,7 +101,7 @@ def json_deserialize(message):
     :param message: The message to deserialize
     """
     def json_to_dataframe(data: str) -> pandas.DataFrame:
-        dataframe = pandas.read_json(data, orient='split')
+        dataframe = pandas.DataFrame.from_dict(data, orient='tight')
         if 'date' in dataframe.columns:
             dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)