From 875e9ab447a6d3d165b13a64f95eb79f94daec74 Mon Sep 17 00:00:00 2001
From: Timothy Pogue <me@pogue.dev>
Date: Thu, 17 Nov 2022 11:59:03 -0700
Subject: [PATCH] change df serialization to avoid mem leak

---
 freqtrade/misc.py                         | 7 +++++--
 freqtrade/rpc/api_server/ws/serializer.py | 7 ++++++-
 scripts/ws_client.py                      | 2 +-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/freqtrade/misc.py b/freqtrade/misc.py
index 49d33d46f..308f0b32d 100644
--- a/freqtrade/misc.py
+++ b/freqtrade/misc.py
@@ -262,7 +262,10 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
     :param dataframe: A pandas DataFrame
     :returns: A JSON string of the pandas DataFrame
     """
-    return dataframe.to_json(orient='split')
+    # https://github.com/pandas-dev/pandas/issues/24889
+    # https://github.com/pandas-dev/pandas/issues/40443
+    # We need to convert to a dict to avoid mem leak
+    return dataframe.to_dict(orient='tight')
 
 
 def json_to_dataframe(data: str) -> pandas.DataFrame:
@@ -271,7 +274,7 @@ def json_to_dataframe(data: str) -> pandas.DataFrame:
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pandas.read_json(data, orient='split')
+    dataframe = pandas.DataFrame.from_dict(data, orient='tight')
     if 'date' in dataframe.columns:
         dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
 
diff --git a/freqtrade/rpc/api_server/ws/serializer.py b/freqtrade/rpc/api_server/ws/serializer.py
index 6c402a100..8d06746f7 100644
--- a/freqtrade/rpc/api_server/ws/serializer.py
+++ b/freqtrade/rpc/api_server/ws/serializer.py
@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 
 import orjson
 import rapidjson
-from pandas import DataFrame
+from pandas import DataFrame, Timestamp
 
 from freqtrade.misc import dataframe_to_json, json_to_dataframe
 from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy
@@ -52,6 +52,11 @@ def _json_default(z):
             '__type__': 'dataframe',
             '__value__': dataframe_to_json(z)
         }
+    # Pandas returns a Timestamp object, we need to
+    # convert it to a timestamp int (with ms) for orjson
+    # to handle it
+    if isinstance(z, Timestamp):
+        return z.timestamp() * 1e3
     raise TypeError
 
 
diff --git a/scripts/ws_client.py b/scripts/ws_client.py
index 090039cde..ff437e63e 100644
--- a/scripts/ws_client.py
+++ b/scripts/ws_client.py
@@ -101,7 +101,7 @@ def json_deserialize(message):
     :param message: The message to deserialize
     """
     def json_to_dataframe(data: str) -> pandas.DataFrame:
-        dataframe = pandas.read_json(data, orient='split')
+        dataframe = pandas.DataFrame.from_dict(data, orient='tight')
         if 'date' in dataframe.columns:
             dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)