Merge branch 'develop' into feat/refactor-ws

freqtrade/misc.py

@@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union
 from typing.io import IO
 from urllib.parse import urlparse
 
-import pandas
+import orjson
+import pandas as pd
 import rapidjson
 
 from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@@ -256,7 +257,7 @@ def parse_db_uri_for_logging(uri: str):
     return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
 
 
-def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
+def dataframe_to_json(dataframe: pd.DataFrame) -> str:
     """
     Serialize a DataFrame for transmission over the wire using JSON
     :param dataframe: A pandas DataFrame
@@ -265,23 +266,28 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
     # https://github.com/pandas-dev/pandas/issues/24889
     # https://github.com/pandas-dev/pandas/issues/40443
     # We need to convert to a dict to avoid mem leak
-    return dataframe.to_dict(orient='tight')
+    def default(z):
+        if isinstance(z, pd.Timestamp):
+            return z.timestamp() * 1e3
+        raise TypeError
+
+    return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
 
 
-def json_to_dataframe(data: str) -> pandas.DataFrame:
+def json_to_dataframe(data: str) -> pd.DataFrame:
     """
     Deserialize JSON into a DataFrame
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pandas.DataFrame.from_dict(data, orient='tight')
+    dataframe = pd.read_json(data, orient='split')
     if 'date' in dataframe.columns:
-        dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
+        dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
 
     return dataframe
 
 
-def remove_entry_exit_signals(dataframe: pandas.DataFrame):
+def remove_entry_exit_signals(dataframe: pd.DataFrame):
     """
     Remove Entry and Exit signals from a DataFrame
 

freqtrade/persistence/trade_model.py

@@ -90,6 +90,13 @@ class Order(_DECL_BASE):
     def safe_filled(self) -> float:
         return self.filled if self.filled is not None else self.amount or 0.0
 
+    @property
+    def safe_remaining(self) -> float:
+        return (
+            self.remaining if self.remaining is not None else
+            self.amount - (self.filled or 0.0)
+        )
+
     @property
     def safe_fee_base(self) -> float:
         return self.ft_fee_base or 0.0

freqtrade/rpc/api_server/ws/serializer.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, Union
 
 import orjson
 import rapidjson
-from pandas import DataFrame, Timestamp
+from pandas import DataFrame
 
 from freqtrade.misc import dataframe_to_json, json_to_dataframe
 from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy
@@ -51,11 +51,6 @@ def _json_default(z):
             '__type__': 'dataframe',
             '__value__': dataframe_to_json(z)
         }
-    # Pandas returns a Timestamp object, we need to
-    # convert it to a timestamp int (with ms) for orjson
-    # to handle it
-    if isinstance(z, Timestamp):
-        return z.timestamp() * 1e3
     raise TypeError
 
 

freqtrade/rpc/rpc.py

@@ -218,9 +218,10 @@ class RPC:
                     stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2),
                     stoploss_entry_dist=stoploss_entry_dist,
                     stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
-                    open_order='({} {} rem={:.8f})'.format(
-                        order.order_type, order.side, order.remaining
-                    ) if order else None,
+                    open_order=(
+                        f'({order.order_type} {order.side} rem={order.safe_remaining:.8f})' if
+                        order else None
+                    ),
                 ))
                 results.append(trade_dict)
             return results