Merge branch 'develop' into feat/refactor-ws

freqtrade/misc.py

@@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union
 from typing.io import IO
 from urllib.parse import urlparse
 
-import pandas
+import orjson
+import pandas as pd
 import rapidjson
 
 from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@@ -256,7 +257,7 @@ def parse_db_uri_for_logging(uri: str):
     return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
 
 
-def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
+def dataframe_to_json(dataframe: pd.DataFrame) -> str:
     """
     Serialize a DataFrame for transmission over the wire using JSON
     :param dataframe: A pandas DataFrame
@@ -265,23 +266,28 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
     # https://github.com/pandas-dev/pandas/issues/24889
     # https://github.com/pandas-dev/pandas/issues/40443
     # We need to convert to a dict to avoid mem leak
-    return dataframe.to_dict(orient='tight')
+    def default(z):
+        if isinstance(z, pd.Timestamp):
+            return z.timestamp() * 1e3
+        raise TypeError
+
+    return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
 
 
-def json_to_dataframe(data: str) -> pandas.DataFrame:
+def json_to_dataframe(data: str) -> pd.DataFrame:
     """
     Deserialize JSON into a DataFrame
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pandas.DataFrame.from_dict(data, orient='tight')
+    dataframe = pd.read_json(data, orient='split')
     if 'date' in dataframe.columns:
-        dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
+        dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
 
     return dataframe
 
 
-def remove_entry_exit_signals(dataframe: pandas.DataFrame):
+def remove_entry_exit_signals(dataframe: pd.DataFrame):
     """
     Remove Entry and Exit signals from a DataFrame
 

freqtrade/persistence/trade_model.py

@@ -90,6 +90,13 @@ class Order(_DECL_BASE):
     def safe_filled(self) -> float:
         return self.filled if self.filled is not None else self.amount or 0.0
 
+    @property
+    def safe_remaining(self) -> float:
+        return (
+            self.remaining if self.remaining is not None else
+            self.amount - (self.filled or 0.0)
+        )
+
     @property
     def safe_fee_base(self) -> float:
         return self.ft_fee_base or 0.0

freqtrade/rpc/api_server/ws/serializer.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, Union
 
 import orjson
 import rapidjson
-from pandas import DataFrame, Timestamp
+from pandas import DataFrame
 
 from freqtrade.misc import dataframe_to_json, json_to_dataframe
 from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy
@@ -51,11 +51,6 @@ def _json_default(z):
             '__type__': 'dataframe',
             '__value__': dataframe_to_json(z)
         }
-    # Pandas returns a Timestamp object, we need to
-    # convert it to a timestamp int (with ms) for orjson
-    # to handle it
-    if isinstance(z, Timestamp):
-        return z.timestamp() * 1e3
     raise TypeError
 
 

freqtrade/rpc/rpc.py

@@ -218,9 +218,10 @@ class RPC:
                     stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2),
                     stoploss_entry_dist=stoploss_entry_dist,
                     stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
-                    open_order='({} {} rem={:.8f})'.format(
+                    open_order=(
-                        order.order_type, order.side, order.remaining
+                        f'({order.order_type} {order.side} rem={order.safe_remaining:.8f})' if
-                    ) if order else None,
+                        order else None
+                    ),
                 ))
                 results.append(trade_dict)
             return results

scripts/ws_client.py

@@ -101,7 +101,7 @@ def json_deserialize(message):
     :param message: The message to deserialize
     """
     def json_to_dataframe(data: str) -> pandas.DataFrame:
-        dataframe = pandas.DataFrame.from_dict(data, orient='tight')
+        dataframe = pandas.read_json(data, orient='split')
         if 'date' in dataframe.columns:
             dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
 

tests/rpc/test_rpc.py

@@ -1,6 +1,7 @@
 # pragma pylint: disable=missing-docstring, C0103
 # pragma pylint: disable=invalid-sequence-index, invalid-name, too-many-arguments
 
+from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from unittest.mock import ANY, MagicMock, PropertyMock
 
@@ -28,113 +29,7 @@ def prec_satoshi(a, b) -> float:
 
 # Unit tests
 def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
-    mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
+    gen_response = {
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        fetch_ticker=ticker,
-        get_fee=fee,
-        _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]),
-    )
-
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    patch_get_signal(freqtradebot)
-    rpc = RPC(freqtradebot)
-
-    freqtradebot.state = State.RUNNING
-    with pytest.raises(RPCException, match=r'.*no active trade*'):
-        rpc._rpc_trade_status()
-
-    freqtradebot.enter_positions()
-
-    # Open order...
-    results = rpc._rpc_trade_status()
-    assert results[0] == {
-        'trade_id': 1,
-        'pair': 'ETH/BTC',
-        'base_currency': 'ETH',
-        'quote_currency': 'BTC',
-        'open_date': ANY,
-        'open_timestamp': ANY,
-        'is_open': ANY,
-        'fee_open': ANY,
-        'fee_open_cost': ANY,
-        'fee_open_currency': ANY,
-        'fee_close': fee.return_value,
-        'fee_close_cost': ANY,
-        'fee_close_currency': ANY,
-        'open_rate_requested': ANY,
-        'open_trade_value': 0.0010025,
-        'close_rate_requested': ANY,
-        'sell_reason': ANY,
-        'exit_reason': ANY,
-        'exit_order_status': ANY,
-        'min_rate': ANY,
-        'max_rate': ANY,
-        'strategy': ANY,
-        'buy_tag': ANY,
-        'enter_tag': ANY,
-        'timeframe': 5,
-        'open_order_id': ANY,
-        'close_date': None,
-        'close_timestamp': None,
-        'open_rate': 1.098e-05,
-        'close_rate': None,
-        'current_rate': 1.099e-05,
-        'amount': 91.07468124,
-        'amount_requested': 91.07468124,
-        'stake_amount': 0.001,
-        'trade_duration': None,
-        'trade_duration_s': None,
-        'close_profit': None,
-        'close_profit_pct': None,
-        'close_profit_abs': None,
-        'current_profit': 0.0,
-        'current_profit_pct': 0.0,
-        'current_profit_abs': 0.0,
-        'profit_ratio': 0.0,
-        'profit_pct': 0.0,
-        'profit_abs': 0.0,
-        'profit_fiat': ANY,
-        'stop_loss_abs': 0.0,
-        'stop_loss_pct': None,
-        'stop_loss_ratio': None,
-        'stoploss_order_id': None,
-        'stoploss_last_update': ANY,
-        'stoploss_last_update_timestamp': ANY,
-        'initial_stop_loss_abs': 0.0,
-        'initial_stop_loss_pct': None,
-        'initial_stop_loss_ratio': None,
-        'stoploss_current_dist': -1.099e-05,
-        'stoploss_current_dist_ratio': -1.0,
-        'stoploss_current_dist_pct': pytest.approx(-100.0),
-        'stoploss_entry_dist': -0.0010025,
-        'stoploss_entry_dist_ratio': -1.0,
-        'open_order': '(limit buy rem=91.07468123)',
-        'realized_profit': 0.0,
-        'exchange': 'binance',
-        'leverage': 1.0,
-        'interest_rate': 0.0,
-        'liquidation_price': None,
-        'is_short': False,
-        'funding_fees': 0.0,
-        'trading_mode': TradingMode.SPOT,
-        'orders': [{
-            'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05,
-            'cost': 0.0009999999999054, 'filled': 0.0, 'ft_order_side': 'buy',
-            'order_date': ANY, 'order_timestamp': ANY, 'order_filled_date': ANY,
-            'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05,
-            'is_open': True, 'pair': 'ETH/BTC', 'order_id': ANY,
-            'remaining': 91.07468123, 'status': ANY, 'ft_is_entry': True,
-        }],
-    }
-
-    # Fill open order ...
-    freqtradebot.manage_open_orders()
-    trades = Trade.get_open_trades()
-    freqtradebot.exit_positions(trades)
-
-    results = rpc._rpc_trade_status()
-    assert results[0] == {
         'trade_id': 1,
         'pair': 'ETH/BTC',
         'base_currency': 'ETH',
@@ -213,91 +108,103 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
             'remaining': ANY, 'status': ANY, 'ft_is_entry': True,
         }],
     }
+    mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=ticker,
+        get_fee=fee,
+        _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]),
+    )
+
+    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    patch_get_signal(freqtradebot)
+    rpc = RPC(freqtradebot)
+
+    freqtradebot.state = State.RUNNING
+    with pytest.raises(RPCException, match=r'.*no active trade*'):
+        rpc._rpc_trade_status()
+
+    freqtradebot.enter_positions()
+
+    # Open order...
+    results = rpc._rpc_trade_status()
+    response_unfilled = deepcopy(gen_response)
+    # Different from "filled" response:
+    response_unfilled.update({
+        'amount': 91.07468124,
+        'profit_ratio': 0.0,
+        'profit_pct': 0.0,
+        'profit_abs': 0.0,
+        'current_profit': 0.0,
+        'current_profit_pct': 0.0,
+        'current_profit_abs': 0.0,
+        'stop_loss_abs': 0.0,
+        'stop_loss_pct': None,
+        'stop_loss_ratio': None,
+        'stoploss_current_dist': -1.099e-05,
+        'stoploss_current_dist_ratio': -1.0,
+        'stoploss_current_dist_pct': pytest.approx(-100.0),
+        'stoploss_entry_dist': -0.0010025,
+        'stoploss_entry_dist_ratio': -1.0,
+        'initial_stop_loss_abs': 0.0,
+        'initial_stop_loss_pct': None,
+        'initial_stop_loss_ratio': None,
+        'open_order': '(limit buy rem=91.07468123)',
+    })
+    response_unfilled['orders'][0].update({
+        'is_open': True,
+        'filled': 0.0,
+        'remaining': 91.07468123
+    })
+    assert results[0] == response_unfilled
+
+    # Open order without remaining
+    trade = Trade.get_open_trades()[0]
+    # kucoin case (no remaining set).
+    trade.orders[0].remaining = None
+    Trade.commit()
+
+    results = rpc._rpc_trade_status()
+    # Reuse above object, only remaining changed.
+    response_unfilled['orders'][0].update({
+        'remaining': None
+    })
+    assert results[0] == response_unfilled
+
+    trade = Trade.get_open_trades()[0]
+    trade.orders[0].remaining = trade.amount
+    Trade.commit()
+
+    # Fill open order ...
+    freqtradebot.manage_open_orders()
+    trades = Trade.get_open_trades()
+    freqtradebot.exit_positions(trades)
+
+    results = rpc._rpc_trade_status()
+
+    response = deepcopy(gen_response)
+    assert results[0] == response
 
     mocker.patch('freqtrade.exchange.Exchange.get_rate',
                  MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available")))
     results = rpc._rpc_trade_status()
     assert isnan(results[0]['current_profit'])
     assert isnan(results[0]['current_rate'])
-    assert results[0] == {
+    response_norate = deepcopy(gen_response)
-        'trade_id': 1,
+    # Update elements that are NaN when no rate is available.
-        'pair': 'ETH/BTC',
+    response_norate.update({
-        'base_currency': 'ETH',
-        'quote_currency': 'BTC',
-        'open_date': ANY,
-        'open_timestamp': ANY,
-        'is_open': ANY,
-        'fee_open': ANY,
-        'fee_open_cost': ANY,
-        'fee_open_currency': ANY,
-        'fee_close': fee.return_value,
-        'fee_close_cost': ANY,
-        'fee_close_currency': ANY,
-        'open_rate_requested': ANY,
-        'open_trade_value': ANY,
-        'close_rate_requested': ANY,
-        'sell_reason': ANY,
-        'exit_reason': ANY,
-        'exit_order_status': ANY,
-        'min_rate': ANY,
-        'max_rate': ANY,
-        'strategy': ANY,
-        'buy_tag': ANY,
-        'enter_tag': ANY,
-        'timeframe': ANY,
-        'open_order_id': ANY,
-        'close_date': None,
-        'close_timestamp': None,
-        'open_rate': 1.098e-05,
-        'close_rate': None,
-        'current_rate': ANY,
-        'amount': 91.07468123,
-        'amount_requested': 91.07468124,
-        'trade_duration': ANY,
-        'trade_duration_s': ANY,
-        'stake_amount': 0.001,
-        'close_profit': None,
-        'close_profit_pct': None,
-        'close_profit_abs': None,
-        'current_profit': ANY,
-        'current_profit_pct': ANY,
-        'current_profit_abs': ANY,
-        'profit_ratio': ANY,
-        'profit_pct': ANY,
-        'profit_abs': ANY,
-        'profit_fiat': ANY,
-        'stop_loss_abs': 9.89e-06,
-        'stop_loss_pct': -10.0,
-        'stop_loss_ratio': -0.1,
-        'stoploss_order_id': None,
-        'stoploss_last_update': ANY,
-        'stoploss_last_update_timestamp': ANY,
-        'initial_stop_loss_abs': 9.89e-06,
-        'initial_stop_loss_pct': -10.0,
-        'initial_stop_loss_ratio': -0.1,
         'stoploss_current_dist': ANY,
         'stoploss_current_dist_ratio': ANY,
         'stoploss_current_dist_pct': ANY,
-        'stoploss_entry_dist': -0.00010402,
+        'profit_ratio': ANY,
-        'stoploss_entry_dist_ratio': -0.10376381,
+        'profit_pct': ANY,
-        'open_order': None,
+        'profit_abs': ANY,
-        'exchange': 'binance',
+        'current_profit_abs': ANY,
-        'realized_profit': 0.0,
+        'current_profit': ANY,
-        'leverage': 1.0,
+        'current_profit_pct': ANY,
-        'interest_rate': 0.0,
+        'current_rate': ANY,
-        'liquidation_price': None,
+    })
-        'is_short': False,
+    assert results[0] == response_norate
-        'funding_fees': 0.0,
-        'trading_mode': TradingMode.SPOT,
-        'orders': [{
-            'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05,
-            'cost': 0.0009999999999054, 'filled': 91.07468123, 'ft_order_side': 'buy',
-            'order_date': ANY, 'order_timestamp': ANY, 'order_filled_date': ANY,
-            'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05,
-            'is_open': False, 'pair': 'ETH/BTC', 'order_id': ANY,
-            'remaining': ANY, 'status': ANY, 'ft_is_entry': True,
-        }],
-    }
 
 
 def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: