From 118ae8a3d0248603ff4cf203de956b196d5756d6 Mon Sep 17 00:00:00 2001 From: Joe Schr Date: Tue, 1 Feb 2022 19:34:50 +0100 Subject: [PATCH 1/5] Fix api_schemas/json_encoders by manually converting NaT values to empty Strings makes import of datetime columns more robust by first checking if value is null because strftime can't handle NaT values use `isnull()` because it handles all NaN/None/NaT cases --- freqtrade/rpc/api_server/api_schemas.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index c280f453c..b0ffeee7e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,4 +1,5 @@ from datetime import date, datetime +from pandas import isnull from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel @@ -355,7 +356,9 @@ class PairHistory(BaseModel): class Config: json_encoders = { - datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT), + datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT) + # needed for aslong NaT doesn't work with strftime + if not isnull(v) else "", } From 926b01798138331877c6a004e8476e9af26f9dfa Mon Sep 17 00:00:00 2001 From: Joe Schr Date: Tue, 8 Feb 2022 16:42:39 +0100 Subject: [PATCH 2/5] Fix freqUI charts not displaying when dtype(datetime) column has NaT values fix dataframe_to_dict() issues by replacing NaT empty string and prepare for proper `.replace({NaT})` fix --- freqtrade/rpc/rpc.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8e122d74d..97614a7b1 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -12,7 +12,7 @@ import psutil from dateutil.relativedelta import relativedelta from dateutil.tz import tzlocal from numpy import NAN, inf, int64, mean -from pandas import DataFrame +from pandas import DataFrame, NaT, isnull from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange @@ -963,6 +963,24 @@ class RPC: sell_mask = (dataframe['sell'] == 1) sell_signals = int(sell_mask.sum()) dataframe.loc[sell_mask, '_sell_signal_close'] = dataframe.loc[sell_mask, 'close'] + """ + band-aid until this is fixed: + https://github.com/pandas-dev/pandas/issues/45836 + """ + datetime_types = ['datetime', 'datetime64', 'datetime64[ns, UTC]'] + date_columns = dataframe.select_dtypes(include=datetime_types) + for date_column in date_columns: + # replace NaT with empty string, + # because if replaced with `None` + # it will be casted into NaT again + dataframe[date_column] = dataframe[date_column].apply( + lambda x: '' if isnull(x) else x) + + """ + try this if above pandas Issue#45836 is fixed: + https://github.com/pandas-dev/pandas/issues/45836 + """ + # dataframe = dataframe.replace({NaT: None}) dataframe = dataframe.replace([inf, -inf], NAN) dataframe = dataframe.replace({NAN: None}) From 6191288ff942c3037c8ab0ff4cf8b71d49c9627d Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Feb 2022 06:36:17 +0100 Subject: [PATCH 3/5] Add test for NaT problematic --- freqtrade/rpc/api_server/api_schemas.py | 3 --- freqtrade/rpc/rpc.py | 2 +- tests/rpc/test_rpc_apiserver.py | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index b0ffeee7e..6f358155e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,5 +1,4 @@ from datetime import date, datetime -from pandas import isnull from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel @@ -357,8 +356,6 @@ class PairHistory(BaseModel): class Config: json_encoders = { datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT) - # needed for aslong NaT doesn't work with strftime - if not isnull(v) else "", } diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 97614a7b1..0ca105d6b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -12,7 +12,7 @@ import psutil from dateutil.relativedelta import relativedelta from dateutil.tz import tzlocal from numpy import NAN, inf, int64, mean -from pandas import DataFrame, NaT, isnull +from pandas import DataFrame, isnull from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1c33dd928..eabbfc252 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -7,6 +7,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock +import pandas as pd import pytest import uvicorn from fastapi import FastAPI @@ -1181,6 +1182,24 @@ def test_api_pair_candles(botclient, ohlcv_history): 0.7039405, 8.885e-05, 0, 0, 1511686800000, None, None] ]) + ohlcv_history['sell'] = ohlcv_history['sell'].astype('float64') + ohlcv_history.at[0, 'sell'] = float('inf') + ohlcv_history['date1'] = ohlcv_history['date'] + ohlcv_history.at[0, 'date1'] = pd.NaT + + ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history) + rc = client_get(client, + f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") + assert_response(rc) + assert (rc.json()['data'] == + [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, + None, 0, None, '', 1511686200000, None, None], + ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, + 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, '2017-11-26 08:55:00', + 1511686500000, 8.893e-05, None], + ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885e-05, 0, 0.0, '2017-11-26 09:00:00', 1511686800000, None, None] + ]) def test_api_pair_history(botclient, ohlcv_history): From 4e2f06fe9c706b67746c4346649a3ca2999f463b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Feb 2022 06:48:26 +0100 Subject: [PATCH 4/5] Simplify band-aid code --- freqtrade/rpc/rpc.py | 13 +++---------- tests/rpc/test_rpc_apiserver.py | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0ca105d6b..033e32843 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -12,7 +12,7 @@ import psutil from dateutil.relativedelta import relativedelta from dateutil.tz import tzlocal from numpy import NAN, inf, int64, mean -from pandas import DataFrame, isnull +from pandas import DataFrame, NaT from freqtrade import __version__ from freqtrade.configuration.timerange import TimeRange @@ -973,16 +973,9 @@ class RPC: # replace NaT with empty string, # because if replaced with `None` # it will be casted into NaT again - dataframe[date_column] = dataframe[date_column].apply( - lambda x: '' if isnull(x) else x) + dataframe[date_column] = dataframe[date_column].astype(object).replace({NaT: None}) - """ - try this if above pandas Issue#45836 is fixed: - https://github.com/pandas-dev/pandas/issues/45836 - """ - # dataframe = dataframe.replace({NaT: None}) - dataframe = dataframe.replace([inf, -inf], NAN) - dataframe = dataframe.replace({NAN: None}) + dataframe = dataframe.replace({inf: None, -inf: None, NAN: None}) res = { 'pair': pair, diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index eabbfc252..5b19e5e05 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1193,7 +1193,7 @@ def test_api_pair_candles(botclient, ohlcv_history): assert_response(rc) assert (rc.json()['data'] == [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, - None, 0, None, '', 1511686200000, None, None], + None, 0, None, None, 1511686200000, None, None], ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0.0, '2017-11-26 08:55:00', 1511686500000, 8.893e-05, None], From 45c03f1440354b1eaf00044f086cfcbf7a19977f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Feb 2022 07:02:44 +0100 Subject: [PATCH 5/5] Docstrings are evaluated, while comments are not --- freqtrade/rpc/api_server/api_schemas.py | 2 +- freqtrade/rpc/rpc.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 6f358155e..c280f453c 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -355,7 +355,7 @@ class PairHistory(BaseModel): class Config: json_encoders = { - datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT) + datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT), } diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 033e32843..6a9692f21 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -963,10 +963,9 @@ class RPC: sell_mask = (dataframe['sell'] == 1) sell_signals = int(sell_mask.sum()) dataframe.loc[sell_mask, '_sell_signal_close'] = dataframe.loc[sell_mask, 'close'] - """ - band-aid until this is fixed: - https://github.com/pandas-dev/pandas/issues/45836 - """ + + # band-aid until this is fixed: + # https://github.com/pandas-dev/pandas/issues/45836 datetime_types = ['datetime', 'datetime64', 'datetime64[ns, UTC]'] date_columns = dataframe.select_dtypes(include=datetime_types) for date_column in date_columns: