From 3e676dbaa42835a41dc18b4b526da0a9b25ab2a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Nov 2022 16:33:57 +0100 Subject: [PATCH 01/14] Add properties to simplify timerange handling --- freqtrade/configuration/timerange.py | 12 +++++++++++- tests/test_timerange.py | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 6979c8cd1..e152a625c 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -3,7 +3,7 @@ This module contains the argument manager class """ import logging import re -from datetime import datetime +from datetime import datetime, timezone from typing import Optional import arrow @@ -29,6 +29,16 @@ class TimeRange: self.startts: int = startts self.stopts: int = stopts + @property + def startdt(self) -> Optional[datetime]: + if self.startts: + return datetime.fromtimestamp(self.startts, tz=timezone.utc) + + @property + def stopdt(self) -> Optional[datetime]: + if self.stopts: + return datetime.fromtimestamp(self.stopts, tz=timezone.utc) + def __eq__(self, other): """Override the default Equals behavior""" return (self.starttype == other.starttype and self.stoptype == other.stoptype diff --git a/tests/test_timerange.py b/tests/test_timerange.py index dcdaad09d..07fad5d68 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -1,4 +1,6 @@ # pragma pylint: disable=missing-docstring, C0103 +from datetime import datetime, timezone + import arrow import pytest @@ -18,6 +20,10 @@ def test_parse_timerange_incorrect(): assert TimeRange(None, 'date', 0, 1233360000) == TimeRange.parse_timerange('-1233360000') timerange = TimeRange.parse_timerange('1231006505-1233360000') assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange + assert isinstance(timerange.startdt, datetime) + assert isinstance(timerange.stopdt, datetime) + assert timerange.startdt == datetime.fromtimestamp(1231006505, tz=timezone.utc) + assert timerange.stopdt == datetime.fromtimestamp(1233360000, tz=timezone.utc) timerange = TimeRange.parse_timerange('1231006505000-1233360000000') assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange @@ -45,6 +51,7 @@ def test_subtract_start(): x = TimeRange(None, 'date', 0, 1438214400) x.subtract_start(300) assert not x.startts + assert not x.startdt x = TimeRange('date', None, 1274486400, 0) x.subtract_start(300) From 57313dd9611e159f90c5f74b9fcc771a77aa1c35 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Nov 2022 18:11:39 +0100 Subject: [PATCH 02/14] Update some usages of timerange to new, simplified method --- freqtrade/configuration/timerange.py | 2 ++ freqtrade/data/converter.py | 7 ++----- freqtrade/data/history/history_utils.py | 6 +++--- freqtrade/data/history/idatahandler.py | 6 ++---- freqtrade/freqai/data_kitchen.py | 18 ++++++++---------- freqtrade/freqai/utils.py | 4 +--- 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index e152a625c..8fcf95b45 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -33,11 +33,13 @@ class TimeRange: def startdt(self) -> Optional[datetime]: if self.startts: return datetime.fromtimestamp(self.startts, tz=timezone.utc) + return None @property def stopdt(self) -> Optional[datetime]: if self.stopts: return datetime.fromtimestamp(self.stopts, tz=timezone.utc) + return None def __eq__(self, other): """Override the default Equals behavior""" diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 98ed15489..718011eb6 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -3,7 +3,6 @@ Functions to convert data from one format to another """ import itertools import logging -from datetime import datetime, timezone from operator import itemgetter from typing import Dict, List @@ -137,11 +136,9 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date', df = df.iloc[startup_candles:, :] else: if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - df = df.loc[df[df_date_col] >= start, :] + df = df.loc[df[df_date_col] >= timerange.startdt, :] if timerange.stoptype == 'date': - stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) - df = df.loc[df[df_date_col] <= stop, :] + df = df.loc[df[df_date_col] <= timerange.stopdt, :] return df diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 93534e919..9a206baa4 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -1,6 +1,6 @@ import logging import operator -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -160,9 +160,9 @@ def _load_cached_data_for_updating( end = None if timerange: if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + start = timerange.startdt if timerange.stoptype == 'date': - end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + end = timerange.stopdt # Intentionally don't pass timerange in - since we need to load the full dataset. data = data_handler.ohlcv_load(pair, timeframe=timeframe, diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index b82d2055b..57441b4be 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -366,13 +366,11 @@ class IDataHandler(ABC): """ if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - if pairdata.iloc[0]['date'] > start: + if pairdata.iloc[0]['date'] > timerange.startdt: logger.warning(f"{pair}, {candle_type}, {timeframe}, " f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}") if timerange.stoptype == 'date': - stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) - if pairdata.iloc[-1]['date'] < stop: + if pairdata.iloc[-1]['date'] < timerange.stopdt: logger.warning(f"{pair}, {candle_type}, {timeframe}, " f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 12a3cd519..5e1238884 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -432,8 +432,8 @@ class FreqaiDataKitchen: timerange_train.stopts = timerange_train.startts + train_period_days first = False - start = datetime.fromtimestamp(timerange_train.startts, tz=timezone.utc) - stop = datetime.fromtimestamp(timerange_train.stopts, tz=timezone.utc) + start = timerange_train.startdt + stop = timerange_train.stopdt tr_training_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")) tr_training_list_timerange.append(copy.deepcopy(timerange_train)) @@ -446,8 +446,8 @@ class FreqaiDataKitchen: if timerange_backtest.stopts > config_timerange.stopts: timerange_backtest.stopts = config_timerange.stopts - start = datetime.fromtimestamp(timerange_backtest.startts, tz=timezone.utc) - stop = datetime.fromtimestamp(timerange_backtest.stopts, tz=timezone.utc) + start = timerange_backtest.startdt + stop = timerange_backtest.stopdt tr_backtesting_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")) tr_backtesting_list_timerange.append(copy.deepcopy(timerange_backtest)) @@ -490,11 +490,9 @@ class FreqaiDataKitchen: it is sliced down to just the present training period. """ - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) - df = df.loc[df["date"] >= start, :] + df = df.loc[df["date"] >= timerange.startdt, :] if not self.live: - df = df.loc[df["date"] < stop, :] + df = df.loc[df["date"] < timerange.stopdt, :] return df @@ -1057,8 +1055,8 @@ class FreqaiDataKitchen: backtest_timerange.startts = ( backtest_timerange.startts - backtest_period_days * SECONDS_IN_DAY ) - start = datetime.fromtimestamp(backtest_timerange.startts, tz=timezone.utc) - stop = datetime.fromtimestamp(backtest_timerange.stopts, tz=timezone.utc) + start = backtest_timerange.startdt + stop = backtest_timerange.stopdt full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d") config_path = Path(self.config["config_files"][0]) diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index e854bcf0b..aa5185075 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -230,7 +230,5 @@ def get_timerange_backtest_live_models(config: Config) -> str: dk = FreqaiDataKitchen(config) models_path = dk.get_full_models_path(config) timerange, _ = dk.get_timerange_and_assets_end_dates_from_ready_models(models_path) - start_date = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - end_date = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) - tr = f"{start_date.strftime('%Y%m%d')}-{end_date.strftime('%Y%m%d')}" + tr = f"{timerange.startdt.strftime('%Y%m%d')}-{timerange.stopdt.strftime('%Y%m%d')}" return tr From 0f9c5f8d41f2ea2674bb6fe8f195f4d9df097f9c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Nov 2022 18:26:14 +0100 Subject: [PATCH 03/14] Simplify timerange handling --- freqtrade/configuration/timerange.py | 35 ++++++++++++++++++++++++++++ freqtrade/freqai/data_kitchen.py | 12 +++------- freqtrade/freqai/freqai_interface.py | 23 ++++-------------- freqtrade/freqai/utils.py | 3 +-- freqtrade/optimize/backtesting.py | 3 +-- tests/test_timerange.py | 12 ++++++++-- 6 files changed, 55 insertions(+), 33 deletions(-) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 8fcf95b45..adc5e65df 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -8,6 +8,7 @@ from typing import Optional import arrow +from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.exceptions import OperationalException @@ -41,6 +42,40 @@ class TimeRange: return datetime.fromtimestamp(self.stopts, tz=timezone.utc) return None + @property + def timerange_str(self) -> str: + """ + Returns a string representation of the timerange as used by parse_timerange. + Follows the format yyyymmdd-yyyymmdd - leaving out the parts that are not set. + """ + start = '' + stop = '' + if startdt := self.startdt: + start = startdt.strftime('%Y%m%d') + if stopdt := self.stopdt: + stop = stopdt.strftime('%Y%m%d') + return f"{start}-{stop}" + + @property + def start_fmt(self) -> str: + """ + Returns a string representation of the start date + """ + val = 'unbounded' + if (startdt := self.startdt) is not None: + val = startdt.strftime(DATETIME_PRINT_FORMAT) + return val + + @property + def stop_fmt(self) -> str: + """ + Returns a string representation of the stop date + """ + val = 'unbounded' + if (stopdt := self.stopdt) is not None: + val = stopdt.strftime(DATETIME_PRINT_FORMAT) + return val + def __eq__(self, other): """Override the default Equals behavior""" return (self.starttype == other.starttype and self.stoptype == other.stoptype diff --git a/freqtrade/freqai/data_kitchen.py b/freqtrade/freqai/data_kitchen.py index 5e1238884..87df7fba0 100644 --- a/freqtrade/freqai/data_kitchen.py +++ b/freqtrade/freqai/data_kitchen.py @@ -432,9 +432,7 @@ class FreqaiDataKitchen: timerange_train.stopts = timerange_train.startts + train_period_days first = False - start = timerange_train.startdt - stop = timerange_train.stopdt - tr_training_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")) + tr_training_list.append(timerange_train.timerange_str) tr_training_list_timerange.append(copy.deepcopy(timerange_train)) # associated backtest period @@ -446,9 +444,7 @@ class FreqaiDataKitchen: if timerange_backtest.stopts > config_timerange.stopts: timerange_backtest.stopts = config_timerange.stopts - start = timerange_backtest.startdt - stop = timerange_backtest.stopdt - tr_backtesting_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")) + tr_backtesting_list.append(timerange_backtest.timerange_str) tr_backtesting_list_timerange.append(copy.deepcopy(timerange_backtest)) # ensure we are predicting on exactly same amount of data as requested by user defined @@ -1055,9 +1051,7 @@ class FreqaiDataKitchen: backtest_timerange.startts = ( backtest_timerange.startts - backtest_period_days * SECONDS_IN_DAY ) - start = backtest_timerange.startdt - stop = backtest_timerange.stopdt - full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d") + full_timerange = backtest_timerange.timerange_str config_path = Path(self.config["config_files"][0]) if not self.full_path.is_dir(): diff --git a/freqtrade/freqai/freqai_interface.py b/freqtrade/freqai/freqai_interface.py index ae123f852..94d471d13 100644 --- a/freqtrade/freqai/freqai_interface.py +++ b/freqtrade/freqai/freqai_interface.py @@ -13,7 +13,7 @@ from numpy.typing import NDArray from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.constants import DATETIME_PRINT_FORMAT, Config +from freqtrade.constants import Config from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_seconds @@ -788,14 +788,8 @@ class IFreqaiModel(ABC): :return: if the data exists or not """ if self.config.get("freqai_backtest_live_models", False) and len(dataframe_backtest) == 0: - tr_backtest_startts_str = datetime.fromtimestamp( - tr_backtest.startts, - tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT) - tr_backtest_stopts_str = datetime.fromtimestamp( - tr_backtest.stopts, - tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT) - logger.info(f"No data found for pair {pair} from {tr_backtest_startts_str} " - f" from {tr_backtest_startts_str} to {tr_backtest_stopts_str}. " + logger.info(f"No data found for pair {pair} from " + f"from { tr_backtest.start_fmt} to {tr_backtest.stop_fmt}. " "Probably more than one training within the same candle period.") return False return True @@ -810,18 +804,11 @@ class IFreqaiModel(ABC): :param pair: the current pair :param total_trains: total trains (total number of slides for the sliding window) """ - tr_train_startts_str = datetime.fromtimestamp( - tr_train.startts, - tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT) - tr_train_stopts_str = datetime.fromtimestamp( - tr_train.stopts, - tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT) - if not self.config.get("freqai_backtest_live_models", False): logger.info( f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs" - f" from {tr_train_startts_str} " - f"to {tr_train_stopts_str}, {train_it}/{total_trains} " + f" from {tr_train.start_fmt} " + f"to {tr_train.stop_fmt}, {train_it}/{total_trains} " "trains" ) # Following methods which are overridden by user made prediction models. diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index aa5185075..b64859f9f 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -230,5 +230,4 @@ def get_timerange_backtest_live_models(config: Config) -> str: dk = FreqaiDataKitchen(config) models_path = dk.get_full_models_path(config) timerange, _ = dk.get_timerange_and_assets_end_dates_from_ready_models(models_path) - tr = f"{timerange.startdt.strftime('%Y%m%d')}-{timerange.stopdt.strftime('%Y%m%d')}" - return tr + return timerange.timerange_str diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3436eac44..bb588119a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1286,8 +1286,7 @@ class Backtesting: def _get_min_cached_backtest_date(self): min_backtest_date = None backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT) - if self.timerange.stopts == 0 or datetime.fromtimestamp( - self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc): + if self.timerange.stopts == 0 or self.timerange.stopdt > datetime.now(tz=timezone.utc): logger.warning('Backtest result caching disabled due to use of open-ended timerange.') elif backtest_cache_age == 'day': min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1) diff --git a/tests/test_timerange.py b/tests/test_timerange.py index 07fad5d68..06ff1983a 100644 --- a/tests/test_timerange.py +++ b/tests/test_timerange.py @@ -10,10 +10,17 @@ from freqtrade.exceptions import OperationalException def test_parse_timerange_incorrect(): - assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-') - assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522') + timerange = TimeRange.parse_timerange('20100522-') + assert TimeRange('date', None, 1274486400, 0) == timerange + assert timerange.timerange_str == '20100522-' + timerange = TimeRange.parse_timerange('-20100522') + assert TimeRange(None, 'date', 0, 1274486400) == timerange + assert timerange.timerange_str == '-20100522' timerange = TimeRange.parse_timerange('20100522-20150730') assert timerange == TimeRange('date', 'date', 1274486400, 1438214400) + assert timerange.timerange_str == '20100522-20150730' + assert timerange.start_fmt == '2010-05-22 00:00:00' + assert timerange.stop_fmt == '2015-07-30 00:00:00' # Added test for unix timestamp - BTC genesis date assert TimeRange('date', None, 1231006505, 0) == TimeRange.parse_timerange('1231006505-') @@ -24,6 +31,7 @@ def test_parse_timerange_incorrect(): assert isinstance(timerange.stopdt, datetime) assert timerange.startdt == datetime.fromtimestamp(1231006505, tz=timezone.utc) assert timerange.stopdt == datetime.fromtimestamp(1233360000, tz=timezone.utc) + assert timerange.timerange_str == '20090103-20090131' timerange = TimeRange.parse_timerange('1231006505000-1233360000000') assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange From 4cece8720a6580f03a1c53a598d713e9c5d0ea58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 07:33:06 +0000 Subject: [PATCH 04/14] Bump mypy from 0.982 to 0.990 Bumps [mypy](https://github.com/python/mypy) from 0.982 to 0.990. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.982...v0.990) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cde38e095..f4575e176 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 flake8==5.0.4 flake8-tidy-imports==4.8.0 -mypy==0.982 +mypy==0.990 pre-commit==2.20.0 pytest==7.2.0 pytest-asyncio==0.20.2 From f27be7ada8c5cd16c957d1831ad85c391c85f886 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Nov 2022 20:52:40 +0100 Subject: [PATCH 05/14] Configure mypy to old behavior based on https://mypy-lang.blogspot.com/2022/11/mypy-0990-released.html release --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8020b0636..2de2c957b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ asyncio_mode = "auto" [tool.mypy] ignore_missing_imports = true +namespace_packages = false +implicit_optional = true warn_unused_ignores = true exclude = [ '^build_helpers\.py$' From 0a702cdd261e8ef3cfff1904dfe880cceba36887 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Nov 2022 20:56:35 +0100 Subject: [PATCH 06/14] Ensure more methods are typechecked --- freqtrade/edge/edge_positioning.py | 4 ++-- freqtrade/freqtradebot.py | 4 ++-- freqtrade/optimize/backtesting.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 45b4cd8f1..4656b7c93 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -392,7 +392,7 @@ class Edge: # Returning a list of pairs in order of "expectancy" return final - def _find_trades_for_stoploss_range(self, df, pair, stoploss_range): + def _find_trades_for_stoploss_range(self, df, pair: str, stoploss_range) -> list: buy_column = df['enter_long'].values sell_column = df['exit_long'].values date_column = df['date'].values @@ -407,7 +407,7 @@ class Edge: return result def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, - ohlc_columns, stoploss, pair): + ohlc_columns, stoploss, pair: str): """ Iterate through ohlc_columns in order to find the next trade Next trade opens from the first buy signal noticed to diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ea7c2f1f9..2e2638126 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -354,7 +354,7 @@ class FreqtradeBot(LoggingMixin): if self.trading_mode == TradingMode.FUTURES: self._schedule.run_pending() - def update_closed_trades_without_assigned_fees(self): + def update_closed_trades_without_assigned_fees(self) -> None: """ Update closed trades without close fees assigned. Only acts when Orders are in the database, otherwise the last order-id is unknown. @@ -379,7 +379,7 @@ class FreqtradeBot(LoggingMixin): stoploss_order=order.ft_order_side == 'stoploss', send_msg=False) - trades: List[Trade] = Trade.get_open_trades_without_assigned_fees() + trades = Trade.get_open_trades_without_assigned_fees() for trade in trades: if trade.is_open and not trade.fee_updated(trade.entry_side): order = trade.select_order(trade.entry_side, False) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3436eac44..427ad121f 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -166,7 +166,7 @@ class Backtesting: PairLocks.use_db = True Trade.use_db = True - def init_backtest_detail(self): + def init_backtest_detail(self) -> None: # Load detail timeframe if specified self.timeframe_detail = str(self.config.get('timeframe_detail', '')) if self.timeframe_detail: From 019577f73d06e5fca1f3b80af33d0aaab3f67d46 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 16 Nov 2022 06:36:26 +0100 Subject: [PATCH 07/14] Temporarily Downgrade cryptography until piwheels has the new wheel available --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bc84cd241..ec8b5ce7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,8 @@ pandas-ta==0.3.14b ccxt==2.1.75 # Pin cryptography for now due to rust build errors with piwheels -cryptography==38.0.3 +cryptography==38.0.1; platform_machine == 'armv7l' +cryptography==38.0.3; platform_machine != 'armv7l' aiohttp==3.8.3 SQLAlchemy==1.4.44 python-telegram-bot==13.14 From 097af973d27608a0dc1f5f9916c81a04610e1308 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 07:10:47 +0100 Subject: [PATCH 08/14] improve RPC testcase to cover open orders --- tests/rpc/test_rpc.py | 86 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 54a4cbe9a..c3a0d539d 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -33,6 +33,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, + _is_dry_limit_order_filled=MagicMock(side_effect=[False, True]), ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -44,6 +45,91 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: rpc._rpc_trade_status() freqtradebot.enter_positions() + + # Open order... + results = rpc._rpc_trade_status() + assert results[0] == { + 'trade_id': 1, + 'pair': 'ETH/BTC', + 'base_currency': 'ETH', + 'quote_currency': 'BTC', + 'open_date': ANY, + 'open_timestamp': ANY, + 'is_open': ANY, + 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, + 'fee_close': fee.return_value, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, + 'open_rate_requested': ANY, + 'open_trade_value': 0.0010025, + 'close_rate_requested': ANY, + 'sell_reason': ANY, + 'exit_reason': ANY, + 'exit_order_status': ANY, + 'min_rate': ANY, + 'max_rate': ANY, + 'strategy': ANY, + 'buy_tag': ANY, + 'enter_tag': ANY, + 'timeframe': 5, + 'open_order_id': ANY, + 'close_date': None, + 'close_timestamp': None, + 'open_rate': 1.098e-05, + 'close_rate': None, + 'current_rate': 1.099e-05, + 'amount': 91.07468124, + 'amount_requested': 91.07468124, + 'stake_amount': 0.001, + 'trade_duration': None, + 'trade_duration_s': None, + 'close_profit': None, + 'close_profit_pct': None, + 'close_profit_abs': None, + 'current_profit': 0.0, + 'current_profit_pct': 0.0, + 'current_profit_abs': 0.0, + 'profit_ratio': 0.0, + 'profit_pct': 0.0, + 'profit_abs': 0.0, + 'profit_fiat': ANY, + 'stop_loss_abs': 0.0, + 'stop_loss_pct': None, + 'stop_loss_ratio': None, + 'stoploss_order_id': None, + 'stoploss_last_update': ANY, + 'stoploss_last_update_timestamp': ANY, + 'initial_stop_loss_abs': 0.0, + 'initial_stop_loss_pct': None, + 'initial_stop_loss_ratio': None, + 'stoploss_current_dist': -1.099e-05, + 'stoploss_current_dist_ratio': -1.0, + 'stoploss_current_dist_pct': pytest.approx(-100.0), + 'stoploss_entry_dist': -0.0010025, + 'stoploss_entry_dist_ratio': -1.0, + 'open_order': '(limit buy rem=91.07468123)', + 'realized_profit': 0.0, + 'exchange': 'binance', + 'leverage': 1.0, + 'interest_rate': 0.0, + 'liquidation_price': None, + 'is_short': False, + 'funding_fees': 0.0, + 'trading_mode': TradingMode.SPOT, + 'orders': [{ + 'amount': 91.07468123, 'average': 1.098e-05, 'safe_price': 1.098e-05, + 'cost': 0.0009999999999054, 'filled': 0.0, 'ft_order_side': 'buy', + 'order_date': ANY, 'order_timestamp': ANY, 'order_filled_date': ANY, + 'order_filled_timestamp': ANY, 'order_type': 'limit', 'price': 1.098e-05, + 'is_open': True, 'pair': 'ETH/BTC', 'order_id': ANY, + 'remaining': 91.07468123, 'status': ANY, 'ft_is_entry': True, + }], + } + + # Fill open order ... + freqtradebot.manage_open_orders() trades = Trade.get_open_trades() freqtradebot.exit_positions(trades) From 93addbe5c3979b6cf2bbe35c0eb8c92b3e0cfc9b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 10:16:38 +0000 Subject: [PATCH 09/14] Improve typechecking --- freqtrade/rpc/api_server/webserver.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index e9a12e4df..6464ae44e 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -2,7 +2,7 @@ import asyncio import logging from ipaddress import IPv4Address from threading import Thread -from typing import Any, Dict +from typing import Any, Dict, Optional import orjson import uvicorn @@ -51,9 +51,9 @@ class ApiServer(RPCHandler): # Exchange - only available in webserver mode. _exchange = None # websocket message queue stuff - _ws_channel_manager = None + _ws_channel_manager: ChannelManager _ws_thread = None - _ws_loop = None + _ws_loop: Optional[asyncio.AbstractEventLoop] = None def __new__(cls, *args, **kwargs): """ @@ -71,7 +71,7 @@ class ApiServer(RPCHandler): return self._standalone: bool = standalone self._server = None - self._ws_queue = None + self._ws_queue: Optional[ThreadedQueue] = None self._ws_background_task = None ApiServer.__initialized = True @@ -186,7 +186,7 @@ class ApiServer(RPCHandler): self._ws_background_task = asyncio.run_coroutine_threadsafe( self._broadcast_queue_data(), loop=self._ws_loop) - async def _broadcast_queue_data(self): + async def _broadcast_queue_data(self) -> None: # Instantiate the queue in this coroutine so it's attached to our loop self._ws_queue = ThreadedQueue() async_queue = self._ws_queue.async_q @@ -210,7 +210,8 @@ class ApiServer(RPCHandler): finally: # Disconnect channels and stop the loop on cancel await self._ws_channel_manager.disconnect_all() - self._ws_loop.stop() + if self._ws_loop: + self._ws_loop.stop() # Avoid adding more items to the queue if they aren't # going to get broadcasted. self._ws_queue = None From afcb86f422b1058057bb94a2b1d450f195eebbfb Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 10:25:51 +0000 Subject: [PATCH 10/14] Improve migration types --- freqtrade/persistence/migrations.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index a54c5570f..edbcd6be3 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -1,5 +1,5 @@ import logging -from typing import List +from typing import List, Optional from sqlalchemy import inspect, select, text, tuple_, update @@ -31,9 +31,9 @@ def get_backup_name(tabs: List[str], backup_prefix: str): return table_back_name -def get_last_sequence_ids(engine, trade_back_name, order_back_name): - order_id: int = None - trade_id: int = None +def get_last_sequence_ids(engine, trade_back_name: str, order_back_name: str): + order_id: Optional[int] = None + trade_id: Optional[int] = None if engine.name == 'postgresql': with engine.begin() as connection: From 0a7f4fd3cc6a4000106f33e50fe1862d759bde8c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 10:36:24 +0000 Subject: [PATCH 11/14] fix httpx test warning --- tests/rpc/test_rpc_apiserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 019b8fc82..969728b6f 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -67,7 +67,7 @@ def botclient(default_conf, mocker): def client_post(client, url, data={}): return client.post(url, - data=data, + content=data, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), 'Origin': 'http://example.com', 'content-type': 'application/json' From 9432bcd0654d4da287723e2d4a990bdbf50ad038 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Nov 2022 19:52:03 +0100 Subject: [PATCH 12/14] Fix telegram error on force_enter exception closes #7727 --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 92a24f024..708a1ce53 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -1062,7 +1062,7 @@ class Telegram(RPCHandler): self._rpc._rpc_force_entry(pair, price, order_side=order_side) except RPCException as e: logger.exception("Forcebuy error!") - self._send_msg(str(e)) + self._send_msg(str(e), ParseMode.HTML) def _force_enter_inline(self, update: Update, _: CallbackContext) -> None: if update.callback_query: From 875e9ab447a6d3d165b13a64f95eb79f94daec74 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 11:59:03 -0700 Subject: [PATCH 13/14] change df serialization to avoid mem leak --- freqtrade/misc.py | 7 +++++-- freqtrade/rpc/api_server/ws/serializer.py | 7 ++++++- scripts/ws_client.py | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 49d33d46f..308f0b32d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -262,7 +262,10 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str: :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - return dataframe.to_json(orient='split') + # https://github.com/pandas-dev/pandas/issues/24889 + # https://github.com/pandas-dev/pandas/issues/40443 + # We need to convert to a dict to avoid mem leak + return dataframe.to_dict(orient='tight') def json_to_dataframe(data: str) -> pandas.DataFrame: @@ -271,7 +274,7 @@ def json_to_dataframe(data: str) -> pandas.DataFrame: :param data: A JSON string :returns: A pandas DataFrame from the JSON string """ - dataframe = pandas.read_json(data, orient='split') + dataframe = pandas.DataFrame.from_dict(data, orient='tight') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) diff --git a/freqtrade/rpc/api_server/ws/serializer.py b/freqtrade/rpc/api_server/ws/serializer.py index 6c402a100..8d06746f7 100644 --- a/freqtrade/rpc/api_server/ws/serializer.py +++ b/freqtrade/rpc/api_server/ws/serializer.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import orjson import rapidjson -from pandas import DataFrame +from pandas import DataFrame, Timestamp from freqtrade.misc import dataframe_to_json, json_to_dataframe from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy @@ -52,6 +52,11 @@ def _json_default(z): '__type__': 'dataframe', '__value__': dataframe_to_json(z) } + # Pandas returns a Timestamp object, we need to + # convert it to a timestamp int (with ms) for orjson + # to handle it + if isinstance(z, Timestamp): + return z.timestamp() * 1e3 raise TypeError diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..ff437e63e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -101,7 +101,7 @@ def json_deserialize(message): :param message: The message to deserialize """ def json_to_dataframe(data: str) -> pandas.DataFrame: - dataframe = pandas.read_json(data, orient='split') + dataframe = pandas.DataFrame.from_dict(data, orient='tight') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) From ce43fa5f431092bff5a23b1586f3381d1ba4cfac Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 12:03:11 -0700 Subject: [PATCH 14/14] small fix to websocketchannel and relay --- freqtrade/rpc/api_server/ws/channel.py | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/freqtrade/rpc/api_server/ws/channel.py b/freqtrade/rpc/api_server/ws/channel.py index 88b4db9ba..4eef738d4 100644 --- a/freqtrade/rpc/api_server/ws/channel.py +++ b/freqtrade/rpc/api_server/ws/channel.py @@ -77,21 +77,24 @@ class WebSocketChannel: # until self.drain_timeout for the relay to drain the outgoing queue # We can't use asyncio.wait_for here because the queue may have been created with a # different eventloop - start = time.time() - while self.queue.full(): - await asyncio.sleep(1) - if (time.time() - start) > self.drain_timeout: + if not self.is_closed(): + start = time.time() + while self.queue.full(): + await asyncio.sleep(1) + if (time.time() - start) > self.drain_timeout: + return False + + # If for some reason the queue is still full, just return False + try: + self.queue.put_nowait(data) + except asyncio.QueueFull: return False - # If for some reason the queue is still full, just return False - try: - self.queue.put_nowait(data) - except asyncio.QueueFull: + # If we got here everything is ok + return True + else: return False - # If we got here everything is ok - return True - async def recv(self): """ Receive data on the wrapped websocket @@ -109,14 +112,14 @@ class WebSocketChannel: Close the WebSocketChannel """ + self._closed.set() + self._relay_task.cancel() + try: await self.raw_websocket.close() except Exception: pass - self._closed.set() - self._relay_task.cancel() - def is_closed(self) -> bool: """ Closed flag