Merge branch 'develop' into fix/pd-mem-leak

freqtrade/configuration/timerange.py

@@ -3,11 +3,12 @@ This module contains the argument manager class
 """
 import logging
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Optional
 
 import arrow
 
+from freqtrade.constants import DATETIME_PRINT_FORMAT
 from freqtrade.exceptions import OperationalException
 
 
@@ -29,6 +30,52 @@ class TimeRange:
         self.startts: int = startts
         self.stopts: int = stopts
 
+    @property
+    def startdt(self) -> Optional[datetime]:
+        if self.startts:
+            return datetime.fromtimestamp(self.startts, tz=timezone.utc)
+        return None
+
+    @property
+    def stopdt(self) -> Optional[datetime]:
+        if self.stopts:
+            return datetime.fromtimestamp(self.stopts, tz=timezone.utc)
+        return None
+
+    @property
+    def timerange_str(self) -> str:
+        """
+        Returns a string representation of the timerange as used by parse_timerange.
+        Follows the format yyyymmdd-yyyymmdd - leaving out the parts that are not set.
+        """
+        start = ''
+        stop = ''
+        if startdt := self.startdt:
+            start = startdt.strftime('%Y%m%d')
+        if stopdt := self.stopdt:
+            stop = stopdt.strftime('%Y%m%d')
+        return f"{start}-{stop}"
+
+    @property
+    def start_fmt(self) -> str:
+        """
+        Returns a string representation of the start date
+        """
+        val = 'unbounded'
+        if (startdt := self.startdt) is not None:
+            val = startdt.strftime(DATETIME_PRINT_FORMAT)
+        return val
+
+    @property
+    def stop_fmt(self) -> str:
+        """
+        Returns a string representation of the stop date
+        """
+        val = 'unbounded'
+        if (stopdt := self.stopdt) is not None:
+            val = stopdt.strftime(DATETIME_PRINT_FORMAT)
+        return val
+
     def __eq__(self, other):
         """Override the default Equals behavior"""
         return (self.starttype == other.starttype and self.stoptype == other.stoptype

freqtrade/data/converter.py

@@ -3,7 +3,6 @@ Functions to convert data from one format to another
 """
 import itertools
 import logging
-from datetime import datetime, timezone
 from operator import itemgetter
 from typing import Dict, List
 
@@ -138,11 +137,9 @@ def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
         df = df.iloc[startup_candles:, :]
     else:
         if timerange.starttype == 'date':
-            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+            df = df.loc[df[df_date_col] >= timerange.startdt, :]
-            df = df.loc[df[df_date_col] >= start, :]
     if timerange.stoptype == 'date':
-        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
+        df = df.loc[df[df_date_col] <= timerange.stopdt, :]
-        df = df.loc[df[df_date_col] <= stop, :]
     return df
 
 

freqtrade/data/history/history_utils.py

@@ -1,6 +1,6 @@
 import logging
 import operator
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
 
@@ -160,9 +160,9 @@ def _load_cached_data_for_updating(
     end = None
     if timerange:
         if timerange.starttype == 'date':
-            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+            start = timerange.startdt
         if timerange.stoptype == 'date':
-            end = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
+            end = timerange.stopdt
 
     # Intentionally don't pass timerange in - since we need to load the full dataset.
     data = data_handler.ohlcv_load(pair, timeframe=timeframe,

freqtrade/data/history/idatahandler.py

@@ -366,13 +366,11 @@ class IDataHandler(ABC):
         """
 
         if timerange.starttype == 'date':
-            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+            if pairdata.iloc[0]['date'] > timerange.startdt:
-            if pairdata.iloc[0]['date'] > start:
                 logger.warning(f"{pair}, {candle_type}, {timeframe}, "
                                f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}")
         if timerange.stoptype == 'date':
-            stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
+            if pairdata.iloc[-1]['date'] < timerange.stopdt:
-            if pairdata.iloc[-1]['date'] < stop:
                 logger.warning(f"{pair}, {candle_type}, {timeframe}, "
                                f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
 

freqtrade/freqai/data_kitchen.py

@@ -433,9 +433,7 @@ class FreqaiDataKitchen:
             timerange_train.stopts = timerange_train.startts + train_period_days
 
             first = False
-            start = datetime.fromtimestamp(timerange_train.startts, tz=timezone.utc)
+            tr_training_list.append(timerange_train.timerange_str)
-            stop = datetime.fromtimestamp(timerange_train.stopts, tz=timezone.utc)
-            tr_training_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d"))
             tr_training_list_timerange.append(copy.deepcopy(timerange_train))
 
             # associated backtest period
@@ -447,9 +445,7 @@ class FreqaiDataKitchen:
             if timerange_backtest.stopts > config_timerange.stopts:
                 timerange_backtest.stopts = config_timerange.stopts
 
-            start = datetime.fromtimestamp(timerange_backtest.startts, tz=timezone.utc)
+            tr_backtesting_list.append(timerange_backtest.timerange_str)
-            stop = datetime.fromtimestamp(timerange_backtest.stopts, tz=timezone.utc)
-            tr_backtesting_list.append(start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d"))
             tr_backtesting_list_timerange.append(copy.deepcopy(timerange_backtest))
 
             # ensure we are predicting on exactly same amount of data as requested by user defined
@@ -491,11 +487,9 @@ class FreqaiDataKitchen:
                    it is sliced down to just the present training period.
         """
 
-        start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+        df = df.loc[df["date"] >= timerange.startdt, :]
-        stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
-        df = df.loc[df["date"] >= start, :]
         if not self.live:
-            df = df.loc[df["date"] < stop, :]
+            df = df.loc[df["date"] < timerange.stopdt, :]
 
         return df
 
@@ -1058,9 +1052,7 @@ class FreqaiDataKitchen:
         backtest_timerange.startts = (
             backtest_timerange.startts - backtest_period_days * SECONDS_IN_DAY
         )
-        start = datetime.fromtimestamp(backtest_timerange.startts, tz=timezone.utc)
+        full_timerange = backtest_timerange.timerange_str
-        stop = datetime.fromtimestamp(backtest_timerange.stopts, tz=timezone.utc)
-        full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")
         config_path = Path(self.config["config_files"][0])
 
         if not self.full_path.is_dir():

freqtrade/freqai/freqai_interface.py

@@ -13,7 +13,7 @@ from numpy.typing import NDArray
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
-from freqtrade.constants import DATETIME_PRINT_FORMAT, Config
+from freqtrade.constants import Config
 from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_seconds
@@ -788,14 +788,8 @@ class IFreqaiModel(ABC):
         :return: if the data exists or not
         """
         if self.config.get("freqai_backtest_live_models", False) and len(dataframe_backtest) == 0:
-            tr_backtest_startts_str = datetime.fromtimestamp(
+            logger.info(f"No data found for pair {pair} from "
-                                            tr_backtest.startts,
+                        f"from { tr_backtest.start_fmt} to {tr_backtest.stop_fmt}. "
-                                            tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
-            tr_backtest_stopts_str = datetime.fromtimestamp(
-                                            tr_backtest.stopts,
-                                            tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
-            logger.info(f"No data found for pair {pair} from {tr_backtest_startts_str} "
-                        f" from {tr_backtest_startts_str} to {tr_backtest_stopts_str}. "
                         "Probably more than one training within the same candle period.")
             return False
         return True
@@ -810,18 +804,11 @@ class IFreqaiModel(ABC):
         :param pair: the current pair
         :param total_trains: total trains (total number of slides for the sliding window)
         """
-        tr_train_startts_str = datetime.fromtimestamp(
-                                            tr_train.startts,
-                                            tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
-        tr_train_stopts_str = datetime.fromtimestamp(
-                                            tr_train.stopts,
-                                            tz=timezone.utc).strftime(DATETIME_PRINT_FORMAT)
-
         if not self.config.get("freqai_backtest_live_models", False):
             logger.info(
                 f"Training {pair}, {self.pair_it}/{self.total_pairs} pairs"
-                f" from {tr_train_startts_str} "
+                f" from {tr_train.start_fmt} "
-                f"to {tr_train_stopts_str}, {train_it}/{total_trains} "
+                f"to {tr_train.stop_fmt}, {train_it}/{total_trains} "
                 "trains"
             )
     # Following methods which are overridden by user made prediction models.

freqtrade/freqai/utils.py

@@ -230,7 +230,4 @@ def get_timerange_backtest_live_models(config: Config) -> str:
     dk = FreqaiDataKitchen(config)
     models_path = dk.get_full_models_path(config)
     timerange, _ = dk.get_timerange_and_assets_end_dates_from_ready_models(models_path)
-    start_date = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+    return timerange.timerange_str
-    end_date = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
-    tr = f"{start_date.strftime('%Y%m%d')}-{end_date.strftime('%Y%m%d')}"
-    return tr

freqtrade/optimize/backtesting.py

@@ -1286,8 +1286,7 @@ class Backtesting:
     def _get_min_cached_backtest_date(self):
         min_backtest_date = None
         backtest_cache_age = self.config.get('backtest_cache', constants.BACKTEST_CACHE_DEFAULT)
-        if self.timerange.stopts == 0 or datetime.fromtimestamp(
+        if self.timerange.stopts == 0 or self.timerange.stopdt > datetime.now(tz=timezone.utc):
-           self.timerange.stopts, tz=timezone.utc) > datetime.now(tz=timezone.utc):
             logger.warning('Backtest result caching disabled due to use of open-ended timerange.')
         elif backtest_cache_age == 'day':
             min_backtest_date = datetime.now(tz=timezone.utc) - timedelta(days=1)

freqtrade/rpc/telegram.py

@@ -1062,7 +1062,7 @@ class Telegram(RPCHandler):
                 self._rpc._rpc_force_entry(pair, price, order_side=order_side)
             except RPCException as e:
                 logger.exception("Forcebuy error!")
-                self._send_msg(str(e))
+                self._send_msg(str(e), ParseMode.HTML)
 
     def _force_enter_inline(self, update: Update, _: CallbackContext) -> None:
         if update.callback_query:

tests/test_timerange.py

@@ -1,4 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
+from datetime import datetime, timezone
+
 import arrow
 import pytest
 
@@ -8,16 +10,28 @@ from freqtrade.exceptions import OperationalException
 
 def test_parse_timerange_incorrect():
 
-    assert TimeRange('date', None, 1274486400, 0) == TimeRange.parse_timerange('20100522-')
+    timerange = TimeRange.parse_timerange('20100522-')
-    assert TimeRange(None, 'date', 0, 1274486400) == TimeRange.parse_timerange('-20100522')
+    assert TimeRange('date', None, 1274486400, 0) == timerange
+    assert timerange.timerange_str == '20100522-'
+    timerange = TimeRange.parse_timerange('-20100522')
+    assert TimeRange(None, 'date', 0, 1274486400) == timerange
+    assert timerange.timerange_str == '-20100522'
     timerange = TimeRange.parse_timerange('20100522-20150730')
     assert timerange == TimeRange('date', 'date', 1274486400, 1438214400)
+    assert timerange.timerange_str == '20100522-20150730'
+    assert timerange.start_fmt == '2010-05-22 00:00:00'
+    assert timerange.stop_fmt == '2015-07-30 00:00:00'
 
     # Added test for unix timestamp - BTC genesis date
     assert TimeRange('date', None, 1231006505, 0) == TimeRange.parse_timerange('1231006505-')
     assert TimeRange(None, 'date', 0, 1233360000) == TimeRange.parse_timerange('-1233360000')
     timerange = TimeRange.parse_timerange('1231006505-1233360000')
     assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange
+    assert isinstance(timerange.startdt, datetime)
+    assert isinstance(timerange.stopdt, datetime)
+    assert timerange.startdt == datetime.fromtimestamp(1231006505, tz=timezone.utc)
+    assert timerange.stopdt == datetime.fromtimestamp(1233360000, tz=timezone.utc)
+    assert timerange.timerange_str == '20090103-20090131'
 
     timerange = TimeRange.parse_timerange('1231006505000-1233360000000')
     assert TimeRange('date', 'date', 1231006505, 1233360000) == timerange
@@ -45,6 +59,7 @@ def test_subtract_start():
     x = TimeRange(None, 'date', 0, 1438214400)
     x.subtract_start(300)
     assert not x.startts
+    assert not x.startdt
 
     x = TimeRange('date', None, 1274486400, 0)
     x.subtract_start(300)