Merge pull request #4887 from freqtrade/timerange_noarrow

Don't use Arrow to get min/max backtest dates

freqtrade/configuration/timerange.py

@@ -3,6 +3,7 @@ This module contains the argument manager class
 """
 import logging
 import re
+from datetime import datetime
 from typing import Optional
 
 import arrow
@@ -43,7 +44,7 @@ class TimeRange:
             self.startts = self.startts - seconds
 
     def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
-                                  min_date: arrow.Arrow) -> None:
+                                  min_date: datetime) -> None:
         """
         Adjust startts by <startup_candles> candles.
         Applies only if no startup-candles have been available.
@@ -54,11 +55,11 @@ class TimeRange:
         :return: None (Modifies the object in place)
         """
         if (not self.starttype or (startup_candles
-                                   and min_date.int_timestamp >= self.startts)):
+                                   and min_date.timestamp() >= self.startts)):
             # If no startts was defined, or backtest-data starts at the defined backtest-date
             logger.warning("Moving start-date by %s candles to account for startup time.",
                            startup_candles)
-            self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles)
+            self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
             self.starttype = 'date'
 
     @staticmethod

freqtrade/data/history/history_utils.py

@@ -367,7 +367,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
                 logger.exception(f'Could not convert {pair} to OHLCV.')
 
 
-def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
+def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
     """
     Get the maximum common timerange for the given backtest data.
 
@@ -375,7 +375,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
     :return: tuple containing min_date, max_date
     """
     timeranges = [
-        (arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
+        (frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime())
         for frame in data.values()
     ]
     return (min(timeranges, key=operator.itemgetter(0))[0],

freqtrade/optimize/backtesting.py

@@ -9,7 +9,7 @@ from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Tuple
 
-from pandas import DataFrame
+from pandas import DataFrame, NaT
 
 from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
 from freqtrade.constants import DATETIME_PRINT_FORMAT
@@ -159,7 +159,7 @@ class Backtesting:
 
         logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
+                    f'({(max_date - min_date).days} days).')
 
         # Adjust startts forward if not enough data is available
         timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
@@ -449,15 +449,17 @@ class Backtesting:
             preprocessed[pair] = trim_dataframe(df, timerange,
                                                 startup_candles=self.required_startup)
         min_date, max_date = history.get_timerange(preprocessed)
-
+        if min_date is NaT or max_date is NaT:
+            raise OperationalException(
+                "No data left after adjusting for startup candles. ")
         logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                     f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
-                    f'({(max_date - min_date).days} days)..')
+                    f'({(max_date - min_date).days} days).')
         # Execute backtest and store results
         results = self.backtest(
             processed=preprocessed,
-            start_date=min_date.datetime,
+            start_date=min_date,
-            end_date=max_date.datetime,
+            end_date=max_date,
             max_open_trades=max_open_trades,
             position_stacking=self.config.get('position_stacking', False),
             enable_protections=self.config.get('enable_protections', False),

freqtrade/optimize/hyperopt.py

@@ -273,8 +273,8 @@ class Hyperopt:
 
         bt_results = self.backtesting.backtest(
             processed=processed,
-            start_date=self.min_date.datetime,
+            start_date=self.min_date,
-            end_date=self.max_date.datetime,
+            end_date=self.max_date,
             max_open_trades=self.max_open_trades,
             position_stacking=self.position_stacking,
             enable_protections=self.config.get('enable_protections', False),
@@ -314,7 +314,7 @@ class Hyperopt:
         if trade_count >= self.config['hyperopt_min_trades']:
             loss = self.calculate_loss(results=backtesting_results['results'],
                                        trade_count=trade_count,
-                                       min_date=min_date.datetime, max_date=max_date.datetime,
+                                       min_date=min_date, max_date=max_date,
                                        config=self.config, processed=processed)
         return {
             'loss': loss,

freqtrade/optimize/optimize_reports.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, List, Union
 
-from arrow import Arrow
 from numpy import int64
 from pandas import DataFrame
 from tabulate import tabulate
@@ -259,7 +258,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
 def generate_strategy_stats(btdata: Dict[str, DataFrame],
                             strategy: str,
                             content: Dict[str, Any],
-                            min_date: Arrow, max_date: Arrow,
+                            min_date: datetime, max_date: datetime,
                             market_change: float
                             ) -> Dict[str, Any]:
     """
@@ -314,10 +313,10 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
         'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
         'profit_total': results['profit_abs'].sum() / starting_balance,
         'profit_total_abs': results['profit_abs'].sum(),
-        'backtest_start': min_date.datetime,
+        'backtest_start': min_date,
-        'backtest_start_ts': min_date.int_timestamp * 1000,
+        'backtest_start_ts': int(min_date.timestamp() * 1000),
-        'backtest_end': max_date.datetime,
+        'backtest_end': max_date,
-        'backtest_end_ts': max_date.int_timestamp * 1000,
+        'backtest_end_ts': int(max_date.timestamp() * 1000),
         'backtest_days': backtest_days,
 
         'backtest_run_start_ts': content['backtest_start_time'],
@@ -397,7 +396,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
 
 def generate_backtest_stats(btdata: Dict[str, DataFrame],
                             all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
-                            min_date: Arrow, max_date: Arrow
+                            min_date: datetime, max_date: datetime
                             ) -> Dict[str, Any]:
     """
     :param btdata: Backtest data

tests/optimize/test_backtesting.py

@@ -366,7 +366,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
     # check the logs, that will contain the backtest result
     exists = [
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:59:00 (0 days)..'
+        'up to 2017-11-14 22:59:00 (0 days).'
     ]
     for line in exists:
         assert log_has(line, caplog)
@@ -791,9 +791,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days).',
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days).',
         'Parameter --enable-position-stacking detected ...'
     ]
 
@@ -864,9 +864,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days).',
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days).',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',
         'Running backtesting for Strategy TestStrategyLegacy',
@@ -960,9 +960,9 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
         'Parameter --timerange detected: 1510694220-1510700340 ...',
         f'Using data directory: {testdatadir} ...',
         'Loading data from 2017-11-14 20:57:00 '
-        'up to 2017-11-14 22:58:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days).',
         'Backtesting with data from 2017-11-14 21:17:00 '
-        'up to 2017-11-14 22:58:00 (0 days)..',
+        'up to 2017-11-14 22:58:00 (0 days).',
         'Parameter --enable-position-stacking detected ...',
         'Running backtesting for Strategy DefaultStrategy',
         'Running backtesting for Strategy TestStrategyLegacy',