diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 6072e296c..6979c8cd1 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -3,6 +3,7 @@ This module contains the argument manager class """ import logging import re +from datetime import datetime from typing import Optional import arrow @@ -43,7 +44,7 @@ class TimeRange: self.startts = self.startts - seconds def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int, - min_date: arrow.Arrow) -> None: + min_date: datetime) -> None: """ Adjust startts by candles. Applies only if no startup-candles have been available. @@ -54,11 +55,11 @@ class TimeRange: :return: None (Modifies the object in place) """ if (not self.starttype or (startup_candles - and min_date.int_timestamp >= self.startts)): + and min_date.timestamp() >= self.startts)): # If no startts was defined, or backtest-data starts at the defined backtest-date logger.warning("Moving start-date by %s candles to account for startup time.", startup_candles) - self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles) + self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles) self.starttype = 'date' @staticmethod diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 58965abe0..32c7ce7f9 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -367,7 +367,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], logger.exception(f'Could not convert {pair} to OHLCV.') -def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: +def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]: """ Get the maximum common timerange for the given backtest data. @@ -375,7 +375,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] :return: tuple containing min_date, max_date """ timeranges = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + (frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime()) for frame in data.values() ] return (min(timeranges, key=operator.itemgetter(0))[0], diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 54e7b806f..899da03e4 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -9,7 +9,7 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple -from pandas import DataFrame +from pandas import DataFrame, NaT from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT @@ -159,7 +159,7 @@ class Backtesting: logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(max_date - min_date).days} days)..') + f'({(max_date - min_date).days} days).') # Adjust startts forward if not enough data is available timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), @@ -449,15 +449,17 @@ class Backtesting: preprocessed[pair] = trim_dataframe(df, timerange, startup_candles=self.required_startup) min_date, max_date = history.get_timerange(preprocessed) - + if min_date is NaT or max_date is NaT: + raise OperationalException( + "No data left after adjusting for startup candles. ") logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(max_date - min_date).days} days)..') + f'({(max_date - min_date).days} days).') # Execute backtest and store results results = self.backtest( processed=preprocessed, - start_date=min_date.datetime, - end_date=max_date.datetime, + start_date=min_date, + end_date=max_date, max_open_trades=max_open_trades, position_stacking=self.config.get('position_stacking', False), enable_protections=self.config.get('enable_protections', False), diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 639e9fb93..e0a6d50a0 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -273,8 +273,8 @@ class Hyperopt: bt_results = self.backtesting.backtest( processed=processed, - start_date=self.min_date.datetime, - end_date=self.max_date.datetime, + start_date=self.min_date, + end_date=self.max_date, max_open_trades=self.max_open_trades, position_stacking=self.position_stacking, enable_protections=self.config.get('enable_protections', False), @@ -314,7 +314,7 @@ class Hyperopt: if trade_count >= self.config['hyperopt_min_trades']: loss = self.calculate_loss(results=backtesting_results['results'], trade_count=trade_count, - min_date=min_date.datetime, max_date=max_date.datetime, + min_date=min_date, max_date=max_date, config=self.config, processed=processed) return { 'loss': loss, diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 9a60c7c4b..170e19ecc 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union -from arrow import Arrow from numpy import int64 from pandas import DataFrame from tabulate import tabulate @@ -259,7 +258,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: def generate_strategy_stats(btdata: Dict[str, DataFrame], strategy: str, content: Dict[str, Any], - min_date: Arrow, max_date: Arrow, + min_date: datetime, max_date: datetime, market_change: float ) -> Dict[str, Any]: """ @@ -314,10 +313,10 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], 'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0, 'profit_total': results['profit_abs'].sum() / starting_balance, 'profit_total_abs': results['profit_abs'].sum(), - 'backtest_start': min_date.datetime, - 'backtest_start_ts': min_date.int_timestamp * 1000, - 'backtest_end': max_date.datetime, - 'backtest_end_ts': max_date.int_timestamp * 1000, + 'backtest_start': min_date, + 'backtest_start_ts': int(min_date.timestamp() * 1000), + 'backtest_end': max_date, + 'backtest_end_ts': int(max_date.timestamp() * 1000), 'backtest_days': backtest_days, 'backtest_run_start_ts': content['backtest_start_time'], @@ -397,7 +396,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], def generate_backtest_stats(btdata: Dict[str, DataFrame], all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]], - min_date: Arrow, max_date: Arrow + min_date: datetime, max_date: datetime ) -> Dict[str, Any]: """ :param btdata: Backtest data diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 5dfc9dbc3..2ebea564b 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -366,7 +366,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: # check the logs, that will contain the backtest result exists = [ 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:59:00 (0 days)..' + 'up to 2017-11-14 22:59:00 (0 days).' ] for line in exists: assert log_has(line, caplog) @@ -791,9 +791,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...' ] @@ -864,9 +864,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategyLegacy', @@ -960,9 +960,9 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategyLegacy',