Merge pull request #4887 from freqtrade/timerange_noarrow

Don't use Arrow to get min/max backtest dates
This commit is contained in:
Matthias 2021-05-07 06:51:16 +02:00 committed by GitHub
commit 4a7d7a5779
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 30 additions and 28 deletions

View File

@ -3,6 +3,7 @@ This module contains the argument manager class
""" """
import logging import logging
import re import re
from datetime import datetime
from typing import Optional from typing import Optional
import arrow import arrow
@ -43,7 +44,7 @@ class TimeRange:
self.startts = self.startts - seconds self.startts = self.startts - seconds
def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int, def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
min_date: arrow.Arrow) -> None: min_date: datetime) -> None:
""" """
Adjust startts by <startup_candles> candles. Adjust startts by <startup_candles> candles.
Applies only if no startup-candles have been available. Applies only if no startup-candles have been available.
@ -54,11 +55,11 @@ class TimeRange:
:return: None (Modifies the object in place) :return: None (Modifies the object in place)
""" """
if (not self.starttype or (startup_candles if (not self.starttype or (startup_candles
and min_date.int_timestamp >= self.startts)): and min_date.timestamp() >= self.startts)):
# If no startts was defined, or backtest-data starts at the defined backtest-date # If no startts was defined, or backtest-data starts at the defined backtest-date
logger.warning("Moving start-date by %s candles to account for startup time.", logger.warning("Moving start-date by %s candles to account for startup time.",
startup_candles) startup_candles)
self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles) self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
self.starttype = 'date' self.starttype = 'date'
@staticmethod @staticmethod

View File

@ -367,7 +367,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
logger.exception(f'Could not convert {pair} to OHLCV.') logger.exception(f'Could not convert {pair} to OHLCV.')
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
""" """
Get the maximum common timerange for the given backtest data. Get the maximum common timerange for the given backtest data.
@ -375,7 +375,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
:return: tuple containing min_date, max_date :return: tuple containing min_date, max_date
""" """
timeranges = [ timeranges = [
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) (frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime())
for frame in data.values() for frame in data.values()
] ]
return (min(timeranges, key=operator.itemgetter(0))[0], return (min(timeranges, key=operator.itemgetter(0))[0],

View File

@ -9,7 +9,7 @@ from copy import deepcopy
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame from pandas import DataFrame, NaT
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.constants import DATETIME_PRINT_FORMAT
@ -159,7 +159,7 @@ class Backtesting:
logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..') f'({(max_date - min_date).days} days).')
# Adjust startts forward if not enough data is available # Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
@ -449,15 +449,17 @@ class Backtesting:
preprocessed[pair] = trim_dataframe(df, timerange, preprocessed[pair] = trim_dataframe(df, timerange,
startup_candles=self.required_startup) startup_candles=self.required_startup)
min_date, max_date = history.get_timerange(preprocessed) min_date, max_date = history.get_timerange(preprocessed)
if min_date is NaT or max_date is NaT:
raise OperationalException(
"No data left after adjusting for startup candles. ")
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..') f'({(max_date - min_date).days} days).')
# Execute backtest and store results # Execute backtest and store results
results = self.backtest( results = self.backtest(
processed=preprocessed, processed=preprocessed,
start_date=min_date.datetime, start_date=min_date,
end_date=max_date.datetime, end_date=max_date,
max_open_trades=max_open_trades, max_open_trades=max_open_trades,
position_stacking=self.config.get('position_stacking', False), position_stacking=self.config.get('position_stacking', False),
enable_protections=self.config.get('enable_protections', False), enable_protections=self.config.get('enable_protections', False),

View File

@ -273,8 +273,8 @@ class Hyperopt:
bt_results = self.backtesting.backtest( bt_results = self.backtesting.backtest(
processed=processed, processed=processed,
start_date=self.min_date.datetime, start_date=self.min_date,
end_date=self.max_date.datetime, end_date=self.max_date,
max_open_trades=self.max_open_trades, max_open_trades=self.max_open_trades,
position_stacking=self.position_stacking, position_stacking=self.position_stacking,
enable_protections=self.config.get('enable_protections', False), enable_protections=self.config.get('enable_protections', False),
@ -314,7 +314,7 @@ class Hyperopt:
if trade_count >= self.config['hyperopt_min_trades']: if trade_count >= self.config['hyperopt_min_trades']:
loss = self.calculate_loss(results=backtesting_results['results'], loss = self.calculate_loss(results=backtesting_results['results'],
trade_count=trade_count, trade_count=trade_count,
min_date=min_date.datetime, max_date=max_date.datetime, min_date=min_date, max_date=max_date,
config=self.config, processed=processed) config=self.config, processed=processed)
return { return {
'loss': loss, 'loss': loss,

View File

@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Union from typing import Any, Dict, List, Union
from arrow import Arrow
from numpy import int64 from numpy import int64
from pandas import DataFrame from pandas import DataFrame
from tabulate import tabulate from tabulate import tabulate
@ -259,7 +258,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
def generate_strategy_stats(btdata: Dict[str, DataFrame], def generate_strategy_stats(btdata: Dict[str, DataFrame],
strategy: str, strategy: str,
content: Dict[str, Any], content: Dict[str, Any],
min_date: Arrow, max_date: Arrow, min_date: datetime, max_date: datetime,
market_change: float market_change: float
) -> Dict[str, Any]: ) -> Dict[str, Any]:
""" """
@ -314,10 +313,10 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0, 'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
'profit_total': results['profit_abs'].sum() / starting_balance, 'profit_total': results['profit_abs'].sum() / starting_balance,
'profit_total_abs': results['profit_abs'].sum(), 'profit_total_abs': results['profit_abs'].sum(),
'backtest_start': min_date.datetime, 'backtest_start': min_date,
'backtest_start_ts': min_date.int_timestamp * 1000, 'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date.datetime, 'backtest_end': max_date,
'backtest_end_ts': max_date.int_timestamp * 1000, 'backtest_end_ts': int(max_date.timestamp() * 1000),
'backtest_days': backtest_days, 'backtest_days': backtest_days,
'backtest_run_start_ts': content['backtest_start_time'], 'backtest_run_start_ts': content['backtest_start_time'],
@ -397,7 +396,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
def generate_backtest_stats(btdata: Dict[str, DataFrame], def generate_backtest_stats(btdata: Dict[str, DataFrame],
all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]], all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
min_date: Arrow, max_date: Arrow min_date: datetime, max_date: datetime
) -> Dict[str, Any]: ) -> Dict[str, Any]:
""" """
:param btdata: Backtest data :param btdata: Backtest data

View File

@ -366,7 +366,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None:
# check the logs, that will contain the backtest result # check the logs, that will contain the backtest result
exists = [ exists = [
'Backtesting with data from 2017-11-14 21:17:00 ' 'Backtesting with data from 2017-11-14 21:17:00 '
'up to 2017-11-14 22:59:00 (0 days)..' 'up to 2017-11-14 22:59:00 (0 days).'
] ]
for line in exists: for line in exists:
assert log_has(line, caplog) assert log_has(line, caplog)
@ -791,9 +791,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir):
'Parameter --timerange detected: 1510694220-1510700340 ...', 'Parameter --timerange detected: 1510694220-1510700340 ...',
f'Using data directory: {testdatadir} ...', f'Using data directory: {testdatadir} ...',
'Loading data from 2017-11-14 20:57:00 ' 'Loading data from 2017-11-14 20:57:00 '
'up to 2017-11-14 22:58:00 (0 days)..', 'up to 2017-11-14 22:58:00 (0 days).',
'Backtesting with data from 2017-11-14 21:17:00 ' 'Backtesting with data from 2017-11-14 21:17:00 '
'up to 2017-11-14 22:58:00 (0 days)..', 'up to 2017-11-14 22:58:00 (0 days).',
'Parameter --enable-position-stacking detected ...' 'Parameter --enable-position-stacking detected ...'
] ]
@ -864,9 +864,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir):
'Parameter --timerange detected: 1510694220-1510700340 ...', 'Parameter --timerange detected: 1510694220-1510700340 ...',
f'Using data directory: {testdatadir} ...', f'Using data directory: {testdatadir} ...',
'Loading data from 2017-11-14 20:57:00 ' 'Loading data from 2017-11-14 20:57:00 '
'up to 2017-11-14 22:58:00 (0 days)..', 'up to 2017-11-14 22:58:00 (0 days).',
'Backtesting with data from 2017-11-14 21:17:00 ' 'Backtesting with data from 2017-11-14 21:17:00 '
'up to 2017-11-14 22:58:00 (0 days)..', 'up to 2017-11-14 22:58:00 (0 days).',
'Parameter --enable-position-stacking detected ...', 'Parameter --enable-position-stacking detected ...',
'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy DefaultStrategy',
'Running backtesting for Strategy TestStrategyLegacy', 'Running backtesting for Strategy TestStrategyLegacy',
@ -960,9 +960,9 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat
'Parameter --timerange detected: 1510694220-1510700340 ...', 'Parameter --timerange detected: 1510694220-1510700340 ...',
f'Using data directory: {testdatadir} ...', f'Using data directory: {testdatadir} ...',
'Loading data from 2017-11-14 20:57:00 ' 'Loading data from 2017-11-14 20:57:00 '
'up to 2017-11-14 22:58:00 (0 days)..', 'up to 2017-11-14 22:58:00 (0 days).',
'Backtesting with data from 2017-11-14 21:17:00 ' 'Backtesting with data from 2017-11-14 21:17:00 '
'up to 2017-11-14 22:58:00 (0 days)..', 'up to 2017-11-14 22:58:00 (0 days).',
'Parameter --enable-position-stacking detected ...', 'Parameter --enable-position-stacking detected ...',
'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy DefaultStrategy',
'Running backtesting for Strategy TestStrategyLegacy', 'Running backtesting for Strategy TestStrategyLegacy',