reduce code duplication, optimize auto data download per tf
This commit is contained in:
robcaulk
parent
ac42c0153d
commit
4b7e640f31
@ -91,9 +91,9 @@ class DataProvider:
|
||||
timerange = TimeRange.parse_timerange(None if self._config.get(
|
||||
'timerange') is None else str(self._config.get('timerange')))
|
||||
# Move informative start time respecting startup_candle_count
|
||||
timerange.subtract_start(
|
||||
self.get_required_startup_seconds(str(timeframe))
|
||||
)
|
||||
startup_candles = self.get_required_startup(str(timeframe))
|
||||
tf_seconds = timeframe_to_seconds(str(timeframe))
|
||||
timerange.subtract_start(tf_seconds * startup_candles)
|
||||
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
|
||||
pair=pair,
|
||||
timeframe=timeframe or self._config['timeframe'],
|
||||
@ -105,16 +105,18 @@ class DataProvider:
|
||||
)
|
||||
return self.__cached_pairs_backtesting[saved_pair].copy()
|
||||
|
||||
def get_required_startup_seconds(self, timeframe: str) -> int:
|
||||
tf_seconds = timeframe_to_seconds(timeframe)
|
||||
base_seconds = tf_seconds * self._config.get('startup_candle_count', 0)
|
||||
if not self._config['freqai']['enabled']:
|
||||
return base_seconds
|
||||
def get_required_startup(self, timeframe: str) -> int:
|
||||
if not self._config.get('freqai', {}).get('enabled', False):
|
||||
return self._config.get('startup_candle_count', 0)
|
||||
else:
|
||||
train_seconds = self._config['freqai']['train_period_days'] * 86400
|
||||
# multiplied by safety factor of 2 because FreqAI users
|
||||
# typically do not know the correct window.
|
||||
return base_seconds * 2 + int(train_seconds)
|
||||
if not self._config['startup_candle_count']:
|
||||
raise OperationalException('FreqAI backtesting module requires strategy '
|
||||
'set startup_candle_count.')
|
||||
tf_seconds = timeframe_to_seconds(timeframe)
|
||||
train_candles = self._config['freqai']['train_period_days'] * 86400 / tf_seconds
|
||||
total_candles = int(self._config.get('startup_candle_count', 0) + train_candles)
|
||||
logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
|
||||
return total_candles
|
||||
|
||||
def get_pair_dataframe(
|
||||
self,
|
||||
|
||||
@ -1006,8 +1006,7 @@ class FreqaiDataKitchen:
|
||||
# Methods called by interface.py (load_freqai_model())
|
||||
|
||||
|
||||
def download_all_data_for_training(timerange: TimeRange,
|
||||
dp: DataProvider, config: dict) -> None:
|
||||
def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
|
||||
"""
|
||||
Called only once upon start of bot to download the necessary data for
|
||||
populating indicators and training the model.
|
||||
@ -1025,51 +1024,31 @@ def download_all_data_for_training(timerange: TimeRange,
|
||||
|
||||
all_pairs = dynamic_expand_pairlist(config, markets)
|
||||
|
||||
new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
|
||||
if not dp._exchange:
|
||||
# Not realistic - this is only called in live mode.
|
||||
raise OperationalException("Dataprovider did not have an exchange attached.")
|
||||
refresh_backtest_ohlcv_data(
|
||||
dp._exchange,
|
||||
pairs=all_pairs,
|
||||
timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
|
||||
datadir=config["datadir"],
|
||||
timerange=timerange,
|
||||
new_pairs_days=new_pairs_days,
|
||||
erase=False,
|
||||
data_format=config.get("dataformat_ohlcv", "json"),
|
||||
trading_mode=config.get("trading_mode", "spot"),
|
||||
prepend=config.get("prepend_data", False),
|
||||
)
|
||||
|
||||
|
||||
def get_required_data_timerange(
|
||||
config: dict
|
||||
) -> TimeRange:
|
||||
"""
|
||||
Used by interface.py to pre-download necessary data for FreqAI
|
||||
user.
|
||||
"""
|
||||
time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
|
||||
data_load_timerange = TimeRange()
|
||||
|
||||
timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")
|
||||
|
||||
max_tf_seconds = 0
|
||||
for tf in timeframes:
|
||||
secs = timeframe_to_seconds(tf)
|
||||
if secs > max_tf_seconds:
|
||||
max_tf_seconds = secs
|
||||
|
||||
max_period = config.get('startup_candle_count', 20) * 2
|
||||
|
||||
additional_seconds = max_period * max_tf_seconds
|
||||
|
||||
data_load_timerange.startts = int(
|
||||
time
|
||||
- config["freqai"].get("train_period_days", 0) * SECONDS_IN_DAY
|
||||
- additional_seconds
|
||||
)
|
||||
data_load_timerange.stopts = int(time)
|
||||
|
||||
return data_load_timerange
|
||||
for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
|
||||
timerange = TimeRange()
|
||||
timerange.startts = int(time)
|
||||
timerange.stopts = int(time)
|
||||
startup_candles = dp.get_required_startup(str(tf))
|
||||
tf_seconds = timeframe_to_seconds(str(tf))
|
||||
timerange.subtract_start(tf_seconds * startup_candles)
|
||||
new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
|
||||
# FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
|
||||
# redownloads the funding rate for each pair.
|
||||
refresh_backtest_ohlcv_data(
|
||||
dp._exchange,
|
||||
pairs=all_pairs,
|
||||
timeframes=[tf],
|
||||
datadir=config["datadir"],
|
||||
timerange=timerange,
|
||||
new_pairs_days=new_pairs_days,
|
||||
erase=False,
|
||||
data_format=config.get("dataformat_ohlcv", "json"),
|
||||
trading_mode=config.get("trading_mode", "spot"),
|
||||
prepend=config.get("prepend_data", False),
|
||||
)
|
||||
|
||||
@ -211,21 +211,12 @@ class Backtesting:
|
||||
"""
|
||||
self.progress.init_step(BacktestState.DATALOAD, 1)
|
||||
|
||||
# if self.config.get('freqai', {}).get('enabled', False):
|
||||
# startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0))
|
||||
# if not startup_candles:
|
||||
# raise OperationalException('FreqAI backtesting module requires user set '
|
||||
# 'startup_candles in config.')
|
||||
# self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0))
|
||||
# logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}')
|
||||
# self.config['startup_candle_count'] = self.required_startup
|
||||
|
||||
data = history.load_data(
|
||||
datadir=self.config['datadir'],
|
||||
pairs=self.pairlists.whitelist,
|
||||
timeframe=self.timeframe,
|
||||
timerange=self.timerange,
|
||||
startup_candles=self.get_required_startup(self.timeframe),
|
||||
startup_candles=self.dataprovider.get_required_startup(self.timeframe),
|
||||
fail_without_data=True,
|
||||
data_format=self.config.get('dataformat_ohlcv', 'json'),
|
||||
candle_type=self.config.get('candle_type_def', CandleType.SPOT)
|
||||
@ -244,21 +235,6 @@ class Backtesting:
|
||||
self.progress.set_new_value(1)
|
||||
return data, self.timerange
|
||||
|
||||
def get_required_startup(self, timeframe: str) -> int:
|
||||
if not self.config['freqai']['enabled']:
|
||||
return self.required_startup
|
||||
else:
|
||||
if not self.config['startup_candle_count']:
|
||||
raise OperationalException('FreqAI backtesting module requires strategy '
|
||||
'set startup_candle_count.')
|
||||
tf_seconds = timeframe_to_seconds(timeframe)
|
||||
train_candles = self.config['freqai']['train_period_days'] * 86400 / tf_seconds
|
||||
# multiplied by safety factor of 2 because FreqAI users
|
||||
# typically do not know the correct window.
|
||||
total_candles = self.required_startup * 2 + train_candles
|
||||
logger.info(f'Increasing startup_candle_count for freqai to {total_candles}')
|
||||
return total_candles
|
||||
|
||||
def load_bt_data_detail(self) -> None:
|
||||
"""
|
||||
Loads backtest detail data (smaller timeframe) if necessary.
|
||||
|
||||
@ -148,8 +148,7 @@ class IStrategy(ABC, HyperStrategyMixin):
|
||||
def load_freqAI_model(self) -> None:
|
||||
if self.config.get('freqai', {}).get('enabled', False):
|
||||
# Import here to avoid importing this if freqAI is disabled
|
||||
from freqtrade.freqai.data_kitchen import (download_all_data_for_training,
|
||||
get_required_data_timerange)
|
||||
from freqtrade.freqai.data_kitchen import (download_all_data_for_training)
|
||||
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
|
||||
self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
|
||||
self.freqai_info = self.config["freqai"]
|
||||
@ -161,8 +160,8 @@ class IStrategy(ABC, HyperStrategyMixin):
|
||||
"corr_pairlist, this may take a while if you do not have the "
|
||||
"data saved"
|
||||
)
|
||||
data_load_timerange = get_required_data_timerange(self.config)
|
||||
download_all_data_for_training(data_load_timerange, self.dp, self.config)
|
||||
# data_load_timerange = get_required_data_timerange(self.config)
|
||||
download_all_data_for_training(self.dp, self.config)
|
||||
|
||||
else:
|
||||
# Gracious failures if freqAI is disabled but "start" is called.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user