adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ensure spice_rack is backtestable. Ensure download-data knows about the spice_rack informative pair requirements

Browse Source
This commit is contained in:
robcaulk
2022-09-18 18:40:03 +02:00
parent 91e2a05aff
commit 7b390b8edb
4 changed files with 68 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
freqtrade/commands/data_commands.py
View File

@@ -12,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
from freqtrade.enums import CandleType, RunMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import market_is_active, timeframe_to_minutes
from freqtrade.freqai.utils import setup_freqai_spice_rack
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
from freqtrade.resolvers import ExchangeResolver
@@ -48,6 +49,10 @@ def start_download_data(args: Dict[str, Any]) -> None:
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
if config.get('freqai_spice_rack', False):
config = setup_freqai_spice_rack(config, exchange)
markets = [p for p, m in exchange.markets.items() if market_is_active(m)
or config.get('include_inactive')]

57
freqtrade/freqai/utils.py
View File

@@ -1,5 +1,6 @@
import logging
from datetime import datetime, timezone
from typing import Any, Dict, Optional
import numpy as np
# for spice rack
@@ -12,7 +13,7 @@ from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.exchange import Exchange, timeframe_to_seconds
from freqtrade.exchange.exchange import market_is_active
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
from freqtrade.strategy import merge_informative_pair
@@ -170,6 +171,60 @@ def auto_populate_any_indicators(
return df
def setup_freqai_spice_rack(config: dict, exchange: Optional[Exchange]) -> Dict[str, Any]:
import difflib
import json
from pathlib import Path
auto_config = config.get('freqai_config', 'lightgbm_config.json')
with open(Path(__file__).parent / Path('spice_rack') / auto_config) as json_file:
freqai_config = json.load(json_file)
config['freqai'] = freqai_config['freqai']
config['freqai']['identifier'] = config['freqai_identifier']
corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist']
timeframes = config['freqai']['feature_parameters']['include_timeframes']
new_corr_pairs = []
new_tfs = []
if not exchange:
logger.warning('No dataprovider available.')
config['freqai']['enabled'] = False
return config
# find the closest pairs to what the default config wants
for pair in corr_pairs:
closest_pair = difflib.get_close_matches(
pair,
exchange.markets
)
if not closest_pair:
logger.warning(f'Could not find {pair} in markets, removing from '
f'corr_pairlist.')
else:
closest_pair = closest_pair[0]
new_corr_pairs.append(closest_pair)
logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.')
# find the closest matching timeframes to what the default config wants
if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'):
logger.warning('Default spice rack is designed for lower base timeframes (e.g. > '
f'15m). But user passed {config["timeframe"]}.')
new_tfs.append(config['timeframe'])
list_tfs = [timeframe_to_seconds(tf) for tf
in exchange.timeframes]
for tf in timeframes:
tf_secs = timeframe_to_seconds(tf)
closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs))
closest_tf = exchange.timeframes[closest_index]
logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.')
new_tfs.append(closest_tf)
config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs})
config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs})
config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'})
return config
# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
# def download_all_data_for_training(dp: DataProvider, config: dict) -> None:
# """

4
freqtrade/optimize/backtesting.py
View File

@@ -89,6 +89,10 @@ class Backtesting:
self._exchange_name, self.config, load_leverage_tiers=True)
self.dataprovider = DataProvider(self.config, self.exchange)
if config.get('freqai_spice_rack', False):
from freqtrade.freqai.utils import setup_freqai_spice_rack
self.config = setup_freqai_spice_rack(self.config, self.exchange)
if self.config.get('strategy_list'):
if self.config.get('freqai', {}).get('enabled', False):
raise OperationalException(

60
freqtrade/strategy/interface.py
View File

@@ -5,7 +5,7 @@ This module defines the interface to apply for strategies
import logging
from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Dict, List, Optional, Tuple, Union
import arrow
from pandas import DataFrame
@@ -147,9 +147,9 @@ class IStrategy(ABC, HyperStrategyMixin):
def load_freqAI_model(self) -> None:
spice_rack = self.config.get('freqai_spice_rack', False)
if self.config.get('freqai', {}).get('enabled', False) or spice_rack:
spice_rack = self.config.get('freqai_spice_rack', False)
if spice_rack:
self.config = self.setup_freqai_spice_rack(self.config)
from freqtrade.freqai.utils import setup_freqai_spice_rack
self.config = setup_freqai_spice_rack(self.config, self.dp._exchange)
# Import here to avoid importing this if freqAI is disabled
from freqtrade.freqai.utils import download_all_data_for_training
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
@@ -190,60 +190,6 @@ class IStrategy(ABC, HyperStrategyMixin):
self.freqai = DummyClass() # type: ignore
def setup_freqai_spice_rack(self, config: dict) -> Dict[str, Any]:
import difflib
import json
from pathlib import Path
auto_config = config.get('freqai_config', 'lightgbm_config.json')
with open(Path('freqtrade') / 'freqai' / 'spice_rack'
/ auto_config) as json_file:
freqai_config = json.load(json_file)
config['freqai'] = freqai_config['freqai']
config['freqai']['identifier'] = config['freqai_identifier']
corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist']
timeframes = config['freqai']['feature_parameters']['include_timeframes']
new_corr_pairs = []
new_tfs = []
if not self.dp:
logger.warning('No dataprovider available.')
config['freqai']['enabled'] = False
return config
# find the closest pairs to what the default config wants
for pair in corr_pairs:
closest_pair = difflib.get_close_matches(
pair,
self.dp._exchange.markets # type: ignore
)
if not closest_pair:
logger.warning(f'Could not find {pair} in markets, removing from '
f'corr_pairlist.')
else:
closest_pair = closest_pair[0]
new_corr_pairs.append(closest_pair)
logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.')
# find the closest matching timeframes to what the default config wants
if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'):
logger.warning('Default spice rack is designed for lower base timeframes (e.g. > '
f'15m). But user passed {config["timeframe"]}.')
new_tfs.append(config['timeframe'])
list_tfs = [timeframe_to_seconds(tf) for tf
in self.dp._exchange.timeframes] # type: ignore
for tf in timeframes:
tf_secs = timeframe_to_seconds(tf)
closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs))
closest_tf = self.dp._exchange.timeframes[closest_index] # type: ignore
logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.')
new_tfs.append(closest_tf)
config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs})
config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs})
config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'})
return config
def ft_bot_start(self, **kwargs) -> None:
"""
Strategy init - runs after dataprovider has been added.