diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 360387aa6..dc7d39bf6 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -12,6 +12,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, timeframe_to_minutes +from freqtrade.freqai.utils import setup_freqai_spice_rack from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist from freqtrade.resolvers import ExchangeResolver @@ -48,6 +49,10 @@ def start_download_data(args: Dict[str, Any]) -> None: # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + if config.get('freqai_spice_rack', False): + config = setup_freqai_spice_rack(config, exchange) + markets = [p for p, m in exchange.markets.items() if market_is_active(m) or config.get('include_inactive')] diff --git a/freqtrade/freqai/utils.py b/freqtrade/freqai/utils.py index e21321f35..60e0d0ffe 100644 --- a/freqtrade/freqai/utils.py +++ b/freqtrade/freqai/utils.py @@ -1,5 +1,6 @@ import logging from datetime import datetime, timezone +from typing import Any, Dict, Optional import numpy as np # for spice rack @@ -12,7 +13,7 @@ from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data from freqtrade.exceptions import OperationalException -from freqtrade.exchange import timeframe_to_seconds +from freqtrade.exchange import Exchange, timeframe_to_seconds from freqtrade.exchange.exchange import market_is_active from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.strategy import merge_informative_pair @@ -170,6 +171,60 @@ def auto_populate_any_indicators( return df + +def setup_freqai_spice_rack(config: dict, exchange: Optional[Exchange]) -> Dict[str, Any]: + import difflib + import json + from pathlib import Path + auto_config = config.get('freqai_config', 'lightgbm_config.json') + with open(Path(__file__).parent / Path('spice_rack') / auto_config) as json_file: + freqai_config = json.load(json_file) + config['freqai'] = freqai_config['freqai'] + config['freqai']['identifier'] = config['freqai_identifier'] + corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist'] + timeframes = config['freqai']['feature_parameters']['include_timeframes'] + new_corr_pairs = [] + new_tfs = [] + + if not exchange: + logger.warning('No dataprovider available.') + config['freqai']['enabled'] = False + return config + # find the closest pairs to what the default config wants + for pair in corr_pairs: + closest_pair = difflib.get_close_matches( + pair, + exchange.markets + ) + if not closest_pair: + logger.warning(f'Could not find {pair} in markets, removing from ' + f'corr_pairlist.') + else: + closest_pair = closest_pair[0] + + new_corr_pairs.append(closest_pair) + logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.') + + # find the closest matching timeframes to what the default config wants + if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'): + logger.warning('Default spice rack is designed for lower base timeframes (e.g. > ' + f'15m). But user passed {config["timeframe"]}.') + new_tfs.append(config['timeframe']) + + list_tfs = [timeframe_to_seconds(tf) for tf + in exchange.timeframes] + for tf in timeframes: + tf_secs = timeframe_to_seconds(tf) + closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs)) + closest_tf = exchange.timeframes[closest_index] + logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.') + new_tfs.append(closest_tf) + + config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs}) + config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs}) + config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'}) + return config + # Keep below for when we wish to download heterogeneously lengthed data for FreqAI. # def download_all_data_for_training(dp: DataProvider, config: dict) -> None: # """ diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 105851e60..f54d59b5a 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -89,6 +89,10 @@ class Backtesting: self._exchange_name, self.config, load_leverage_tiers=True) self.dataprovider = DataProvider(self.config, self.exchange) + if config.get('freqai_spice_rack', False): + from freqtrade.freqai.utils import setup_freqai_spice_rack + self.config = setup_freqai_spice_rack(self.config, self.exchange) + if self.config.get('strategy_list'): if self.config.get('freqai', {}).get('enabled', False): raise OperationalException( diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 2272e42a0..a29d6228a 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -5,7 +5,7 @@ This module defines the interface to apply for strategies import logging from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame @@ -147,9 +147,9 @@ class IStrategy(ABC, HyperStrategyMixin): def load_freqAI_model(self) -> None: spice_rack = self.config.get('freqai_spice_rack', False) if self.config.get('freqai', {}).get('enabled', False) or spice_rack: - spice_rack = self.config.get('freqai_spice_rack', False) if spice_rack: - self.config = self.setup_freqai_spice_rack(self.config) + from freqtrade.freqai.utils import setup_freqai_spice_rack + self.config = setup_freqai_spice_rack(self.config, self.dp._exchange) # Import here to avoid importing this if freqAI is disabled from freqtrade.freqai.utils import download_all_data_for_training from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver @@ -190,60 +190,6 @@ class IStrategy(ABC, HyperStrategyMixin): self.freqai = DummyClass() # type: ignore - def setup_freqai_spice_rack(self, config: dict) -> Dict[str, Any]: - import difflib - import json - from pathlib import Path - auto_config = config.get('freqai_config', 'lightgbm_config.json') - with open(Path('freqtrade') / 'freqai' / 'spice_rack' - / auto_config) as json_file: - freqai_config = json.load(json_file) - config['freqai'] = freqai_config['freqai'] - config['freqai']['identifier'] = config['freqai_identifier'] - corr_pairs = config['freqai']['feature_parameters']['include_corr_pairlist'] - timeframes = config['freqai']['feature_parameters']['include_timeframes'] - new_corr_pairs = [] - new_tfs = [] - - if not self.dp: - logger.warning('No dataprovider available.') - config['freqai']['enabled'] = False - return config - # find the closest pairs to what the default config wants - for pair in corr_pairs: - closest_pair = difflib.get_close_matches( - pair, - self.dp._exchange.markets # type: ignore - ) - if not closest_pair: - logger.warning(f'Could not find {pair} in markets, removing from ' - f'corr_pairlist.') - else: - closest_pair = closest_pair[0] - - new_corr_pairs.append(closest_pair) - logger.info(f'Spice rack will use {closest_pair} as informative in FreqAI model.') - - # find the closest matching timeframes to what the default config wants - if timeframe_to_seconds(config['timeframe']) > timeframe_to_seconds('15m'): - logger.warning('Default spice rack is designed for lower base timeframes (e.g. > ' - f'15m). But user passed {config["timeframe"]}.') - new_tfs.append(config['timeframe']) - - list_tfs = [timeframe_to_seconds(tf) for tf - in self.dp._exchange.timeframes] # type: ignore - for tf in timeframes: - tf_secs = timeframe_to_seconds(tf) - closest_index = min(range(len(list_tfs)), key=lambda i: abs(list_tfs[i] - tf_secs)) - closest_tf = self.dp._exchange.timeframes[closest_index] # type: ignore - logger.info(f'Spice rack will use {closest_tf} as informative tf in FreqAI model.') - new_tfs.append(closest_tf) - - config['freqai']['feature_parameters'].update({'include_timeframes': new_tfs}) - config['freqai']['feature_parameters'].update({'include_corr_pairlist': new_corr_pairs}) - config.update({"freqaimodel": 'LightGBMRegressorMultiTarget'}) - return config - def ft_bot_start(self, **kwargs) -> None: """ Strategy init - runs after dataprovider has been added.