Merge branch 'develop' of https://github.com/theluxaz/freqtrade into main

# Conflicts:
#	freqtrade/freqtradebot.py
#	freqtrade/optimize/backtesting.py
This commit is contained in:
theluxaz
2021-10-13 02:01:26 +03:00
139 changed files with 3989 additions and 3843 deletions

View File

@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.configuration import TimeRange, validate_config_consistency
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
from freqtrade.data.btanalysis import trade_list_to_dataframe
@@ -61,8 +61,7 @@ class Backtesting:
self.config = config
self.results: Optional[Dict[str, Any]] = None
# Reset keys for backtesting
remove_credentials(self.config)
config['dry_run'] = True
self.strategylist: List[IStrategy] = []
self.all_results: Dict[str, Dict] = {}
@@ -86,7 +85,7 @@ class Backtesting:
"configuration or as cli argument `--timeframe 5m`")
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.init_backtest_detail()
self.pairlists = PairListManager(self.exchange, self.config)
if 'VolumePairList' in self.pairlists.name_list:
raise OperationalException("VolumePairList not allowed for backtesting.")
@@ -109,14 +108,6 @@ class Backtesting:
else:
self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
Trade.use_db = False
Trade.reset_trades()
PairLocks.timeframe = self.config['timeframe']
PairLocks.use_db = False
PairLocks.reset_locks()
self.wallets = Wallets(self.config, self.exchange, log=False)
self.timerange = TimeRange.parse_timerange(
None if self.config.get('timerange') is None else str(self.config.get('timerange')))
@@ -125,9 +116,7 @@ class Backtesting:
# Add maximum startup candle count to configuration for informative pairs support
self.config['startup_candle_count'] = self.required_startup
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
self.progress = BTProgress()
self.abort = False
self.init_backtest()
def __del__(self):
self.cleanup()
@@ -137,6 +126,28 @@ class Backtesting:
PairLocks.use_db = True
Trade.use_db = True
def init_backtest_detail(self):
# Load detail timeframe if specified
self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
if self.timeframe_detail:
self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
if self.timeframe_min <= self.timeframe_detail_min:
raise OperationalException(
"Detail timeframe must be smaller than strategy timeframe.")
else:
self.timeframe_detail_min = 0
self.detail_data: Dict[str, DataFrame] = {}
def init_backtest(self):
self.prepare_backtest(False)
self.wallets = Wallets(self.config, self.exchange, log=False)
self.progress = BTProgress()
self.abort = False
def _set_strategy(self, strategy: IStrategy):
"""
Load strategy into backtesting
@@ -144,7 +155,7 @@ class Backtesting:
self.strategy: IStrategy = strategy
strategy.dp = self.dataprovider
# Attach Wallets to Strategy baseclass
IStrategy.wallets = self.wallets
strategy.wallets = self.wallets
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
@@ -188,6 +199,23 @@ class Backtesting:
self.progress.set_new_value(1)
return data, self.timerange
def load_bt_data_detail(self) -> None:
"""
Loads backtest detail data (smaller timeframe) if necessary.
"""
if self.timeframe_detail:
self.detail_data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe_detail,
timerange=self.timerange,
startup_candles=0,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
else:
self.detail_data = {}
def prepare_backtest(self, enable_protections):
"""
Backtesting setup method - called once for every call to "backtest()".
@@ -199,7 +227,8 @@ class Backtesting:
Trade.reset_trades()
self.rejected_trades = 0
self.dataprovider.clear_cache()
self._load_protections(self.strategy)
if enable_protections:
self._load_protections(self.strategy)
def check_abort(self):
"""
@@ -320,10 +349,8 @@ class Backtesting:
else:
return sell_row[OPEN_IDX]
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
def _get_sell_trade_entry_for_candle(self, trade: LocalTrade,
sell_row: Tuple) -> Optional[LocalTrade]:
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore
sell_candle_time, sell_row[BUY_IDX],
@@ -353,6 +380,32 @@ class Backtesting:
return None
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
if self.timeframe_detail and trade.pair in self.detail_data:
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min)
detail_data = self.detail_data[trade.pair]
detail_data = detail_data.loc[
(detail_data['date'] >= sell_candle_time) &
(detail_data['date'] < sell_candle_end)
].copy()
if len(detail_data) == 0:
# Fall back to "regular" data if no detail data was found for this candle
return self._get_sell_trade_entry_for_candle(trade, sell_row)
detail_data.loc[:, 'buy'] = sell_row[BUY_IDX]
detail_data.loc[:, 'sell'] = sell_row[SELL_IDX]
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
for det_row in detail_data[headers].values.tolist():
res = self._get_sell_trade_entry_for_candle(trade, det_row)
if res:
return res
return None
else:
return self._get_sell_trade_entry_for_candle(trade, sell_row)
def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
try:
stake_amount = self.wallets.get_trade_stake_amount(pair, None)
@@ -601,6 +654,7 @@ class Backtesting:
data: Dict[str, Any] = {}
data, timerange = self.load_bt_data()
self.load_bt_data_detail()
logger.info("Dataload complete. Calculating indicators")
for strat in self.strategylist:

View File

@@ -7,7 +7,8 @@ import logging
from typing import Any, Dict
from freqtrade import constants
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.configuration import TimeRange, validate_config_consistency
from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.optimize.optimize_reports import generate_edge_table
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -28,11 +29,12 @@ class EdgeCli:
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
# Reset keys for edge
remove_credentials(self.config)
# Ensure using dry-run
self.config['dry_run'] = True
self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
self.strategy = StrategyResolver.load_strategy(self.config)
self.strategy.dp = DataProvider(config, None)
validate_config_consistency(self.config)

View File

@@ -22,6 +22,7 @@ from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.exceptions import OperationalException
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
@@ -30,7 +31,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
from freqtrade.optimize.optimize_reports import generate_strategy_stats
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
# Suppress scikit-learn FutureWarnings from skopt
@@ -44,7 +45,7 @@ progressbar.streams.wrap_stdout()
logger = logging.getLogger(__name__)
INITIAL_POINTS = 30
INITIAL_POINTS = 5
# Keep no more than SKOPT_MODEL_QUEUE_SIZE models
# in the skopt model queue, to optimize memory consumption
@@ -78,10 +79,10 @@ class Hyperopt:
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
self.auto_hyperopt = True
else:
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
self.auto_hyperopt = False
raise OperationalException(
"Using separate Hyperopt files has been removed in 2021.9. Please convert "
"your existing Hyperopt file to the new Hyperoptable strategy interface")
self.backtesting._set_strategy(self.backtesting.strategylist[0])
self.custom_hyperopt.strategy = self.backtesting.strategy
@@ -103,31 +104,6 @@ class Hyperopt:
self.num_epochs_saved = 0
self.current_best_epoch: Optional[Dict[str, Any]] = None
if not self.auto_hyperopt:
# Populate "fallback" functions here
# (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
logger.warning(
"DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. "
"Please move these methods to your strategy."
)
self.backtesting.strategy.populate_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
logger.warning(
"DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. "
"Please move these methods to your strategy."
)
self.backtesting.strategy.populate_buy_trend = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
logger.warning(
"DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. "
"Please move these methods to your strategy."
)
self.backtesting.strategy.populate_sell_trend = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
self.max_open_trades = self.config['max_open_trades']
@@ -256,7 +232,7 @@ class Hyperopt:
"""
Assign the dimensions in the hyperoptimization space.
"""
if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'):
if HyperoptTools.has_space(self.config, 'protection'):
# Protections can only be optimized when using the Parameter interface
logger.debug("Hyperopt has 'protection' space")
# Enable Protections if protection space is selected.
@@ -265,7 +241,7 @@ class Hyperopt:
if HyperoptTools.has_space(self.config, 'buy'):
logger.debug("Hyperopt has 'buy' space")
self.buy_space = self.custom_hyperopt.indicator_space()
self.buy_space = self.custom_hyperopt.buy_indicator_space()
if HyperoptTools.has_space(self.config, 'sell'):
logger.debug("Hyperopt has 'sell' space")
@@ -285,6 +261,15 @@ class Hyperopt:
self.dimensions = (self.buy_space + self.sell_space + self.protection_space
+ self.roi_space + self.stoploss_space + self.trailing_space)
def assign_params(self, params_dict: Dict, category: str) -> None:
"""
Assign hyperoptable parameters
"""
for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params_dict[attr_name]
def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
"""
Used Optimize function.
@@ -296,18 +281,13 @@ class Hyperopt:
# Apply parameters
if HyperoptTools.has_space(self.config, 'buy'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.buy_strategy_generator(params_dict))
self.assign_params(params_dict, 'buy')
if HyperoptTools.has_space(self.config, 'sell'):
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.sell_strategy_generator(params_dict))
self.assign_params(params_dict, 'sell')
if HyperoptTools.has_space(self.config, 'protection'):
for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params_dict[attr_name]
self.assign_params(params_dict, 'protection')
if HyperoptTools.has_space(self.config, 'roi'):
self.backtesting.strategy.minimal_roi = ( # type: ignore
@@ -385,10 +365,20 @@ class Hyperopt:
}
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
estimator = self.custom_hyperopt.generate_estimator()
acq_optimizer = "sampling"
if isinstance(estimator, str):
if estimator not in ("GP", "RF", "ET", "GBRT"):
raise OperationalException(f"Estimator {estimator} not supported.")
else:
acq_optimizer = "auto"
logger.info(f"Using estimator {estimator}.")
return Optimizer(
dimensions,
base_estimator="ET",
acq_optimizer="auto",
base_estimator=estimator,
acq_optimizer=acq_optimizer,
n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.random_state,
@@ -517,11 +507,10 @@ class Hyperopt:
f"saved to '{self.results_file}'.")
if self.current_best_epoch:
if self.auto_hyperopt:
HyperoptTools.try_export_params(
self.config,
self.backtesting.strategy.get_strategy_name(),
self.current_best_epoch)
HyperoptTools.try_export_params(
self.config,
self.backtesting.strategy.get_strategy_name(),
self.current_best_epoch)
HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
self.print_json)

View File

@@ -4,15 +4,23 @@ This module implements a convenience auto-hyperopt class, which can be used toge
that implement IHyperStrategy interface.
"""
from contextlib import suppress
from typing import Any, Callable, Dict, List
from typing import Callable, Dict, List
from pandas import DataFrame
from freqtrade.exceptions import OperationalException
with suppress(ImportError):
from skopt.space import Dimension
from freqtrade.optimize.hyperopt_interface import IHyperOpt
from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
def _format_exception_message(space: str) -> str:
raise OperationalException(
f"The '{space}' space is included into the hyperoptimization "
f"but no parameter for this space was not found in your Strategy. "
f"Please make sure to have parameters for this space enabled for optimization "
f"or remove the '{space}' space from hyperoptimization.")
class HyperOptAuto(IHyperOpt):
@@ -22,26 +30,6 @@ class HyperOptAuto(IHyperOpt):
sell_indicator_space methods, but other hyperopt methods can be overridden as well.
"""
def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def populate_buy_trend(dataframe: DataFrame, metadata: dict):
for attr_name, attr in self.strategy.enumerate_parameters('buy'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params[attr_name]
return self.strategy.populate_buy_trend(dataframe, metadata)
return populate_buy_trend
def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def populate_sell_trend(dataframe: DataFrame, metadata: dict):
for attr_name, attr in self.strategy.enumerate_parameters('sell'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params[attr_name]
return self.strategy.populate_sell_trend(dataframe, metadata)
return populate_sell_trend
def _get_func(self, name) -> Callable:
"""
Return a function defined in Strategy.HyperOpt class, or one defined in super() class.
@@ -60,21 +48,22 @@ class HyperOptAuto(IHyperOpt):
if attr.optimize:
yield attr.get_space(attr_name)
def _get_indicator_space(self, category, fallback_method_name):
def _get_indicator_space(self, category):
# TODO: is this necessary, or can we call "generate_space" directly?
indicator_space = list(self._generate_indicator_space(category))
if len(indicator_space) > 0:
return indicator_space
else:
return self._get_func(fallback_method_name)()
_format_exception_message(category)
def indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy', 'indicator_space')
def buy_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy')
def sell_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('sell', 'sell_indicator_space')
return self._get_indicator_space('sell')
def protection_space(self) -> List['Dimension']:
return self._get_indicator_space('protection', 'protection_space')
return self._get_indicator_space('protection')
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
return self._get_func('generate_roi_table')(params)
@@ -90,3 +79,6 @@ class HyperOptAuto(IHyperOpt):
def trailing_space(self) -> List['Dimension']:
return self._get_func('trailing_space')()
def generate_estimator(self) -> EstimatorType:
return self._get_func('generate_estimator')()

View File

@@ -5,11 +5,11 @@ This module defines the interface to apply for hyperopt
import logging
import math
from abc import ABC
from typing import Any, Callable, Dict, List
from typing import Dict, List, Union
from sklearn.base import RegressorMixin
from skopt.space import Categorical, Dimension, Integer
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import round_dict
from freqtrade.optimize.space import SKDecimal
@@ -18,12 +18,7 @@ from freqtrade.strategy import IStrategy
logger = logging.getLogger(__name__)
def _format_exception_message(method: str, space: str) -> str:
return (f"The '{space}' space is included into the hyperoptimization "
f"but {method}() method is not found in your "
f"custom Hyperopt class. You should either implement this "
f"method or remove the '{space}' space from hyperoptimization.")
EstimatorType = Union[RegressorMixin, str]
class IHyperOpt(ABC):
@@ -45,36 +40,13 @@ class IHyperOpt(ABC):
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
IHyperOpt.timeframe = str(config['timeframe'])
def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def generate_estimator(self) -> EstimatorType:
"""
Create a buy strategy generator.
Return base_estimator.
Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
inheriting from RegressorMixin (from sklearn).
"""
raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy'))
def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
"""
Create a sell strategy generator.
"""
raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
def protection_space(self) -> List[Dimension]:
"""
Create a protection space.
Only supported by the Parameter interface.
"""
raise OperationalException(_format_exception_message('indicator_space', 'protection'))
def indicator_space(self) -> List[Dimension]:
"""
Create an indicator space.
"""
raise OperationalException(_format_exception_message('indicator_space', 'buy'))
def sell_indicator_space(self) -> List[Dimension]:
"""
Create a sell indicator space.
"""
raise OperationalException(_format_exception_message('sell_indicator_space', 'sell'))
return 'ET'
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
"""

View File

@@ -0,0 +1,41 @@
"""
MaxDrawDownHyperOptLoss
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
from datetime import datetime
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss
class MaxDrawDownHyperOptLoss(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation optimizes for max draw down and profit
Less max drawdown more profit -> Lower return value
"""
@staticmethod
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
*args, **kwargs) -> float:
"""
Objective function.
Uses profit ratio weighted max_drawdown when drawdown is available.
Otherwise directly optimizes profit ratio.
"""
total_profit = results['profit_abs'].sum()
try:
max_drawdown = calculate_max_drawdown(results, value_col='profit_abs')
except ValueError:
# No losing trade, therefore no drawdown.
return -total_profit
return -total_profit / max_drawdown[0]

View File

@@ -7,6 +7,7 @@ from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Tuple
import numpy as np
import pandas as pd
import rapidjson
import tabulate
from colorama import Fore, Style
@@ -298,8 +299,8 @@ class HyperoptTools():
f"Objective: {results['loss']:.5f}")
@staticmethod
def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str:
def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool,
has_drawdown: bool) -> pd.DataFrame:
trials['Best'] = ''
if 'results_metrics.winsdrawslosses' not in trials.columns:
@@ -435,8 +436,7 @@ class HyperoptTools():
return table
@staticmethod
def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
csv_file: str) -> None:
def export_csv_file(config: dict, results: list, csv_file: str) -> None:
"""
Log result to csv-file
"""

View File

@@ -464,6 +464,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'max_open_trades_setting': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
'timeframe_detail': config.get('timeframe_detail', ''),
'timerange': config.get('timerange', ''),
'enable_protections': config.get('enable_protections', False),
'strategy_name': strategy,