Merge branch 'develop' into feature_keyval_storage

This commit is contained in:
eSeR1805
2022-06-19 13:32:25 +03:00
35 changed files with 1126 additions and 172 deletions

View File

@@ -6,6 +6,7 @@ Contains all start-commands, subcommands and CLI Interface creation.
Note: Be careful with file-scoped imports in these subfiles.
as they are parsed on startup, nothing containing optional modules should be loaded.
"""
from freqtrade.commands.analyze_commands import start_analysis_entries_exits
from freqtrade.commands.arguments import Arguments
from freqtrade.commands.build_config_commands import start_new_config
from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,

View File

@@ -0,0 +1,69 @@
import logging
from pathlib import Path
from typing import Any, Dict
from freqtrade.configuration import setup_utils_configuration
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
def setup_analyze_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]:
"""
Prepare the configuration for the entry/exit reason analysis module
:param args: Cli args from Arguments()
:param method: Bot running mode
:return: Configuration
"""
config = setup_utils_configuration(args, method)
no_unlimited_runmodes = {
RunMode.BACKTEST: 'backtesting',
}
if method in no_unlimited_runmodes.keys():
from freqtrade.data.btanalysis import get_latest_backtest_filename
if 'exportfilename' in config:
if config['exportfilename'].is_dir():
btfile = Path(get_latest_backtest_filename(config['exportfilename']))
signals_file = f"{config['exportfilename']}/{btfile.stem}_signals.pkl"
else:
if config['exportfilename'].exists():
btfile = Path(config['exportfilename'])
signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl"
else:
raise OperationalException(f"{config['exportfilename']} does not exist.")
else:
raise OperationalException('exportfilename not in config.')
if (not Path(signals_file).exists()):
raise OperationalException(
(f"Cannot find latest backtest signals file: {signals_file}."
"Run backtesting with `--export signals`.")
)
return config
def start_analysis_entries_exits(args: Dict[str, Any]) -> None:
"""
Start analysis script
:param args: Cli args from Arguments()
:return: None
"""
from freqtrade.data.entryexitanalysis import process_entry_exit_reasons
# Initialize configuration
config = setup_analyze_configuration(args, RunMode.BACKTEST)
logger.info('Starting freqtrade in analysis mode')
process_entry_exit_reasons(config['exportfilename'],
config['exchange']['pair_whitelist'],
config['analysis_groups'],
config['enter_reason_list'],
config['exit_reason_list'],
config['indicator_list']
)

View File

@@ -101,6 +101,9 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
"disableparamexport", "backtest_breakdown"]
ARGS_ANALYZE_ENTRIES_EXITS = ["exportfilename", "analysis_groups", "enter_reason_list",
"exit_reason_list", "indicator_list"]
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies", "list-data",
"hyperopt-list", "hyperopt-show", "backtest-filter",
@@ -182,8 +185,9 @@ class Arguments:
self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
self._build_args(optionlist=['version'], parser=self.parser)
from freqtrade.commands import (start_backtesting, start_backtesting_show,
start_convert_data, start_convert_db, start_convert_trades,
from freqtrade.commands import (start_analysis_entries_exits, start_backtesting,
start_backtesting_show, start_convert_data,
start_convert_db, start_convert_trades,
start_create_userdir, start_download_data, start_edge,
start_hyperopt, start_hyperopt_list, start_hyperopt_show,
start_install_ui, start_list_data, start_list_exchanges,
@@ -283,6 +287,13 @@ class Arguments:
backtesting_show_cmd.set_defaults(func=start_backtesting_show)
self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd)
# Add backtesting analysis subcommand
analysis_cmd = subparsers.add_parser('backtesting-analysis',
help='Backtest Analysis module.',
parents=[_common_parser])
analysis_cmd.set_defaults(func=start_analysis_entries_exits)
self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd)
# Add edge subcommand
edge_cmd = subparsers.add_parser('edge', help='Edge module.',
parents=[_common_parser, _strategy_parser])

View File

@@ -614,4 +614,37 @@ AVAILABLE_CLI_OPTIONS = {
"that do not contain any parameters."),
action="store_true",
),
"analysis_groups": Arg(
"--analysis-groups",
help=("grouping output - "
"0: simple wins/losses by enter tag, "
"1: by enter_tag, "
"2: by enter_tag and exit_tag, "
"3: by pair and enter_tag, "
"4: by pair, enter_ and exit_tag (this can get quite large)"),
nargs='+',
default=['0', '1', '2'],
choices=['0', '1', '2', '3', '4'],
),
"enter_reason_list": Arg(
"--enter-reason-list",
help=("Comma separated list of entry signals to analyse. Default: all. "
"e.g. 'entry_tag_a,entry_tag_b'"),
nargs='+',
default=['all'],
),
"exit_reason_list": Arg(
"--exit-reason-list",
help=("Comma separated list of exit signals to analyse. Default: all. "
"e.g. 'exit_tag_a,roi,stop_loss,trailing_stop_loss'"),
nargs='+',
default=['all'],
),
"indicator_list": Arg(
"--indicator-list",
help=("Comma separated list of indicators to analyse. "
"e.g. 'close,rsi,bb_lowerband,profit_abs'"),
nargs='+',
default=[],
),
}

View File

@@ -95,6 +95,8 @@ class Configuration:
self._process_data_options(config)
self._process_analyze_options(config)
# Check if the exchange set by the user is supported
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
@@ -433,6 +435,19 @@ class Configuration:
self._args_to_config(config, argname='candle_types',
logstring='Detected --candle-types: {}')
def _process_analyze_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='analysis_groups',
logstring='Analysis reason groups: {}')
self._args_to_config(config, argname='enter_reason_list',
logstring='Analysis enter tag list: {}')
self._args_to_config(config, argname='exit_reason_list',
logstring='Analysis exit tag list: {}')
self._args_to_config(config, argname='indicator_list',
logstring='Analysis indicator list: {}')
def _process_runmode(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='dry_run',

View File

@@ -0,0 +1,227 @@
import logging
from pathlib import Path
from typing import List, Optional
import joblib
import pandas as pd
from tabulate import tabulate
from freqtrade.data.btanalysis import (get_latest_backtest_filename, load_backtest_data,
load_backtest_stats)
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
def _load_signal_candles(backtest_dir: Path):
if backtest_dir.is_dir():
scpf = Path(backtest_dir,
Path(get_latest_backtest_filename(backtest_dir)).stem + "_signals.pkl"
)
else:
scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_signals.pkl")
try:
scp = open(scpf, "rb")
signal_candles = joblib.load(scp)
logger.info(f"Loaded signal candles: {str(scpf)}")
except Exception as e:
logger.error("Cannot load signal candles from pickled results: ", e)
return signal_candles
def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_candles):
analysed_trades_dict = {}
analysed_trades_dict[strategy_name] = {}
try:
logger.info(f"Processing {strategy_name} : {len(pairlist)} pairs")
for pair in pairlist:
if pair in signal_candles[strategy_name]:
analysed_trades_dict[strategy_name][pair] = _analyze_candles_and_indicators(
pair,
trades,
signal_candles[strategy_name][pair])
except Exception as e:
print(f"Cannot process entry/exit reasons for {strategy_name}: ", e)
return analysed_trades_dict
def _analyze_candles_and_indicators(pair, trades, signal_candles):
buyf = signal_candles
if len(buyf) > 0:
buyf = buyf.set_index('date', drop=False)
trades_red = trades.loc[trades['pair'] == pair].copy()
trades_inds = pd.DataFrame()
if trades_red.shape[0] > 0 and buyf.shape[0] > 0:
for t, v in trades_red.open_date.items():
allinds = buyf.loc[(buyf['date'] < v)]
if allinds.shape[0] > 0:
tmp_inds = allinds.iloc[[-1]]
trades_red.loc[t, 'signal_date'] = tmp_inds['date'].values[0]
trades_red.loc[t, 'enter_reason'] = trades_red.loc[t, 'enter_tag']
tmp_inds.index.rename('signal_date', inplace=True)
trades_inds = pd.concat([trades_inds, tmp_inds])
if 'signal_date' in trades_red:
trades_red['signal_date'] = pd.to_datetime(trades_red['signal_date'], utc=True)
trades_red.set_index('signal_date', inplace=True)
try:
trades_red = pd.merge(trades_red, trades_inds, on='signal_date', how='outer')
except Exception as e:
raise e
return trades_red
else:
return pd.DataFrame()
def _do_group_table_output(bigdf, glist):
for g in glist:
# 0: summary wins/losses grouped by enter tag
if g == "0":
group_mask = ['enter_reason']
wins = bigdf.loc[bigdf['profit_abs'] >= 0] \
.groupby(group_mask) \
.agg({'profit_abs': ['sum']})
wins.columns = ['profit_abs_wins']
loss = bigdf.loc[bigdf['profit_abs'] < 0] \
.groupby(group_mask) \
.agg({'profit_abs': ['sum']})
loss.columns = ['profit_abs_loss']
new = bigdf.groupby(group_mask).agg({'profit_abs': [
'count',
lambda x: sum(x > 0),
lambda x: sum(x <= 0)]})
new = pd.concat([new, wins, loss], axis=1).fillna(0)
new['profit_tot'] = new['profit_abs_wins'] - abs(new['profit_abs_loss'])
new['wl_ratio_pct'] = (new.iloc[:, 1] / new.iloc[:, 0] * 100).fillna(0)
new['avg_win'] = (new['profit_abs_wins'] / new.iloc[:, 1]).fillna(0)
new['avg_loss'] = (new['profit_abs_loss'] / new.iloc[:, 2]).fillna(0)
new.columns = ['total_num_buys', 'wins', 'losses', 'profit_abs_wins', 'profit_abs_loss',
'profit_tot', 'wl_ratio_pct', 'avg_win', 'avg_loss']
sortcols = ['total_num_buys']
_print_table(new, sortcols, show_index=True)
else:
agg_mask = {'profit_abs': ['count', 'sum', 'median', 'mean'],
'profit_ratio': ['sum', 'median', 'mean']}
agg_cols = ['num_buys', 'profit_abs_sum', 'profit_abs_median',
'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct',
'total_profit_pct']
sortcols = ['profit_abs_sum', 'enter_reason']
# 1: profit summaries grouped by enter_tag
if g == "1":
group_mask = ['enter_reason']
# 2: profit summaries grouped by enter_tag and exit_tag
if g == "2":
group_mask = ['enter_reason', 'exit_reason']
# 3: profit summaries grouped by pair and enter_tag
if g == "3":
group_mask = ['pair', 'enter_reason']
# 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large)
if g == "4":
group_mask = ['pair', 'enter_reason', 'exit_reason']
if group_mask:
new = bigdf.groupby(group_mask).agg(agg_mask).reset_index()
new.columns = group_mask + agg_cols
new['median_profit_pct'] = new['median_profit_pct'] * 100
new['mean_profit_pct'] = new['mean_profit_pct'] * 100
new['total_profit_pct'] = new['total_profit_pct'] * 100
_print_table(new, sortcols)
else:
logger.warning("Invalid group mask specified.")
def _print_results(analysed_trades, stratname, analysis_groups,
enter_reason_list, exit_reason_list,
indicator_list, columns=None):
if columns is None:
columns = ['pair', 'open_date', 'close_date', 'profit_abs', 'enter_reason', 'exit_reason']
bigdf = pd.DataFrame()
for pair, trades in analysed_trades[stratname].items():
bigdf = pd.concat([bigdf, trades], ignore_index=True)
if bigdf.shape[0] > 0 and ('enter_reason' in bigdf.columns):
if analysis_groups:
_do_group_table_output(bigdf, analysis_groups)
if enter_reason_list and "all" not in enter_reason_list:
bigdf = bigdf.loc[(bigdf['enter_reason'].isin(enter_reason_list))]
if exit_reason_list and "all" not in exit_reason_list:
bigdf = bigdf.loc[(bigdf['exit_reason'].isin(exit_reason_list))]
if "all" in indicator_list:
print(bigdf)
elif indicator_list is not None:
available_inds = []
for ind in indicator_list:
if ind in bigdf:
available_inds.append(ind)
ilist = ["pair", "enter_reason", "exit_reason"] + available_inds
_print_table(bigdf[ilist], sortcols=['exit_reason'], show_index=False)
else:
print("\\_ No trades to show")
def _print_table(df, sortcols=None, show_index=False):
if (sortcols is not None):
data = df.sort_values(sortcols)
else:
data = df
print(
tabulate(
data,
headers='keys',
tablefmt='psql',
showindex=show_index
)
)
def process_entry_exit_reasons(backtest_dir: Path,
pairlist: List[str],
analysis_groups: Optional[List[str]] = ["0", "1", "2"],
enter_reason_list: Optional[List[str]] = ["all"],
exit_reason_list: Optional[List[str]] = ["all"],
indicator_list: Optional[List[str]] = []):
try:
backtest_stats = load_backtest_stats(backtest_dir)
for strategy_name, results in backtest_stats['strategy'].items():
trades = load_backtest_data(backtest_dir, strategy_name)
if not trades.empty:
signal_candles = _load_signal_candles(backtest_dir)
analysed_trades_dict = _process_candles_and_indicators(pairlist, strategy_name,
trades, signal_candles)
_print_results(analysed_trades_dict,
strategy_name,
analysis_groups,
enter_reason_list,
exit_reason_list,
indicator_list)
except ValueError as e:
raise OperationalException(e) from e

View File

@@ -221,7 +221,7 @@ def _download_pair_history(pair: str, *,
prepend=prepend)
logger.info(f'({process}) - Download history data for "{pair}", {timeframe}, '
f'{candle_type} and store in {datadir}.'
f'{candle_type} and store in {datadir}. '
f'From {format_ms_time(since_ms) if since_ms else "start"} to '
f'{format_ms_time(until_ms) if until_ms else "now"}'
)

View File

@@ -93,7 +93,7 @@ class Exchange:
:return: None
"""
self._api: ccxt.Exchange
self._api_async: ccxt_async.Exchange
self._api_async: ccxt_async.Exchange = None
self._markets: Dict = {}
self._trading_fees: Dict[str, Any] = {}
self._leverage_tiers: Dict[str, List[Dict]] = {}

View File

@@ -3,6 +3,7 @@ import logging
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from freqtrade.constants import BuySell
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
@@ -24,6 +25,8 @@ class Gateio(Exchange):
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
"ohlcv_volume_currency": "quote",
"time_in_force_parameter": "timeInForce",
"order_time_in_force": ['gtc', 'ioc'],
"stoploss_order_types": {"limit": "limit"},
"stoploss_on_exchange": True,
}
@@ -40,13 +43,33 @@ class Gateio(Exchange):
]
def validate_ordertypes(self, order_types: Dict) -> None:
super().validate_ordertypes(order_types)
if self.trading_mode != TradingMode.FUTURES:
if any(v == 'market' for k, v in order_types.items()):
raise OperationalException(
f'Exchange {self.name} does not support market orders.')
def _get_params(
self,
side: BuySell,
ordertype: str,
leverage: float,
reduceOnly: bool,
time_in_force: str = 'gtc',
) -> Dict:
params = super()._get_params(
side=side,
ordertype=ordertype,
leverage=leverage,
reduceOnly=reduceOnly,
time_in_force=time_in_force,
)
if ordertype == 'market' and self.trading_mode == TradingMode.FUTURES:
params['type'] = 'market'
param = self._ft_has.get('time_in_force_parameter', '')
params.update({param: 'ioc'})
return params
def get_trades_for_order(self, order_id: str, pair: str, since: datetime,
params: Optional[Dict] = None) -> List:
trades = super().get_trades_for_order(order_id, pair, since, params)
@@ -61,7 +84,8 @@ class Gateio(Exchange):
pair_fees = self._trading_fees.get(pair, {})
if pair_fees:
for idx, trade in enumerate(trades):
if trade.get('fee', {}).get('cost') is None:
fee = trade.get('fee', {})
if fee and fee.get('cost') is None:
takerOrMaker = trade.get('takerOrMaker', 'taker')
if pair_fees.get(takerOrMaker) is not None:
trades[idx]['fee'] = {

View File

@@ -73,8 +73,6 @@ class FreqtradeBot(LoggingMixin):
PairLocks.timeframe = self.config['timeframe']
self.protections = ProtectionManager(self.config, self.strategy.protections)
# RPC runs in separate threads, can start handling external commands just after
# initialization, even before Freqtradebot has a chance to start its throttling,
# so anything in the Freqtradebot instance should be ready (initialized), including
@@ -124,6 +122,8 @@ class FreqtradeBot(LoggingMixin):
self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc)
self.strategy.ft_bot_start()
# Initialize protections AFTER bot start - otherwise parameters are not loaded.
self.protections = ProtectionManager(self.config, self.strategy.protections)
def notify_status(self, msg: str) -> None:
"""

View File

@@ -1058,6 +1058,7 @@ class Backtesting:
# Close trade
open_trade_count -= 1
open_trades[pair].remove(t)
LocalTrade.trades_open.remove(t)
self.wallets.update()
# 2. Process entries.
@@ -1081,6 +1082,8 @@ class Backtesting:
open_trade_count += 1
# logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
open_trades[pair].append(trade)
LocalTrade.add_bt_trade(trade)
self.wallets.update()
for trade in list(open_trades[pair]):
# 3. Process entry orders.
@@ -1088,7 +1091,6 @@ class Backtesting:
if order and self._get_order_filled(order.price, row):
order.close_bt_order(current_time, trade)
trade.open_order_id = None
LocalTrade.add_bt_trade(trade)
self.wallets.update()
# 4. Create exit orders (if any)
@@ -1267,13 +1269,14 @@ class Backtesting:
self.results['strategy_comparison'].extend(results['strategy_comparison'])
else:
self.results = results
dt_appendix = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
if self.config.get('export', 'none') in ('trades', 'signals'):
store_backtest_stats(self.config['exportfilename'], self.results)
store_backtest_stats(self.config['exportfilename'], self.results, dt_appendix)
if (self.config.get('export', 'none') == 'signals' and
self.dataprovider.runmode == RunMode.BACKTEST):
store_backtest_signal_candles(self.config['exportfilename'], self.processed_dfs)
store_backtest_signal_candles(
self.config['exportfilename'], self.processed_dfs, dt_appendix)
# Results may be mixed up now. Sort them so they follow --strategy-list order.
if 'strategy_list' in self.config and len(self.results) > 0:

View File

@@ -17,21 +17,21 @@ from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
logger = logging.getLogger(__name__)
def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> None:
def store_backtest_stats(
recordfilename: Path, stats: Dict[str, DataFrame], dtappendix: str) -> None:
"""
Stores backtest results
:param recordfilename: Path object, which can either be a filename or a directory.
Filenames will be appended with a timestamp right before the suffix
while for directories, <directory>/backtest-result-<datetime>.json will be used as filename
:param stats: Dataframe containing the backtesting statistics
:param dtappendix: Datetime to use for the filename
"""
if recordfilename.is_dir():
filename = (recordfilename /
f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json')
filename = (recordfilename / f'backtest-result-{dtappendix}.json')
else:
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
recordfilename.parent, f'{recordfilename.stem}-{dtappendix}'
).with_suffix(recordfilename.suffix)
# Store metadata separately.
@@ -44,7 +44,8 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
file_dump_json(latest_filename, {'latest_backtest': str(filename.name)})
def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]) -> Path:
def store_backtest_signal_candles(
recordfilename: Path, candles: Dict[str, Dict], dtappendix: str) -> Path:
"""
Stores backtest trade signal candles
:param recordfilename: Path object, which can either be a filename or a directory.
@@ -52,14 +53,13 @@ def store_backtest_signal_candles(recordfilename: Path, candles: Dict[str, Dict]
while for directories, <directory>/backtest-result-<datetime>_signals.pkl will be used
as filename
:param stats: Dict containing the backtesting signal candles
:param dtappendix: Datetime to use for the filename
"""
if recordfilename.is_dir():
filename = (recordfilename /
f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl')
filename = (recordfilename / f'backtest-result-{dtappendix}_signals.pkl')
else:
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_signals.pkl'
recordfilename.parent, f'{recordfilename.stem}-{dtappendix}_signals.pkl'
)
file_dump_joblib(filename, candles)
@@ -416,6 +416,9 @@ def generate_strategy_stats(pairlist: List[str],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
winning_profit = results.loc[results['profit_abs'] > 0, 'profit_abs'].sum()
losing_profit = results.loc[results['profit_abs'] < 0, 'profit_abs'].sum()
profit_factor = winning_profit / abs(losing_profit) if losing_profit else 0.0
backtest_days = (max_date - min_date).days or 1
strat_stats = {
@@ -443,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str],
'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
'profit_factor': profit_factor,
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
@@ -497,8 +501,10 @@ def generate_strategy_stats(pairlist: List[str],
(drawdown_abs, drawdown_start, drawdown_end, high_val, low_val,
max_drawdown) = calculate_max_drawdown(
results, value_col='profit_abs', starting_balance=start_balance)
# max_relative_drawdown = Underwater
(_, _, _, _, _, max_relative_drawdown) = calculate_max_drawdown(
results, value_col='profit_abs', starting_balance=start_balance, relative=True)
strat_stats.update({
'max_drawdown': max_drawdown_legacy, # Deprecated - do not use
'max_drawdown_account': max_drawdown,
@@ -777,6 +783,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])),
('Total profit %', f"{strat_results['profit_total']:.2%}"),
('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'),
('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
in strat_results else 'N/A'),
('Trades per day', strat_results['trades_per_day']),
('Avg. daily profit %',
f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),

View File

@@ -627,8 +627,8 @@ class LocalTrade():
"""
self.close_rate = rate
self.close_date = self.close_date or datetime.utcnow()
self.close_profit = self.calc_profit_ratio()
self.close_profit_abs = self.calc_profit()
self.close_profit = self.calc_profit_ratio(rate)
self.close_profit_abs = self.calc_profit(rate)
self.is_open = False
self.exit_order_status = 'closed'
self.open_order_id = None
@@ -696,10 +696,9 @@ class LocalTrade():
"""
self.open_trade_value = self._calc_open_trade_value()
def calculate_interest(self, interest_rate: Optional[float] = None) -> Decimal:
def calculate_interest(self) -> Decimal:
"""
:param interest_rate: interest_charge for borrowing this coin(optional).
If interest_rate is not set self.interest_rate will be used
Calculate interest for this trade. Only applicable for Margin trading.
"""
zero = Decimal(0.0)
# If nothing was borrowed
@@ -712,34 +711,26 @@ class LocalTrade():
total_seconds = Decimal((now - open_date).total_seconds())
hours = total_seconds / sec_per_hour or zero
rate = Decimal(interest_rate or self.interest_rate)
rate = Decimal(self.interest_rate)
borrowed = Decimal(self.borrowed)
return interest(exchange_name=self.exchange, borrowed=borrowed, rate=rate, hours=hours)
def _calc_base_close(self, amount: Decimal, rate: Optional[float] = None,
fee: Optional[float] = None) -> Decimal:
def _calc_base_close(self, amount: Decimal, rate: float, fee: float) -> Decimal:
close_trade = Decimal(amount) * Decimal(rate or self.close_rate) # type: ignore
fees = close_trade * Decimal(fee or self.fee_close)
close_trade = amount * Decimal(rate)
fees = close_trade * Decimal(fee)
if self.is_short:
return close_trade + fees
else:
return close_trade - fees
def calc_close_trade_value(self, rate: Optional[float] = None,
fee: Optional[float] = None,
interest_rate: Optional[float] = None) -> float:
def calc_close_trade_value(self, rate: float) -> float:
"""
Calculate the close_rate including fee
:param fee: fee to use on the close rate (optional).
If rate is not set self.fee will be used
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
:param interest_rate: interest_charge for borrowing this coin (optional).
If interest_rate is not set self.interest_rate will be used
:return: Price in BTC of the open trade
Calculate the Trade's close value including fees
:param rate: rate to compare with.
:return: value in stake currency of the open trade
"""
if rate is None and not self.close_rate:
return 0.0
@@ -748,49 +739,38 @@ class LocalTrade():
trading_mode = self.trading_mode or TradingMode.SPOT
if trading_mode == TradingMode.SPOT:
return float(self._calc_base_close(amount, rate, fee))
return float(self._calc_base_close(amount, rate, self.fee_close))
elif (trading_mode == TradingMode.MARGIN):
total_interest = self.calculate_interest(interest_rate)
total_interest = self.calculate_interest()
if self.is_short:
amount = amount + total_interest
return float(self._calc_base_close(amount, rate, fee))
return float(self._calc_base_close(amount, rate, self.fee_close))
else:
# Currency already owned for longs, no need to purchase
return float(self._calc_base_close(amount, rate, fee) - total_interest)
return float(self._calc_base_close(amount, rate, self.fee_close) - total_interest)
elif (trading_mode == TradingMode.FUTURES):
funding_fees = self.funding_fees or 0.0
# Positive funding_fees -> Trade has gained from fees.
# Negative funding_fees -> Trade had to pay the fees.
if self.is_short:
return float(self._calc_base_close(amount, rate, fee)) - funding_fees
return float(self._calc_base_close(amount, rate, self.fee_close)) - funding_fees
else:
return float(self._calc_base_close(amount, rate, fee)) + funding_fees
return float(self._calc_base_close(amount, rate, self.fee_close)) + funding_fees
else:
raise OperationalException(
f"{self.trading_mode.value} trading is not yet available using freqtrade")
def calc_profit(self, rate: Optional[float] = None,
fee: Optional[float] = None,
interest_rate: Optional[float] = None) -> float:
def calc_profit(self, rate: float) -> float:
"""
Calculate the absolute profit in stake currency between Close and Open trade
:param fee: fee to use on the close rate (optional).
If fee is not set self.fee will be used
:param rate: close rate to compare with (optional).
If rate is not set self.close_rate will be used
:param interest_rate: interest_charge for borrowing this coin (optional).
If interest_rate is not set self.interest_rate will be used
:return: profit in stake currency as float
:param rate: close rate to compare with.
:return: profit in stake currency as float
"""
close_trade_value = self.calc_close_trade_value(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close),
interest_rate=(interest_rate or self.interest_rate)
)
close_trade_value = self.calc_close_trade_value(rate)
if self.is_short:
profit = self.open_trade_value - close_trade_value
@@ -798,23 +778,13 @@ class LocalTrade():
profit = close_trade_value - self.open_trade_value
return float(f"{profit:.8f}")
def calc_profit_ratio(self, rate: Optional[float] = None,
fee: Optional[float] = None,
interest_rate: Optional[float] = None) -> float:
def calc_profit_ratio(self, rate: float) -> float:
"""
Calculates the profit as ratio (including fee).
:param rate: rate to compare with (optional).
If rate is not set self.close_rate will be used
:param fee: fee to use on the close rate (optional).
:param interest_rate: interest_charge for borrowing this coin (optional).
If interest_rate is not set self.interest_rate will be used
:param rate: rate to compare with.
:return: profit ratio as float
"""
close_trade_value = self.calc_close_trade_value(
rate=(rate or self.close_rate),
fee=(fee or self.fee_close),
interest_rate=(interest_rate or self.interest_rate)
)
close_trade_value = self.calc_close_trade_value(rate)
short_close_zero = (self.is_short and close_trade_value == 0.0)
long_close_zero = (not self.is_short and self.open_trade_value == 0.0)
@@ -831,14 +801,6 @@ class LocalTrade():
return float(f"{profit_ratio:.8f}")
def recalc_trade_from_orders(self):
# We need at least 2 entry orders for averaging amounts and rates.
# TODO: this condition could probably be removed
if len(self.select_filled_orders(self.entry_side)) < 2:
self.stake_amount = self.amount * self.open_rate / self.leverage
# Just in case, still recalc open trade value
self.recalc_open_trade_value()
return
total_amount = 0.0
total_stake = 0.0
@@ -1410,3 +1372,18 @@ class Trade(_DECL_BASE, LocalTrade):
def get_kvals(self, key: Optional[str]) -> List[KeyValue]:
return super().get_kvals(key=key)
@staticmethod
def get_trading_volume(start_date: datetime = datetime.fromtimestamp(0)) -> float:
"""
Get Trade volume based on Orders
NOTE: Not supported in Backtesting.
:returns: Tuple containing (pair, profit_sum)
"""
trading_volume = Order.query.with_entities(
func.sum(Order.cost).label('volume')
).filter(
Order.order_filled_date >= start_date,
Order.status == 'closed'
).scalar()
return trading_volume

View File

@@ -1,6 +1,7 @@
import asyncio
import logging
from copy import deepcopy
from datetime import datetime
from typing import Any, Dict, List
from fastapi import APIRouter, BackgroundTasks, Depends
@@ -102,7 +103,10 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
min_date=min_date, max_date=max_date)
if btconfig.get('export', 'none') == 'trades':
store_backtest_stats(btconfig['exportfilename'], ApiServer._bt.results)
store_backtest_stats(
btconfig['exportfilename'], ApiServer._bt.results,
datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
)
logger.info("Backtest finished.")

View File

@@ -104,6 +104,10 @@ class Profit(BaseModel):
best_pair_profit_ratio: float
winning_trades: int
losing_trades: int
profit_factor: float
max_drawdown: float
max_drawdown_abs: float
trading_volume: Optional[float]
class SellReason(BaseModel):

View File

@@ -18,6 +18,7 @@ from freqtrade import __version__
from freqtrade.configuration.timerange import TimeRange
from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT
from freqtrade.data.history import load_data
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State,
TradingMode)
from freqtrade.exceptions import ExchangeError, PricingError
@@ -415,6 +416,8 @@ class RPC:
durations = []
winning_trades = 0
losing_trades = 0
winning_profit = 0.0
losing_profit = 0.0
for trade in trades:
current_rate: float = 0.0
@@ -430,8 +433,10 @@ class RPC:
profit_closed_ratio.append(profit_ratio)
if trade.close_profit >= 0:
winning_trades += 1
winning_profit += trade.close_profit_abs
else:
losing_trades += 1
losing_profit += trade.close_profit_abs
else:
# Get current rate
try:
@@ -447,6 +452,7 @@ class RPC:
profit_all_ratio.append(profit_ratio)
best_pair = Trade.get_best_pair(start_date)
trading_volume = Trade.get_trading_volume(start_date)
# Prepare data to display
profit_closed_coin_sum = round(sum(profit_closed_coin), 8)
@@ -470,6 +476,21 @@ class RPC:
profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance
profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance
profit_factor = winning_profit / abs(losing_profit) if losing_profit else float('inf')
trades_df = DataFrame([{'close_date': trade.close_date.strftime(DATETIME_PRINT_FORMAT),
'profit_abs': trade.close_profit_abs}
for trade in trades if not trade.is_open])
max_drawdown_abs = 0.0
max_drawdown = 0.0
if len(trades_df) > 0:
try:
(max_drawdown_abs, _, _, _, _, max_drawdown) = calculate_max_drawdown(
trades_df, value_col='profit_abs', starting_balance=starting_balance)
except ValueError:
# ValueError if no losing trade.
pass
profit_all_fiat = self._fiat_converter.convert_amount(
profit_all_coin_sum,
stake_currency,
@@ -508,11 +529,15 @@ class RPC:
'best_pair_profit_ratio': best_pair[1] if best_pair else 0,
'winning_trades': winning_trades,
'losing_trades': losing_trades,
'profit_factor': profit_factor,
'max_drawdown': max_drawdown,
'max_drawdown_abs': max_drawdown_abs,
'trading_volume': trading_volume,
}
def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:
""" Returns current account balance per crypto """
currencies = []
currencies: List[Dict] = []
total = 0.0
try:
tickers = self._freqtrade.exchange.get_tickers(cached=True)
@@ -547,13 +572,12 @@ class RPC:
except (ExchangeError):
logger.warning(f" Could not get rate for pair {coin}.")
continue
total = total + (est_stake or 0)
total = total + est_stake
currencies.append({
'currency': coin,
# TODO: The below can be simplified if we don't assign None to values.
'free': balance.free if balance.free is not None else 0,
'balance': balance.total if balance.total is not None else 0,
'used': balance.used if balance.used is not None else 0,
'free': balance.free,
'balance': balance.total,
'used': balance.used,
'est_stake': est_stake or 0,
'stake': stake_currency,
'side': 'long',
@@ -583,7 +607,6 @@ class RPC:
total, stake_currency, fiat_display_currency) if self._fiat_converter else 0
trade_count = len(Trade.get_trades_proxy())
starting_capital_ratio = 0.0
starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0
starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0
@@ -891,7 +914,7 @@ class RPC:
else:
errors[pair] = {
'error_msg': f"Pair {pair} is not in the current blacklist."
}
}
resp = self._rpc_blacklist()
resp['errors'] = errors
return resp

View File

@@ -236,6 +236,14 @@ class Telegram(RPCHandler):
# This can take up to `timeout` from the call to `start_polling`.
self._updater.stop()
def _exchange_from_msg(self, msg: Dict[str, Any]) -> str:
"""
Extracts the exchange name from the given message.
:param msg: The message to extract the exchange name from.
:return: The exchange name.
"""
return f"{msg['exchange']}{' (dry)' if self._config['dry_run'] else ''}"
def _format_entry_msg(self, msg: Dict[str, Any]) -> str:
if self._rpc._fiat_converter:
msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
@@ -248,7 +256,7 @@ class Telegram(RPCHandler):
entry_side = ({'enter': 'Long', 'entered': 'Longed'} if msg['direction'] == 'Long'
else {'enter': 'Short', 'entered': 'Shorted'})
message = (
f"{emoji} *{msg['exchange']}:*"
f"{emoji} *{self._exchange_from_msg(msg)}:*"
f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}"
f" (#{msg['trade_id']})\n"
)
@@ -297,7 +305,7 @@ class Telegram(RPCHandler):
msg['profit_extra'] = ''
is_fill = msg['type'] == RPCMessageType.EXIT_FILL
message = (
f"{msg['emoji']} *{msg['exchange']}:* "
f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* "
f"{'Exited' if is_fill else 'Exiting'} {msg['pair']} (#{msg['trade_id']})\n"
f"*{'Profit' if is_fill else 'Unrealized Profit'}:* "
f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n"
@@ -327,33 +335,33 @@ class Telegram(RPCHandler):
elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL):
msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit'
message = ("\N{WARNING SIGN} *{exchange}:* "
"Cancelling {message_side} Order for {pair} (#{trade_id}). "
"Reason: {reason}.".format(**msg))
message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* "
f"Cancelling {msg['message_side']} Order for {msg['pair']} "
f"(#{msg['trade_id']}). Reason: {msg['reason']}.")
elif msg_type == RPCMessageType.PROTECTION_TRIGGER:
message = (
"*Protection* triggered due to {reason}. "
"`{pair}` will be locked until `{lock_end_time}`."
).format(**msg)
f"*Protection* triggered due to {msg['reason']}. "
f"`{msg['pair']}` will be locked until `{msg['lock_end_time']}`."
)
elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL:
message = (
"*Protection* triggered due to {reason}. "
"*All pairs* will be locked until `{lock_end_time}`."
).format(**msg)
f"*Protection* triggered due to {msg['reason']}. "
f"*All pairs* will be locked until `{msg['lock_end_time']}`."
)
elif msg_type == RPCMessageType.STATUS:
message = '*Status:* `{status}`'.format(**msg)
message = f"*Status:* `{msg['status']}`"
elif msg_type == RPCMessageType.WARNING:
message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
message = f"\N{WARNING SIGN} *Warning:* `{msg['status']}`"
elif msg_type == RPCMessageType.STARTUP:
message = '{status}'.format(**msg)
message = f"{msg['status']}"
else:
raise NotImplementedError('Unknown message type: {}'.format(msg_type))
raise NotImplementedError(f"Unknown message type: {msg_type}")
return message
def send_msg(self, msg: Dict[str, Any]) -> None:
@@ -723,12 +731,18 @@ class Telegram(RPCHandler):
f"*Total Trade Count:* `{trade_count}`\n"
f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* "
f"`{first_trade_date}`\n"
f"*Latest Trade opened:* `{latest_trade_date}\n`"
f"*Latest Trade opened:* `{latest_trade_date}`\n"
f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`"
)
if stats['closed_trade_count'] > 0:
markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n"
f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`")
markdown_msg += (
f"\n*Avg. Duration:* `{avg_duration}`\n"
f"*Best Performing:* `{best_pair}: {best_pair_profit_ratio:.2%}`\n"
f"*Trading volume:* `{round_coin_value(stats['trading_volume'], stake_cur)}`\n"
f"*Profit factor:* `{stats['profit_factor']:.2f}`\n"
f"*Max Drawdown:* `{stats['max_drawdown']:.2%} "
f"({round_coin_value(stats['max_drawdown_abs'], stake_cur)})`"
)
self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit",
query=update.callback_query)
@@ -868,7 +882,7 @@ class Telegram(RPCHandler):
:return: None
"""
msg = self._rpc._rpc_start()
self._send_msg('Status: `{status}`'.format(**msg))
self._send_msg(f"Status: `{msg['status']}`")
@authorized_only
def _stop(self, update: Update, context: CallbackContext) -> None:
@@ -880,7 +894,7 @@ class Telegram(RPCHandler):
:return: None
"""
msg = self._rpc._rpc_stop()
self._send_msg('Status: `{status}`'.format(**msg))
self._send_msg(f"Status: `{msg['status']}`")
@authorized_only
def _reload_config(self, update: Update, context: CallbackContext) -> None:
@@ -892,7 +906,7 @@ class Telegram(RPCHandler):
:return: None
"""
msg = self._rpc._rpc_reload_config()
self._send_msg('Status: `{status}`'.format(**msg))
self._send_msg(f"Status: `{msg['status']}`")
@authorized_only
def _stopbuy(self, update: Update, context: CallbackContext) -> None:
@@ -904,7 +918,7 @@ class Telegram(RPCHandler):
:return: None
"""
msg = self._rpc._rpc_stopbuy()
self._send_msg('Status: `{status}`'.format(**msg))
self._send_msg(f"Status: `{msg['status']}`")
@authorized_only
def _force_exit(self, update: Update, context: CallbackContext) -> None:
@@ -1066,9 +1080,9 @@ class Telegram(RPCHandler):
trade_id = int(context.args[0])
msg = self._rpc._rpc_delete(trade_id)
self._send_msg((
'`{result_msg}`\n'
f"`{msg['result_msg']}`\n"
'Please make sure to take care of this asset on the exchange manually.'
).format(**msg))
))
except RPCException as e:
self._send_msg(str(e))
@@ -1396,7 +1410,7 @@ class Telegram(RPCHandler):
"*/stopbuy:* `Stops buying, but handles open trades gracefully` \n"
"*/forceexit <trade_id>|all:* `Instantly exits the given trade or all trades, "
"regardless of profit`\n"
"*/fe <trade_id>|all:* `Alias to /forceexit`\n"
"*/fx <trade_id>|all:* `Alias to /forceexit`\n"
f"{force_enter_text if self._config.get('force_entry_enable', False) else ''}"
"*/delete <trade_id>:* `Instantly delete the given trade in the database`\n"
"*/whitelist:* `Show current whitelist` \n"