adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feat/short

Browse Source
This commit is contained in:
Matthias
2022-01-07 10:09:17 +01:00
parent 173524ea5b 7f20f6834b
commit 46809f08fe
27 changed files with 211 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
freqtrade/data/btanalysis.py
View File

@@ -9,21 +9,13 @@ import numpy as np
import pandas as pd
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.exceptions import OperationalException
from freqtrade.misc import json_load
from freqtrade.persistence import LocalTrade, Trade, init_db
logger = logging.getLogger(__name__)
# Old format - maybe remove?
BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
"trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
# Mid-term format, created by BacktestResult Named Tuple
BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
'fee_close', 'amount', 'profit_abs', 'profit_ratio']
# Newest format
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'open_rate', 'close_rate',
@@ -170,23 +162,9 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
)
else:
# old format - only with lists.
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
if not df.empty:
df['open_date'] = pd.to_datetime(df['open_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
# Create compatibility with new format
df['profit_abs'] = df['close_rate'] - df['open_rate']
raise OperationalException(
"Backtest-results with only trades data are no longer supported.")
if not df.empty:
if 'profit_ratio' not in df.columns:
df['profit_ratio'] = df['profit_percent']
df = df.sort_values("open_date").reset_index(drop=True)
return df
@@ -395,15 +373,17 @@ def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date',
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float]:
value_col: str = 'profit_abs', starting_balance: float = 0
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio')
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown,
high and low time and high and low value.
:param value_col: Column in DataFrame to use for values (defaults to 'profit_abs')
:param starting_balance: Portfolio starting balance - properly calculate relative drawdown.
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown)
with absolute max drawdown, high and low time and high and low value,
and the relative account drawdown
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
@@ -419,7 +399,18 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin]
['high_value'].idxmax(), 'cumulative']
low_val = max_drawdown_df.loc[idxmin, 'cumulative']
return abs(min(max_drawdown_df['drawdown'])), high_date, low_date, high_val, low_val
max_drawdown_rel = 0.0
if high_val + starting_balance != 0:
max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance)
return (
abs(min(max_drawdown_df['drawdown'])),
high_date,
low_date,
high_val,
low_val,
max_drawdown_rel
)
def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]:

5
freqtrade/exchange/exchange.py
View File

@@ -65,6 +65,8 @@ class Exchange:
"ohlcv_params": {},
"ohlcv_candle_limit": 500,
"ohlcv_partial_candle": True,
# Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency
"ohlcv_volume_currency": "base", # "base" or "quote"
"trades_pagination": "time", # Possible are "time" or "id"
"trades_pagination_arg": "since",
"l2_limit_range": None,
@@ -797,7 +799,8 @@ class Exchange:
max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage))
remaining_amount = amount
filled_amount = 0
filled_amount = 0.0
book_entry_price = 0.0
for book_entry in ob[ob_type]:
book_entry_price = book_entry[0]
book_entry_coin_volume = book_entry[1]

1
freqtrade/exchange/ftx.py
View File

@@ -20,6 +20,7 @@ class Ftx(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"ohlcv_candle_limit": 1500,
"ohlcv_volume_currency": "quote",
"mark_ohlcv_price": "index",
"mark_ohlcv_timeframe": "1h",
}

1
freqtrade/exchange/gateio.py
View File

@@ -22,6 +22,7 @@ class Gateio(Exchange):
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
"ohlcv_volume_currency": "quote",
}
_headers = {'X-Gate-Channel-Id': 'freqtrade'}

2
freqtrade/exchange/okex.py
View File

@@ -15,7 +15,7 @@ class Okex(Exchange):
"""
_ft_has: Dict = {
"ohlcv_candle_limit": 100,
"ohlcv_candle_limit": 300,
"mark_ohlcv_timeframe": "4h",
"funding_fee_timeframe": "8h",
}

4
freqtrade/optimize/backtesting.py
View File

@@ -448,8 +448,10 @@ class Backtesting:
detail_data.loc[:, 'exit_long'] = sell_row[ELONG_IDX]
detail_data.loc[:, 'enter_long'] = sell_row[LONG_IDX]
detail_data.loc[:, 'exit_long'] = sell_row[ELONG_IDX]
detail_data.loc[:, 'enter_tag'] = sell_row[ENTER_TAG_IDX]
detail_data.loc[:, 'exit_tag'] = sell_row[EXIT_TAG_IDX]
headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short']
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
for det_row in detail_data[headers].values.tolist():
res = self._get_sell_trade_entry_for_candle(trade, det_row)
if res:

3
freqtrade/optimize/hyperopt.py
View File

@@ -76,6 +76,7 @@ class Hyperopt:
self.config = config
self.backtesting = Backtesting(self.config)
self.pairlist = self.backtesting.pairlists.whitelist
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
@@ -332,7 +333,7 @@ class Hyperopt:
params_details = self._get_params_details(params_dict)
strat_stats = generate_strategy_stats(
processed, self.backtesting.strategy.get_strategy_name(),
self.pairlist, self.backtesting.strategy.get_strategy_name(),
backtesting_results, min_date, max_date, market_change=0
)
results_explanation = HyperoptTools.format_results_explanation_string(

3
freqtrade/optimize/hyperopt_loss_calmar.py
View File

@@ -47,10 +47,9 @@ class CalmarHyperOptLoss(IHyperOptLoss):
# calculate max drawdown
try:
_, _, _, high_val, low_val = calculate_max_drawdown(
_, _, _, _, _, max_drawdown = calculate_max_drawdown(
results, value_col="profit_abs"
)
max_drawdown = (high_val - low_val) / high_val
except ValueError:
max_drawdown = 0

43
freqtrade/optimize/hyperopt_tools.py
View File

@@ -308,8 +308,7 @@ class HyperoptTools():
if not has_drawdown:
# Ensure compatibility with older versions of hyperopt results
trials['results_metrics.max_drawdown_abs'] = None
trials['results_metrics.max_drawdown'] = None
trials['results_metrics.max_drawdown_account'] = None
# New mode, using backtest result for metrics
trials['results_metrics.winsdrawslosses'] = trials.apply(
@@ -320,12 +319,15 @@ class HyperoptTools():
'results_metrics.winsdrawslosses',
'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
'results_metrics.profit_total', 'results_metrics.holding_avg',
'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs',
'results_metrics.max_drawdown',
'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'Max Drawdown',
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best']
trials.columns = [
'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account',
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'
]
return trials
@@ -341,9 +343,9 @@ class HyperoptTools():
tabulate.PRESERVE_WHITESPACE = True
trials = json_normalize(results, max_level=1)
has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns
has_account_drawdown = 'results_metrics.max_drawdown_account' in trials.columns
trials = HyperoptTools.prepare_trials_columns(trials, has_drawdown)
trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown)
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '* '
@@ -368,19 +370,20 @@ class HyperoptTools():
stake_currency = config['stake_currency']
if has_drawdown:
trials['Max Drawdown'] = trials.apply(
lambda x: '{} {}'.format(
round_coin_value(x['max_drawdown_abs'], stake_currency),
f"({x['Max Drawdown']:,.2%})".rjust(10, ' ')
).rjust(25 + len(stake_currency))
if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)),
axis=1
)
else:
trials = trials.drop(columns=['Max Drawdown'])
trials[f"Max Drawdown{' (Acct)' if has_account_drawdown else ''}"] = trials.apply(
lambda x: "{} {}".format(
round_coin_value(x['max_drawdown_abs'], stake_currency),
(f"({x['max_drawdown_account']:,.2%})"
if has_account_drawdown
else f"({x['max_drawdown']:,.2%})"
).rjust(10, ' ')
).rjust(25 + len(stake_currency))
if x['max_drawdown'] != 0.0 or x['max_drawdown_account'] != 0.0
else '--'.rjust(25 + len(stake_currency)),
axis=1
)
trials = trials.drop(columns=['max_drawdown_abs'])
trials = trials.drop(columns=['max_drawdown_abs', 'max_drawdown', 'max_drawdown_account'])
trials['Profit'] = trials.apply(
lambda x: '{} {}'.format(

69
freqtrade/optimize/optimize_reports.py
View File

@@ -1,4 +1,5 @@
import logging
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Union
@@ -98,11 +99,11 @@ def _generate_result_line(result: DataFrame, starting_balance: int, first_column
}
def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_balance: int,
def generate_pair_metrics(pairlist: List[str], stake_currency: str, starting_balance: int,
results: DataFrame, skip_nan: bool = False) -> List[Dict]:
"""
Generates and returns a list for the given backtest data and the results dataframe
:param data: Dict of <pair: dataframe> containing data that was used during backtesting.
:param pairlist: Pairlist used
:param stake_currency: stake-currency - used to correctly name headers
:param starting_balance: Starting balance
:param results: Dataframe containing the backtest results
@@ -112,7 +113,7 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_b
tabular_data = []
for pair in data:
for pair in pairlist:
result = results[results['pair'] == pair]
if skip_nan and result['profit_abs'].isnull().all():
continue
@@ -194,29 +195,21 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
return tabular_data
def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
def generate_strategy_comparison(bt_stats: Dict) -> List[Dict]:
"""
Generate summary per strategy
:param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
:param bt_stats: Dict of <Strategyname: DataFrame> containing results for all strategies
:return: List of Dicts containing the metrics per Strategy
"""
tabular_data = []
for strategy, results in all_results.items():
tabular_data.append(_generate_result_line(
results['results'], results['config']['dry_run_wallet'], strategy)
)
try:
max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_ratio')
max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_abs')
except ValueError:
max_drawdown_per = 0
max_drawdown_abs = 0
tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2)
tabular_data[-1]['max_drawdown_abs'] = \
round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False)
for strategy, result in bt_stats.items():
tabular_data.append(deepcopy(result['results_per_pair'][-1]))
# Update "key" to strategy (results_per_pair has it as "Total").
tabular_data[-1]['key'] = strategy
tabular_data[-1]['max_drawdown_account'] = result['max_drawdown_account']
tabular_data[-1]['max_drawdown_abs'] = round_coin_value(
result['max_drawdown_abs'], result['stake_currency'], False)
return tabular_data
@@ -352,14 +345,14 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
}
def generate_strategy_stats(btdata: Dict[str, DataFrame],
def generate_strategy_stats(pairlist: List[str],
strategy: str,
content: Dict[str, Any],
min_date: datetime, max_date: datetime,
market_change: float
) -> Dict[str, Any]:
"""
:param btdata: Backtest data
:param pairlist: List of pairs to backtest
:param strategy: Strategy name
:param content: Backtest result data in the format:
{'results: results, 'config: config}}.
@@ -372,11 +365,11 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
if not isinstance(results, DataFrame):
return {}
config = content['config']
max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
max_open_trades = min(config['max_open_trades'], len(pairlist))
start_balance = config['dry_run_wallet']
stake_currency = config['stake_currency']
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
pair_results = generate_pair_metrics(pairlist, stake_currency=stake_currency,
starting_balance=start_balance,
results=results, skip_nan=False)
@@ -385,7 +378,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
results=results)
left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
left_open_results = generate_pair_metrics(pairlist, stake_currency=stake_currency,
starting_balance=start_balance,
results=results.loc[results['is_open']],
skip_nan=True)
@@ -435,7 +428,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'trades_per_day': round(len(results) / backtest_days, 2),
'market_change': market_change,
'pairlist': list(btdata.keys()),
'pairlist': pairlist,
'stake_amount': config['stake_amount'],
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
@@ -468,12 +461,14 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
}
try:
max_drawdown, _, _, _, _ = calculate_max_drawdown(
max_drawdown_legacy, _, _, _, _, _ = calculate_max_drawdown(
results, value_col='profit_ratio')
drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
results, value_col='profit_abs')
(drawdown_abs, drawdown_start, drawdown_end, high_val, low_val,
max_drawdown) = calculate_max_drawdown(
results, value_col='profit_abs', starting_balance=start_balance)
strat_stats.update({
'max_drawdown': max_drawdown,
'max_drawdown': max_drawdown_legacy, # Deprecated - do not use
'max_drawdown_account': max_drawdown,
'max_drawdown_abs': drawdown_abs,
'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
@@ -493,6 +488,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
except ValueError:
strat_stats.update({
'max_drawdown': 0.0,
'max_drawdown_account': 0.0,
'max_drawdown_abs': 0.0,
'max_drawdown_low': 0.0,
'max_drawdown_high': 0.0,
@@ -521,13 +517,13 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
"""
result: Dict[str, Any] = {'strategy': {}}
market_change = calculate_market_change(btdata, 'close')
pairlist = list(btdata.keys())
for strategy, content in all_results.items():
strat_stats = generate_strategy_stats(btdata, strategy, content,
strat_stats = generate_strategy_stats(pairlist, strategy, content,
min_date, max_date, market_change=market_change)
result['strategy'][strategy] = strat_stats
strategy_results = generate_strategy_comparison(all_results=all_results)
strategy_results = generate_strategy_comparison(bt_stats=result['strategy'])
result['strategy_comparison'] = strategy_results
@@ -652,7 +648,7 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
headers.append('Drawdown')
# Align drawdown string on the center two space separator.
drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
drawdown = [f'{t["max_drawdown_account"] * 100:.2f}' for t in strategy_results]
dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results])
dd_pad_per = max([len(dd) for dd in drawdown])
drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%'
@@ -737,7 +733,10 @@ def text_table_add_metrics(strat_results: Dict) -> str:
('Max balance', round_coin_value(strat_results['csum_max'],
strat_results['stake_currency'])),
('Drawdown', f"{strat_results['max_drawdown']:.2%}"),
# Compatibility to show old hyperopt results
('Drawdown (Account)', f"{strat_results['max_drawdown_account']:.2%}")
if 'max_drawdown_account' in strat_results else (
'Drawdown', f"{strat_results['max_drawdown']:.2%}"),
('Drawdown', round_coin_value(strat_results['max_drawdown_abs'],
strat_results['stake_currency'])),
('Drawdown high', round_coin_value(strat_results['max_drawdown_high'],

2
freqtrade/plot/plotting.py
View File

@@ -161,7 +161,7 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame,
Add scatter points indicating max drawdown
"""
try:
max_drawdown, highdate, lowdate, _, _ = calculate_max_drawdown(trades)
_, highdate, lowdate, _, _, max_drawdown = calculate_max_drawdown(trades)
drawdown = go.Scatter(
x=[highdate, lowdate],

26
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -4,7 +4,6 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from functools import partial
from typing import Any, Dict, List
import arrow
@@ -122,10 +121,17 @@ class VolumePairList(IPairList):
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
_pairlist = [k for k in self._exchange.get_markets(
quote_currencies=[self._stake_currency],
tradable_only=True, active_only=True).keys()]
# No point in testing for blacklisted pairs...
_pairlist = self.verify_blacklist(_pairlist, logger.info)
filtered_tickers = [
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and (self._use_range or v[self._sort_key] is not None))]
and (self._use_range or v[self._sort_key] is not None)
and v['symbol'] in _pairlist)]
pairlist = [s['symbol'] for s in filtered_tickers]
pairlist = self.filter_pairlist(pairlist, tickers)
@@ -181,12 +187,16 @@ class VolumePairList(IPairList):
) in candles else None
# in case of candle data calculate typical price and quoteVolume for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
if self._exchange._ft_has["ohlcv_volume_currency"] == "base":
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
else:
# Exchange ohlcv data is in quote volume already.
pair_candles['quoteVolume'] = pair_candles['volume']
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
quoteVolume = (pair_candles['quoteVolume']
@@ -207,7 +217,7 @@ class VolumePairList(IPairList):
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
pairs = self.verify_blacklist(pairs, logmethod=logger.info)
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]

3
freqtrade/plugins/protections/max_drawdown_protection.py
View File

@@ -55,7 +55,8 @@ class MaxDrawdown(IProtection):
# Drawdown is always positive
try:
drawdown, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit')
# TODO: This should use absolute profit calculation, considering account balance.
drawdown, _, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit')
except ValueError:
return False, None, None