Merge branch 'develop' into pairlocks_direction

This commit is contained in:
Matthias
2022-05-01 14:59:04 +02:00
64 changed files with 637 additions and 547 deletions

View File

@@ -22,6 +22,6 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
# Ensure these modes are using Dry-run
config['dry_run'] = True
validate_config_consistency(config)
validate_config_consistency(config, preliminary=True)
return config

View File

@@ -39,7 +39,7 @@ def _extend_validator(validator_class):
FreqtradeValidator = _extend_validator(Draft4Validator)
def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
def validate_config_schema(conf: Dict[str, Any], preliminary: bool = False) -> Dict[str, Any]:
"""
Validate the configuration follow the Config Schema
:param conf: Config in JSON format
@@ -49,7 +49,10 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE):
conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED
elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT):
conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED
if preliminary:
conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED
else:
conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED_FINAL
else:
conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED
try:
@@ -64,7 +67,7 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]:
)
def validate_config_consistency(conf: Dict[str, Any]) -> None:
def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False) -> None:
"""
Validate the configuration consistency.
Should be ran after loading both configuration and strategy,
@@ -85,7 +88,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
# validate configuration before returning
logger.info('Validating configuration ...')
validate_config_schema(conf)
validate_config_schema(conf, preliminary=preliminary)
def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:

View File

@@ -462,6 +462,10 @@ SCHEMA_BACKTEST_REQUIRED = [
'dataformat_ohlcv',
'dataformat_trades',
]
SCHEMA_BACKTEST_REQUIRED_FINAL = SCHEMA_BACKTEST_REQUIRED + [
'stoploss',
'minimal_roi',
]
SCHEMA_MINIMAL_REQUIRED = [
'exchange',

View File

@@ -5,14 +5,15 @@ import logging
from copy import copy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Union
import numpy as np
import pandas as pd
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.exceptions import OperationalException
from freqtrade.misc import get_backtest_metadata_filename, json_load
from freqtrade.misc import json_load
from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
from freqtrade.persistence import LocalTrade, Trade, init_db
@@ -399,157 +400,3 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame,
trades = trades.loc[(trades['open_date'] >= trades_start) &
(trades['close_date'] <= trades_stop)]
return trades
def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
"""
Calculate market change based on "column".
Calculation is done by taking the first non-null and the last non-null element of each column
and calculating the pctchange as "(last - first) / first".
Then the results per pair are combined as mean.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return:
"""
tmp_means = []
for pair, df in data.items():
start = df[column].dropna().iloc[0]
end = df[column].dropna().iloc[-1]
tmp_means.append((end - start) / start)
return float(np.mean(tmp_means))
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
Combine multiple dataframes "column"
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
:raise: ValueError if no data is provided.
"""
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)
return df_comb
def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
timeframe: str) -> pd.DataFrame:
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:param col_name: Column name that will be assigned the results
:param timeframe: Timeframe used during the operations
:return: Returns df with one additional column, col_name, containing the cumulative profit.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to timeframe to make sure trades match candles
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
)[['profit_abs']].sum()
df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous
df[col_name] = df[col_name].ffill()
return df
def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str
) -> pd.DataFrame:
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
max_drawdown_df['date'] = profit_results.loc[:, date_col]
return max_drawdown_df
def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
):
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio')
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown,
high and low time and high and low value.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
return max_drawdown_df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_abs', starting_balance: float = 0
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_abs')
:param starting_balance: Portfolio starting balance - properly calculate relative drawdown.
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown)
with absolute max drawdown, high and low time and high and low value,
and the relative account drawdown
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
idxmin = max_drawdown_df['drawdown'].idxmin()
if idxmin == 0:
raise ValueError("No losing trade, therefore no drawdown.")
high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
low_date = profit_results.loc[idxmin, date_col]
high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin]
['high_value'].idxmax(), 'cumulative']
low_val = max_drawdown_df.loc[idxmin, 'cumulative']
max_drawdown_rel = 0.0
if high_val + starting_balance != 0:
max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance)
return (
abs(min(max_drawdown_df['drawdown'])),
high_date,
low_date,
high_val,
low_val,
max_drawdown_rel
)
def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]:
"""
Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane
:param trades: DataFrame containing trades (requires columns close_date and profit_percent)
:param starting_balance: Add starting balance to results, to show the wallets high / low points
:return: Tuple (float, float) with cumsum of profit_abs
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
csum_df = pd.DataFrame()
csum_df['sum'] = trades['profit_abs'].cumsum()
csum_min = csum_df['sum'].min() + starting_balance
csum_max = csum_df['sum'].max() + starting_balance
return csum_min, csum_max

173
freqtrade/data/metrics.py Normal file
View File

@@ -0,0 +1,173 @@
import logging
from typing import Dict, Tuple
import numpy as np
import pandas as pd
logger = logging.getLogger(__name__)
def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float:
"""
Calculate market change based on "column".
Calculation is done by taking the first non-null and the last non-null element of each column
and calculating the pctchange as "(last - first) / first".
Then the results per pair are combined as mean.
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return:
"""
tmp_means = []
for pair, df in data.items():
start = df[column].dropna().iloc[0]
end = df[column].dropna().iloc[-1]
tmp_means.append((end - start) / start)
return float(np.mean(tmp_means))
def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
column: str = "close") -> pd.DataFrame:
"""
Combine multiple dataframes "column"
:param data: Dict of Dataframes, dict key should be pair.
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
:raise: ValueError if no data is provided.
"""
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
df_comb['mean'] = df_comb.mean(axis=1)
return df_comb
def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
timeframe: str) -> pd.DataFrame:
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:param col_name: Column name that will be assigned the results
:param timeframe: Timeframe used during the operations
:return: Returns df with one additional column, col_name, containing the cumulative profit.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to timeframe to make sure trades match candles
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
)[['profit_abs']].sum()
df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous
df[col_name] = df[col_name].ffill()
return df
def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str
) -> pd.DataFrame:
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
max_drawdown_df['date'] = profit_results.loc[:, date_col]
return max_drawdown_df
def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
):
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio')
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown,
high and low time and high and low value.
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
return max_drawdown_df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_abs', starting_balance: float = 0
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_abs')
:param starting_balance: Portfolio starting balance - properly calculate relative drawdown.
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown)
with absolute max drawdown, high and low time and high and low value,
and the relative account drawdown
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
idxmin = max_drawdown_df['drawdown'].idxmin()
if idxmin == 0:
raise ValueError("No losing trade, therefore no drawdown.")
high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col]
low_date = profit_results.loc[idxmin, date_col]
high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin]
['high_value'].idxmax(), 'cumulative']
low_val = max_drawdown_df.loc[idxmin, 'cumulative']
max_drawdown_rel = 0.0
if high_val + starting_balance != 0:
max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance)
return (
abs(min(max_drawdown_df['drawdown'])),
high_date,
low_date,
high_val,
low_val,
max_drawdown_rel
)
def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]:
"""
Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane
:param trades: DataFrame containing trades (requires columns close_date and profit_percent)
:param starting_balance: Add starting balance to results, to show the wallets high / low points
:return: Tuple (float, float) with cumsum of profit_abs
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
csum_df = pd.DataFrame()
csum_df['sum'] = trades['profit_abs'].cumsum()
csum_min = csum_df['sum'].min() + starting_balance
csum_max = csum_df['sum'].max() + starting_balance
return csum_min, csum_max
def calculate_cagr(days_passed: int, starting_balance: float, final_balance: float) -> float:
"""
Calculate CAGR
:param days_passed: Days passed between start and ending balance
:param starting_balance: Starting balance
:param final_balance: Final balance to calculate CAGR against
:return: CAGR
"""
return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1

View File

@@ -9,6 +9,7 @@ import logging
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from math import ceil
from threading import Lock
from typing import Any, Coroutine, Dict, List, Literal, Optional, Tuple, Union
import arrow
@@ -64,6 +65,7 @@ class Exchange:
"ohlcv_params": {},
"ohlcv_candle_limit": 500,
"ohlcv_partial_candle": True,
"ohlcv_require_since": False,
# Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency
"ohlcv_volume_currency": "base", # "base" or "quote"
"tickers_have_quoteVolume": True,
@@ -95,6 +97,9 @@ class Exchange:
self._markets: Dict = {}
self._trading_fees: Dict[str, Any] = {}
self._leverage_tiers: Dict[str, List[Dict]] = {}
# Lock event loop. This is necessary to avoid race-conditions when using force* commands
# Due to funding fee fetching.
self._loop_lock = Lock()
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self._config: Dict = {}
@@ -166,7 +171,7 @@ class Exchange:
self._api_async = self._init_ccxt(
exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config)
logger.info('Using Exchange "%s"', self.name)
logger.info(f'Using Exchange "{self.name}"')
if validate:
# Check if timeframe is available
@@ -368,6 +373,9 @@ class Exchange:
return (
market.get('quote', None) is not None
and market.get('base', None) is not None
and (self.precisionMode != TICK_SIZE
# Too low precision will falsify calculations
or market.get('precision', {}).get('price', None) > 1e-11)
and ((self.trading_mode == TradingMode.SPOT and self.market_is_spot(market))
or (self.trading_mode == TradingMode.MARGIN and self.market_is_margin(market))
or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market)))
@@ -551,7 +559,7 @@ class Exchange:
# Therefore we also show that.
raise OperationalException(
f"The ccxt library does not provide the list of timeframes "
f"for the exchange \"{self.name}\" and this exchange "
f"for the exchange {self.name} and this exchange "
f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}")
if timeframe and (timeframe not in self.timeframes):
@@ -781,7 +789,9 @@ class Exchange:
rate: float, leverage: float, params: Dict = {},
stop_loss: bool = False) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount)
# Rounding here must respect to contract sizes
_amount = self._contracts_to_amount(
pair, self.amount_to_precision(pair, self._amount_to_contracts(pair, amount)))
dry_order: Dict[str, Any] = {
'id': order_id,
'symbol': pair,
@@ -1710,7 +1720,8 @@ class Exchange:
def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
since_ms: Optional[int]) -> Coroutine:
if not since_ms and self.required_candle_call_count > 1:
if (not since_ms
and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)):
# Multiple calls for one pair - to get more history
one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
move_to = one_call * self.required_candle_call_count
@@ -1770,7 +1781,8 @@ class Exchange:
async def gather_stuff():
return await asyncio.gather(*input_coro, return_exceptions=True)
results = self.loop.run_until_complete(gather_stuff())
with self._loop_lock:
results = self.loop.run_until_complete(gather_stuff())
for res in results:
if isinstance(res, Exception):
@@ -1829,17 +1841,18 @@ class Exchange:
pair, timeframe, since_ms, s
)
params = deepcopy(self._ft_has.get('ohlcv_params', {}))
candle_limit = self.ohlcv_candle_limit(timeframe)
if candle_type != CandleType.SPOT:
params.update({'price': candle_type})
if candle_type != CandleType.FUNDING_RATE:
data = await self._api_async.fetch_ohlcv(
pair, timeframe=timeframe, since=since_ms,
limit=self.ohlcv_candle_limit(timeframe), params=params)
limit=candle_limit, params=params)
else:
# Funding rate
data = await self._api_async.fetch_funding_rate_history(
pair, since=since_ms,
limit=self.ohlcv_candle_limit(timeframe))
limit=candle_limit)
# Convert funding rate to candle pattern
data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data]
# Some exchanges sort OHLCV in ASC order and others in DESC.
@@ -2026,9 +2039,10 @@ class Exchange:
if not self.exchange_has("fetchTrades"):
raise OperationalException("This exchange does not support downloading Trades.")
return self.loop.run_until_complete(
self._async_get_trade_history(pair=pair, since=since,
until=until, from_id=from_id))
with self._loop_lock:
return self.loop.run_until_complete(
self._async_get_trade_history(pair=pair, since=since,
until=until, from_id=from_id))
@retrier
def _get_funding_fees_from_exchange(self, pair: str, since: Union[datetime, int]) -> float:

View File

@@ -20,6 +20,7 @@ class Ftx(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"ohlcv_candle_limit": 1500,
"ohlcv_require_since": True,
"ohlcv_volume_currency": "quote",
"mark_ohlcv_price": "index",
"mark_ohlcv_timeframe": "1h",

View File

@@ -122,6 +122,8 @@ class FreqtradeBot(LoggingMixin):
self._schedule.every().day.at(t).do(update)
self.last_process = datetime(1970, 1, 1, tzinfo=timezone.utc)
self.strategy.bot_start()
def notify_status(self, msg: str) -> None:
"""
Public method for users of this class (worker, etc.) to send notifications
@@ -588,7 +590,6 @@ class FreqtradeBot(LoggingMixin):
Executes a limit buy for the given pair
:param pair: pair for which we want to create a LIMIT_BUY
:param stake_amount: amount of stake-currency for the pair
:param leverage: amount of leverage applied to this trade
:return: True if a buy order is created, false if it fails.
"""
time_in_force = self.strategy.order_time_in_force['entry']
@@ -667,16 +668,6 @@ class FreqtradeBot(LoggingMixin):
amount = safe_value_fallback(order, 'filled', 'amount')
enter_limit_filled_price = safe_value_fallback(order, 'average', 'price')
# TODO: this might be unnecessary, as we're calling it in update_trade_state.
isolated_liq = self.exchange.get_liquidation_price(
leverage=leverage,
pair=pair,
amount=amount,
open_rate=enter_limit_filled_price,
is_short=is_short
)
interest_rate = self.exchange.get_interest_rate()
# Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
base_currency = self.exchange.get_pair_base_currency(pair)
@@ -705,8 +696,6 @@ class FreqtradeBot(LoggingMixin):
timeframe=timeframe_to_minutes(self.config['timeframe']),
leverage=leverage,
is_short=is_short,
interest_rate=interest_rate,
liquidation_price=isolated_liq,
trading_mode=self.trading_mode,
funding_fees=funding_fees
)
@@ -1376,7 +1365,8 @@ class FreqtradeBot(LoggingMixin):
default_retval=proposed_limit_rate)(
pair=trade.pair, trade=trade,
current_time=datetime.now(timezone.utc),
proposed_rate=proposed_limit_rate, current_profit=current_profit)
proposed_rate=proposed_limit_rate, current_profit=current_profit,
exit_tag=exit_check.exit_reason)
limit = self.get_valid_price(custom_exit_price, proposed_limit_rate)

View File

@@ -2,13 +2,11 @@
Various tool function for Freqtrade and scripts
"""
import gzip
import hashlib
import logging
import re
from copy import deepcopy
from datetime import datetime
from pathlib import Path
from typing import Any, Iterator, List, Union
from typing import Any, Iterator, List
from typing.io import IO
from urllib.parse import urlparse
@@ -251,34 +249,3 @@ def parse_db_uri_for_logging(uri: str):
return uri
pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
def get_strategy_run_id(strategy) -> str:
"""
Generate unique identification hash for a backtest run. Identical config and strategy file will
always return an identical hash.
:param strategy: strategy object.
:return: hex string id.
"""
digest = hashlib.sha1()
config = deepcopy(strategy.config)
# Options that have no impact on results of individual backtest.
not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server')
for k in not_important_keys:
if k in config:
del config[k]
# Explicitly allow NaN values (e.g. max_open_trades).
# as it does not matter for getting the hash.
digest.update(rapidjson.dumps(config, default=str,
number_mode=rapidjson.NM_NAN).encode('utf-8'))
with open(strategy.__file__, 'rb') as fp:
digest.update(fp.read())
return digest.hexdigest().lower()
def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path:
"""Return metadata filename for specified backtest results file."""
filename = Path(filename)
return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}')

View File

@@ -0,0 +1,40 @@
import hashlib
from copy import deepcopy
from pathlib import Path
from typing import Union
import rapidjson
def get_strategy_run_id(strategy) -> str:
"""
Generate unique identification hash for a backtest run. Identical config and strategy file will
always return an identical hash.
:param strategy: strategy object.
:return: hex string id.
"""
digest = hashlib.sha1()
config = deepcopy(strategy.config)
# Options that have no impact on results of individual backtest.
not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server')
for k in not_important_keys:
if k in config:
del config[k]
# Explicitly allow NaN values (e.g. max_open_trades).
# as it does not matter for getting the hash.
digest.update(rapidjson.dumps(config, default=str,
number_mode=rapidjson.NM_NAN).encode('utf-8'))
# Include _ft_params_from_file - so changing parameter files cause cache eviction
digest.update(rapidjson.dumps(
strategy._ft_params_from_file, default=str, number_mode=rapidjson.NM_NAN).encode('utf-8'))
with open(strategy.__file__, 'rb') as fp:
digest.update(fp.read())
return digest.hexdigest().lower()
def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path:
"""Return metadata filename for specified backtest results file."""
filename = Path(filename)
return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}')

44
freqtrade/optimize/backtesting.py Normal file → Executable file
View File

@@ -9,6 +9,7 @@ from copy import deepcopy
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional, Tuple
import pandas as pd
from numpy import nan
from pandas import DataFrame
@@ -23,8 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
TradingMode)
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.misc import get_strategy_run_id
from freqtrade.mixins import LoggingMixin
from freqtrade.optimize.backtest_caching import get_strategy_run_id
from freqtrade.optimize.bt_progress import BTProgress
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_signal_candles,
@@ -53,6 +54,11 @@ ESHORT_IDX = 8 # Exit short
ENTER_TAG_IDX = 9
EXIT_TAG_IDX = 10
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
HEADERS = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
class Backtesting:
"""
@@ -181,6 +187,7 @@ class Backtesting:
# since a "perfect" stoploss-exit is assumed anyway
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
self.strategy.bot_start()
def _load_protections(self, strategy: IStrategy):
if self.config.get('enable_protections', False):
@@ -263,10 +270,18 @@ class Backtesting:
candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"])
)
# Combine data to avoid combining the data per trade.
unavailable_pairs = []
for pair in self.pairlists.whitelist:
if pair not in self.exchange._leverage_tiers:
unavailable_pairs.append(pair)
continue
self.futures_data[pair] = funding_rates_dict[pair].merge(
mark_rates_dict[pair], on='date', how="inner", suffixes=["_fund", "_mark"])
if unavailable_pairs:
raise OperationalException(
f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. "
"It is therefore impossible to backtest with this pair at the moment.")
else:
self.futures_data = {}
@@ -304,10 +319,7 @@ class Backtesting:
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
data: Dict = {}
self.progress.init_step(BacktestState.CONVERT, len(processed))
@@ -319,7 +331,7 @@ class Backtesting:
if not pair_data.empty:
# Cleanup from prior runs
pair_data.drop(headers[5:] + ['buy', 'sell'], axis=1, errors='ignore')
pair_data.drop(HEADERS[5:] + ['buy', 'sell'], axis=1, errors='ignore')
df_analyzed = self.strategy.advise_exit(
self.strategy.advise_entry(pair_data, {'pair': pair}),
@@ -338,7 +350,7 @@ class Backtesting:
# To avoid using data from future, we use entry/exit signals shifted
# from the previous candle
for col in headers[5:]:
for col in HEADERS[5:]:
tag_col = col in ('enter_tag', 'exit_tag')
if col in df_analyzed.columns:
df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace(
@@ -350,7 +362,7 @@ class Backtesting:
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = df_analyzed[headers].values.tolist() if not df_analyzed.empty else []
data[pair] = df_analyzed[HEADERS].values.tolist() if not df_analyzed.empty else []
return data
def _get_close_rate(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple,
@@ -514,10 +526,10 @@ class Backtesting:
exit_candle_time: datetime = row[DATE_IDX].to_pydatetime()
enter = row[SHORT_IDX] if trade.is_short else row[LONG_IDX]
exit_ = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX]
exit_sig = row[ESHORT_IDX] if trade.is_short else row[ELONG_IDX]
exit_ = self.strategy.should_exit(
trade, row[OPEN_IDX], exit_candle_time, # type: ignore
enter=enter, exit_=exit_,
enter=enter, exit_=exit_sig,
low=row[LOW_IDX], high=row[HIGH_IDX]
)
@@ -539,7 +551,8 @@ class Backtesting:
default_retval=closerate)(
pair=trade.pair, trade=trade,
current_time=exit_candle_time,
proposed_rate=closerate, current_profit=current_profit)
proposed_rate=closerate, current_profit=current_profit,
exit_tag=exit_.exit_reason)
# We can't place orders lower than current low.
# freqtrade does not support this in live, and the order would fill immediately
if trade.is_short:
@@ -566,6 +579,7 @@ class Backtesting:
len(row) > EXIT_TAG_IDX
and row[EXIT_TAG_IDX] is not None
and len(row[EXIT_TAG_IDX]) > 0
and exit_.exit_type in (ExitType.EXIT_SIGNAL,)
):
trade.exit_reason = row[EXIT_TAG_IDX]
@@ -624,9 +638,7 @@ class Backtesting:
detail_data.loc[:, 'exit_short'] = row[ESHORT_IDX]
detail_data.loc[:, 'enter_tag'] = row[ENTER_TAG_IDX]
detail_data.loc[:, 'exit_tag'] = row[EXIT_TAG_IDX]
headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long',
'enter_short', 'exit_short', 'enter_tag', 'exit_tag']
for det_row in detail_data[headers].values.tolist():
for det_row in detail_data[HEADERS].values.tolist():
res = self._get_exit_trade_entry_for_candle(trade, det_row)
if res:
return res
@@ -1028,7 +1040,7 @@ class Backtesting:
timerange: TimeRange):
self.progress.init_step(BacktestState.ANALYZE, 0)
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
logger.info(f"Running backtesting for Strategy {strat.get_strategy_name()}")
backtest_start_time = datetime.now(timezone.utc)
self._set_strategy(strat)
@@ -1095,7 +1107,7 @@ class Backtesting:
for t, v in pairresults.open_date.items():
allinds = pairdf.loc[(pairdf['date'] < v)]
signal_inds = allinds.iloc[[-1]]
signal_candles_only_df = signal_candles_only_df.append(signal_inds)
signal_candles_only_df = pd.concat([signal_candles_only_df, signal_inds])
signal_candles_only[pair] = signal_candles_only_df

View File

@@ -44,6 +44,7 @@ class EdgeCli:
self.edge._timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
self.strategy.bot_start()
def start(self) -> None:
result = self.edge.calculate(self.config['exchange']['pair_whitelist'])

View File

@@ -468,6 +468,7 @@ class Hyperopt:
self.backtesting.exchange._api = None
self.backtesting.exchange._api_async = None
self.backtesting.exchange.loop = None # type: ignore
self.backtesting.exchange._loop_lock = None # type: ignore
# self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore

View File

@@ -10,7 +10,7 @@ from typing import Any, Dict
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss

View File

@@ -8,7 +8,7 @@ from datetime import datetime
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss

View File

@@ -9,7 +9,7 @@ individual needs.
"""
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss

View File

@@ -9,10 +9,10 @@ from pandas import DataFrame, to_datetime
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.misc import (decimals_per_coin, file_dump_joblib, file_dump_json,
get_backtest_metadata_filename, round_coin_value)
from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value
from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
logger = logging.getLogger(__name__)
@@ -446,6 +446,7 @@ def generate_strategy_stats(pairlist: List[str],
'profit_total_abs': results['profit_abs'].sum(),
'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
@@ -746,6 +747,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
strat_results['stake_currency'])),
('Total profit %', f"{strat_results['profit_total']:.2%}"),
('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'),
('Trades per day', strat_results['trades_per_day']),
('Avg. daily profit %',
f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),

View File

@@ -429,12 +429,10 @@ class LocalTrade():
def __repr__(self):
open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed'
leverage = self.leverage or 1.0
is_short = self.is_short or False
return (
f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, '
f'is_short={is_short}, leverage={leverage}, '
f'is_short={self.is_short or False}, leverage={self.leverage or 1.0}, '
f'open_rate={self.open_rate:.8f}, open_since={open_since})'
)

View File

@@ -5,12 +5,13 @@ from typing import Any, Dict, List, Optional
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown,
calculate_underwater, combine_dataframes_with_mean,
create_cum_profit, extract_trades_of_period, load_trades)
from freqtrade.data.btanalysis import (analyze_trade_parallelism, extract_trades_of_period,
load_trades)
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import get_timerange, load_data
from freqtrade.data.metrics import (calculate_max_drawdown, calculate_underwater,
combine_dataframes_with_mean, create_cum_profit)
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds
@@ -610,6 +611,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
IStrategy.dp = DataProvider(config, exchange)
strategy.bot_start()
plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count)
timerange = plot_elements['timerange']
trades = plot_elements['trades']

View File

@@ -6,7 +6,7 @@ from typing import Any, Dict, Optional
import pandas as pd
from freqtrade.constants import LongShort
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.data.metrics import calculate_max_drawdown
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn

View File

@@ -23,7 +23,7 @@ class HyperOptLossResolver(IResolver):
object_type = IHyperOptLoss
object_type_str = "HyperoptLoss"
user_subdir = USERPATH_HYPEROPTS
initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve()
initial_search_path = Path(__file__).parent.parent.joinpath('optimize/hyperopt_loss').resolve()
@staticmethod
def load_hyperoptloss(config: Dict) -> IHyperOptLoss:

View File

@@ -217,15 +217,19 @@ class StrategyResolver(IResolver):
raise OperationalException(
"`populate_exit_trend` or `populate_sell_trend` must be implemented.")
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
_populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
_buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
_sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
if any(x == 2 for x in [
strategy._populate_fun_len,
strategy._buy_fun_len,
strategy._sell_fun_len
_populate_fun_len,
_buy_fun_len,
_sell_fun_len
]):
strategy.INTERFACE_VERSION = 1
raise OperationalException(
"Strategy Interface v1 is no longer supported. "
"Please update your strategy to implement "
"`populate_indicators`, `populate_entry_trend` and `populate_exit_trend` "
"with the metadata argument. ")
return strategy
@staticmethod

View File

@@ -84,6 +84,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
ApiServer._bt.strategylist = [strat]
ApiServer._bt.results = {}
ApiServer._bt.load_prior_backtest()

View File

@@ -2,7 +2,7 @@ import logging
from ipaddress import IPv4Address
from typing import Any, Dict
import rapidjson
import orjson
import uvicorn
from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
@@ -24,7 +24,7 @@ class FTJSONResponse(JSONResponse):
Use rapidjson for responses
Handles NaN and Inf / -Inf in a javascript way by default.
"""
return rapidjson.dumps(content).encode("utf-8")
return orjson.dumps(content, option=orjson.OPT_SERIALIZE_NUMPY)
class ApiServer(RPCHandler):

View File

@@ -943,7 +943,7 @@ class Telegram(RPCHandler):
else:
fiat_currency = self._config.get('fiat_display_currency', '')
try:
statlist, head, fiat_profit_sum = self._rpc._rpc_status_table(
statlist, _, _ = self._rpc._rpc_status_table(
self._config['stake_currency'], fiat_currency)
except RPCException:
self._send_msg(msg='No open trade found.')

View File

@@ -23,7 +23,7 @@ class InformativeData:
def informative(timeframe: str, asset: str = '',
fmt: Optional[Union[str, Callable[[Any], str]]] = None,
*,
candle_type: Optional[CandleType] = None,
candle_type: Optional[Union[CandleType, str]] = None,
ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]:
"""
A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to

View File

@@ -3,7 +3,6 @@ IStrategy interface
This module defines the interface to apply for strategies
"""
import logging
import warnings
from abc import ABC, abstractmethod
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple, Union
@@ -44,14 +43,11 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
# Strategy interface version
# Default to version 2
# Version 1 is the initial interface without metadata dict
# Version 1 is the initial interface without metadata dict - deprecated and no longer supported.
# Version 2 populate_* include metadata dict
# Version 3 - First version with short and leverage support
INTERFACE_VERSION: int = 3
_populate_fun_len: int = 0
_buy_fun_len: int = 0
_sell_fun_len: int = 0
_ft_params_from_file: Dict
# associated minimal roi
minimal_roi: Dict = {}
@@ -114,7 +110,7 @@ class IStrategy(ABC, HyperStrategyMixin):
# Class level variables (intentional) containing
# the dataprovider (dp) (access to other candles, historic data, ...)
# and wallets - access to the current balance.
dp: Optional[DataProvider]
dp: DataProvider
wallets: Optional[Wallets] = None
# Filled from configuration
stake_currency: str
@@ -197,6 +193,13 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
return self.populate_sell_trend(dataframe, metadata)
def bot_start(self, **kwargs) -> None:
"""
Called only once after bot instantiation.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
"""
pass
def bot_loop_start(self, **kwargs) -> None:
"""
Called at the start of the bot iteration (one loop).
@@ -359,7 +362,7 @@ class IStrategy(ABC, HyperStrategyMixin):
def custom_exit_price(self, pair: str, trade: Trade,
current_time: datetime, proposed_rate: float,
current_profit: float, **kwargs) -> float:
current_profit: float, exit_tag: Optional[str], **kwargs) -> float:
"""
Custom exit price logic, returning the new exit price.
@@ -372,6 +375,7 @@ class IStrategy(ABC, HyperStrategyMixin):
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in exit_pricing.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param exit_tag: Exit reason.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New exit price value if provided
"""
@@ -1092,12 +1096,7 @@ class IStrategy(ABC, HyperStrategyMixin):
dataframe = _create_and_merge_informative_pair(
self, dataframe, metadata, inf_data, populate_fn)
if self._populate_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
return self.populate_indicators(dataframe) # type: ignore
else:
return self.populate_indicators(dataframe, metadata)
return self.populate_indicators(dataframe, metadata)
def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
@@ -1111,12 +1110,7 @@ class IStrategy(ABC, HyperStrategyMixin):
logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.")
if self._buy_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
df = self.populate_buy_trend(dataframe) # type: ignore
else:
df = self.populate_entry_trend(dataframe, metadata)
df = self.populate_entry_trend(dataframe, metadata)
if 'enter_long' not in df.columns:
df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns')
@@ -1131,14 +1125,8 @@ class IStrategy(ABC, HyperStrategyMixin):
currently traded pair
:return: DataFrame with exit column
"""
logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.")
if self._sell_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
df = self.populate_sell_trend(dataframe) # type: ignore
else:
df = self.populate_exit_trend(dataframe, metadata)
df = self.populate_exit_trend(dataframe, metadata)
if 'exit_long' not in df.columns:
df = df.rename({'sell': 'exit_long'}, axis='columns')
return df

View File

@@ -32,7 +32,7 @@ def custom_entry_price(self, pair: str, current_time: 'datetime', proposed_rate:
def custom_exit_price(self, pair: str, trade: 'Trade',
current_time: 'datetime', proposed_rate: float,
current_profit: float, **kwargs) -> float:
current_profit: float, exit_tag: Optional[str], **kwargs) -> float:
"""
Custom exit price logic, returning the new exit price.
@@ -45,6 +45,7 @@ def custom_exit_price(self, pair: str, trade: 'Trade',
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in exit_pricing.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param exit_tag: Exit reason.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New exit price value if provided
"""