adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into pr/froggleston/7861

Browse Source
This commit is contained in:
Matthias
2023-03-19 15:00:20 +01:00
parent af6fc886f6 7aa56adf15
commit cd9c2c4c23
259 changed files with 23487 additions and 14834 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
freqtrade/data/btanalysis.py
View File

@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Union
import numpy as np
import pandas as pd
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.constants import LAST_BT_RESULT_FN, IntOrInf
from freqtrade.exceptions import OperationalException
from freqtrade.misc import json_load
from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
@@ -20,8 +20,8 @@ from freqtrade.persistence import LocalTrade, Trade, init_db
logger = logging.getLogger(__name__)
# Newest format
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'open_rate', 'close_rate',
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'max_stake_amount', 'amount',
'open_date', 'close_date', 'open_rate', 'close_rate',
'fee_open', 'fee_close', 'trade_duration',
'profit_ratio', 'profit_abs', 'exit_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
@@ -90,7 +90,8 @@ def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str:
return 'hyperopt_results.pickle'
def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = None) -> Path:
def get_latest_hyperopt_file(
directory: Union[Path, str], predef_filename: Optional[str] = None) -> Path:
"""
Get latest hyperopt export based on '.last_result.json'.
:param directory: Directory to search for last result
@@ -193,7 +194,7 @@ def get_backtest_resultlist(dirname: Path):
def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
min_backtest_date: datetime = None) -> Dict[str, Any]:
min_backtest_date: Optional[datetime] = None) -> Dict[str, Any]:
"""
Find existing backtest stats that match specified run IDs and load them.
:param dirname: pathlib.Path object, or string pointing to the file.
@@ -241,6 +242,33 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s
return results
def _load_backtest_data_df_compatibility(df: pd.DataFrame) -> pd.DataFrame:
"""
Compatibility support for older backtest data.
"""
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
# Compatibility support for pre short Columns
if 'is_short' not in df.columns:
df['is_short'] = False
if 'leverage' not in df.columns:
df['leverage'] = 1.0
if 'enter_tag' not in df.columns:
df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1)
if 'max_stake_amount' not in df.columns:
df['max_stake_amount'] = df['stake_amount']
if 'orders' not in df.columns:
df['orders'] = None
return df
def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame:
"""
Load backtest data file.
@@ -269,24 +297,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
data = data['strategy'][strategy]['trades']
df = pd.DataFrame(data)
if not df.empty:
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
# Compatibility support for pre short Columns
if 'is_short' not in df.columns:
df['is_short'] = 0
if 'leverage' not in df.columns:
df['leverage'] = 1.0
if 'enter_tag' not in df.columns:
df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1)
if 'orders' not in df.columns:
df['orders'] = None
df = _load_backtest_data_df_compatibility(df)
else:
# old format - only with lists.
@@ -322,7 +333,7 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF
def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
max_open_trades: int) -> pd.DataFrame:
max_open_trades: IntOrInf) -> pd.DataFrame:
"""
Find overlapping trades by expanding each trade once per period it was open
and then counting overlaps
@@ -335,7 +346,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str,
return df_final[df_final['open_trades'] > max_open_trades]
def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame:
def trade_list_to_dataframe(trades: Union[List[Trade], List[LocalTrade]]) -> pd.DataFrame:
"""
Convert list of Trade objects to pandas Dataframe
:param trades: List of trade objects
@@ -362,7 +373,7 @@ def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataF
filters = []
if strategy:
filters.append(Trade.strategy == strategy)
trades = trade_list_to_dataframe(Trade.get_trades(filters).all())
trades = trade_list_to_dataframe(list(Trade.get_trades(filters).all()))
return trades

108
freqtrade/data/dataprovider.py
View File

@@ -9,14 +9,17 @@ from collections import deque
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from pandas import DataFrame, Timedelta, Timestamp, to_timedelta
from freqtrade.configuration import TimeRange
from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe
from freqtrade.constants import (FULL_DATAFRAME_THRESHOLD, Config, ListPairsWithTimeframes,
PairWithTimeframe)
from freqtrade.data.history import load_pair_history
from freqtrade.enums import CandleType, RPCMessageType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange, timeframe_to_seconds
from freqtrade.exchange.types import OrderBook
from freqtrade.misc import append_candles_to_dataframe
from freqtrade.rpc import RPCManager
from freqtrade.util import PeriodicCache
@@ -120,7 +123,7 @@ class DataProvider:
'type': RPCMessageType.ANALYZED_DF,
'data': {
'key': pair_key,
'df': dataframe,
'df': dataframe.tail(1),
'la': datetime.now(timezone.utc)
}
}
@@ -131,7 +134,7 @@ class DataProvider:
'data': pair_key,
})
def _add_external_df(
def _replace_external_df(
self,
pair: str,
dataframe: DataFrame,
@@ -157,6 +160,87 @@ class DataProvider:
self.__producer_pairs_df[producer_name][pair_key] = (dataframe, _last_analyzed)
logger.debug(f"External DataFrame for {pair_key} from {producer_name} added.")
def _add_external_df(
self,
pair: str,
dataframe: DataFrame,
last_analyzed: datetime,
timeframe: str,
candle_type: CandleType,
producer_name: str = "default"
) -> Tuple[bool, int]:
"""
Append a candle to the existing external dataframe. The incoming dataframe
must have at least 1 candle.
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param candle_type: Any of the enum CandleType (must match trading mode!)
:returns: False if the candle could not be appended, or the int number of missing candles.
"""
pair_key = (pair, timeframe, candle_type)
if dataframe.empty:
# The incoming dataframe must have at least 1 candle
return (False, 0)
if len(dataframe) >= FULL_DATAFRAME_THRESHOLD:
# This is likely a full dataframe
# Add the dataframe to the dataprovider
self._replace_external_df(
pair,
dataframe,
last_analyzed=last_analyzed,
timeframe=timeframe,
candle_type=candle_type,
producer_name=producer_name
)
return (True, 0)
if (producer_name not in self.__producer_pairs_df
or pair_key not in self.__producer_pairs_df[producer_name]):
# We don't have data from this producer yet,
# or we don't have data for this pair_key
# return False and 1000 for the full df
return (False, 1000)
existing_df, _ = self.__producer_pairs_df[producer_name][pair_key]
# CHECK FOR MISSING CANDLES
# Convert the timeframe to a timedelta for pandas
timeframe_delta: Timedelta = to_timedelta(timeframe)
local_last: Timestamp = existing_df.iloc[-1]['date'] # We want the last date from our copy
# We want the first date from the incoming
incoming_first: Timestamp = dataframe.iloc[0]['date']
# Remove existing candles that are newer than the incoming first candle
existing_df1 = existing_df[existing_df['date'] < incoming_first]
candle_difference = (incoming_first - local_last) / timeframe_delta
# If the difference divided by the timeframe is 1, then this
# is the candle we want and the incoming data isn't missing any.
# If the candle_difference is more than 1, that means
# we missed some candles between our data and the incoming
# so return False and candle_difference.
if candle_difference > 1:
return (False, int(candle_difference))
if existing_df1.empty:
appended_df = dataframe
else:
appended_df = append_candles_to_dataframe(existing_df1, dataframe)
# Everything is good, we appended
self._replace_external_df(
pair,
appended_df,
last_analyzed=last_analyzed,
timeframe=timeframe,
candle_type=candle_type,
producer_name=producer_name
)
return (True, 0)
def get_producer_df(
self,
pair: str,
@@ -200,7 +284,7 @@ class DataProvider:
def historic_ohlcv(
self,
pair: str,
timeframe: str = None,
timeframe: Optional[str] = None,
candle_type: str = ''
) -> DataFrame:
"""
@@ -252,7 +336,7 @@ class DataProvider:
def get_pair_dataframe(
self,
pair: str,
timeframe: str = None,
timeframe: Optional[str] = None,
candle_type: str = ''
) -> DataFrame:
"""
@@ -334,16 +418,14 @@ class DataProvider:
def refresh(self,
pairlist: ListPairsWithTimeframes,
helping_pairs: ListPairsWithTimeframes = None) -> None:
helping_pairs: Optional[ListPairsWithTimeframes] = None) -> None:
"""
Refresh data, called with each cycle
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
if helping_pairs:
self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
else:
self._exchange.refresh_latest_ohlcv(pairlist)
final_pairs = (pairlist + helping_pairs) if helping_pairs else pairlist
self._exchange.refresh_latest_ohlcv(final_pairs)
@property
def available_pairs(self) -> ListPairsWithTimeframes:
@@ -358,7 +440,7 @@ class DataProvider:
def ohlcv(
self,
pair: str,
timeframe: str = None,
timeframe: Optional[str] = None,
copy: bool = True,
candle_type: str = ''
) -> DataFrame:
@@ -406,7 +488,7 @@ class DataProvider:
except ExchangeError:
return {}
def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
def orderbook(self, pair: str, maximum: int) -> OrderBook:
"""
Fetch latest l2 orderbook data
Warning: Does a network request - so use with common sense.

17
freqtrade/data/entryexitanalysis.py Executable file → Normal file
View File

@@ -24,11 +24,12 @@ def _load_backtest_analysis_data(backtest_dir: Path, name: str):
scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl")
try:
scp = open(scpf, "rb")
loaded_data = joblib.load(scp)
logger.info(f"Loaded {name} data: {str(scpf)}")
with scpf.open("rb") as scp:
loaded_data = joblib.load(scp)
logger.info(f"Loaded {name} candles: {str(scpf)}")
except Exception as e:
logger.error(f"Cannot load {name} data from pickled results: ", e)
return None
return loaded_data
@@ -60,7 +61,7 @@ def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_cand
return analysed_trades_dict
def _analyze_candles_and_indicators(pair, trades, signal_candles):
def _analyze_candles_and_indicators(pair, trades: pd.DataFrame, signal_candles: pd.DataFrame):
buyf = signal_candles
if len(buyf) > 0:
@@ -129,7 +130,7 @@ def _do_group_table_output(bigdf, glist, to_csv=False, csv_path=None):
else:
agg_mask = {'profit_abs': ['count', 'sum', 'median', 'mean'],
'profit_ratio': ['sum', 'median', 'mean']}
'profit_ratio': ['median', 'mean', 'sum']}
agg_cols = ['num_buys', 'profit_abs_sum', 'profit_abs_median',
'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct',
'total_profit_pct']
@@ -150,6 +151,12 @@ def _do_group_table_output(bigdf, glist, to_csv=False, csv_path=None):
# 4: profit summaries grouped by pair, enter_ and exit_tag (this can get quite large)
if g == "4":
group_mask = ['pair', 'enter_reason', 'exit_reason']
# 5: profit summaries grouped by exit_tag
if g == "5":
group_mask = ['exit_reason']
sortcols = ['exit_reason']
if group_mask:
new = bigdf.groupby(group_mask).agg(agg_mask).reset_index()
new.columns = group_mask + agg_cols

12
freqtrade/data/history/history_utils.py
View File

@@ -28,8 +28,8 @@ def load_pair_history(pair: str,
fill_up_missing: bool = True,
drop_incomplete: bool = False,
startup_candles: int = 0,
data_format: str = None,
data_handler: IDataHandler = None,
data_format: Optional[str] = None,
data_handler: Optional[IDataHandler] = None,
candle_type: CandleType = CandleType.SPOT
) -> DataFrame:
"""
@@ -69,7 +69,7 @@ def load_data(datadir: Path,
fail_without_data: bool = False,
data_format: str = 'json',
candle_type: CandleType = CandleType.SPOT,
user_futures_funding_rate: int = None,
user_futures_funding_rate: Optional[int] = None,
) -> Dict[str, DataFrame]:
"""
Load ohlcv history data for a list of pairs.
@@ -116,7 +116,7 @@ def refresh_data(*, datadir: Path,
timeframe: str,
pairs: List[str],
exchange: Exchange,
data_format: str = None,
data_format: Optional[str] = None,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
) -> None:
@@ -189,7 +189,7 @@ def _download_pair_history(pair: str, *,
timeframe: str = '5m',
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
data_handler: Optional[IDataHandler] = None,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
erase: bool = False,
@@ -272,7 +272,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
datadir: Path, trading_mode: str,
timerange: Optional[TimeRange] = None,
new_pairs_days: int = 30, erase: bool = False,
data_format: str = None,
data_format: Optional[str] = None,
prepend: bool = False,
) -> List[str]:
"""

23
freqtrade/data/history/idatahandler.py
View File

@@ -308,7 +308,7 @@ class IDataHandler(ABC):
timerange=timerange_startup,
candle_type=candle_type
)
if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True):
if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
return pairdf
else:
enddate = pairdf.iloc[-1]['date']
@@ -316,7 +316,7 @@ class IDataHandler(ABC):
if timerange_startup:
self._validate_pairdata(pair, pairdf, timeframe, candle_type, timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data, True):
return pairdf
# incomplete candles should only be dropped if we didn't trim the end beforehand.
@@ -374,6 +374,21 @@ class IDataHandler(ABC):
logger.warning(f"{pair}, {candle_type}, {timeframe}, "
f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")
def rename_futures_data(
self, pair: str, new_pair: str, timeframe: str, candle_type: CandleType):
"""
Temporary method to migrate data from old naming to new naming (BTC/USDT -> BTC/USDT:USDT)
Only used for binance to support the binance futures naming unification.
"""
file_old = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
file_new = self._pair_data_filename(self._datadir, new_pair, timeframe, candle_type)
# print(file_old, file_new)
if file_new.exists():
logger.warning(f"{file_new} exists already, can't migrate {pair}.")
return
file_old.rename(file_new)
def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
"""
@@ -403,8 +418,8 @@ def get_datahandlerclass(datatype: str) -> Type[IDataHandler]:
raise ValueError(f"No datahandler for datatype {datatype} available.")
def get_datahandler(datadir: Path, data_format: str = None,
data_handler: IDataHandler = None) -> IDataHandler:
def get_datahandler(datadir: Path, data_format: Optional[str] = None,
data_handler: Optional[IDataHandler] = None) -> IDataHandler:
"""
:param datadir: Folder to save data
:param data_format: dataformat to use

118
freqtrade/data/metrics.py
View File

@@ -1,4 +1,6 @@
import logging
import math
from datetime import datetime
from typing import Dict, Tuple
import numpy as np
@@ -190,3 +192,119 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
:return: CAGR
"""
return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
def calculate_expectancy(trades: pd.DataFrame) -> float:
"""
Calculate expectancy
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:return: expectancy
"""
if len(trades) == 0:
return 0
expectancy = 1
profit_sum = trades.loc[trades['profit_abs'] > 0, 'profit_abs'].sum()
loss_sum = abs(trades.loc[trades['profit_abs'] < 0, 'profit_abs'].sum())
nb_win_trades = len(trades.loc[trades['profit_abs'] > 0])
nb_loss_trades = len(trades.loc[trades['profit_abs'] < 0])
if (nb_win_trades > 0) and (nb_loss_trades > 0):
average_win = profit_sum / nb_win_trades
average_loss = loss_sum / nb_loss_trades
risk_reward_ratio = average_win / average_loss
winrate = nb_win_trades / len(trades)
expectancy = ((1 + risk_reward_ratio) * winrate) - 1
elif nb_win_trades == 0:
expectancy = 0
return expectancy
def calculate_sortino(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
starting_balance: float) -> float:
"""
Calculate sortino
:param trades: DataFrame containing trades (requires columns profit_abs)
:return: sortino
"""
if (len(trades) == 0) or (min_date is None) or (max_date is None) or (min_date == max_date):
return 0
total_profit = trades['profit_abs'] / starting_balance
days_period = max(1, (max_date - min_date).days)
expected_returns_mean = total_profit.sum() / days_period
down_stdev = np.std(trades.loc[trades['profit_abs'] < 0, 'profit_abs'] / starting_balance)
if down_stdev != 0 and not np.isnan(down_stdev):
sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365)
else:
# Define high (negative) sortino ratio to be clear that this is NOT optimal.
sortino_ratio = -100
# print(expected_returns_mean, down_stdev, sortino_ratio)
return sortino_ratio
def calculate_sharpe(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
starting_balance: float) -> float:
"""
Calculate sharpe
:param trades: DataFrame containing trades (requires column profit_abs)
:return: sharpe
"""
if (len(trades) == 0) or (min_date is None) or (max_date is None) or (min_date == max_date):
return 0
total_profit = trades['profit_abs'] / starting_balance
days_period = max(1, (max_date - min_date).days)
expected_returns_mean = total_profit.sum() / days_period
up_stdev = np.std(total_profit)
if up_stdev != 0:
sharp_ratio = expected_returns_mean / up_stdev * np.sqrt(365)
else:
# Define high (negative) sharpe ratio to be clear that this is NOT optimal.
sharp_ratio = -100
# print(expected_returns_mean, up_stdev, sharp_ratio)
return sharp_ratio
def calculate_calmar(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
starting_balance: float) -> float:
"""
Calculate calmar
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:return: calmar
"""
if (len(trades) == 0) or (min_date is None) or (max_date is None) or (min_date == max_date):
return 0
total_profit = trades['profit_abs'].sum() / starting_balance
days_period = max(1, (max_date - min_date).days)
# adding slippage of 0.1% per trade
# total_profit = total_profit - 0.0005
expected_returns_mean = total_profit / days_period * 100
# calculate max drawdown
try:
_, _, _, _, _, max_drawdown = calculate_max_drawdown(
trades, value_col="profit_abs", starting_balance=starting_balance
)
except ValueError:
max_drawdown = 0
if max_drawdown != 0:
calmar_ratio = expected_returns_mean / max_drawdown * math.sqrt(365)
else:
# Define high (negative) calmar ratio to be clear that this is NOT optimal.
calmar_ratio = -100
# print(expected_returns_mean, max_drawdown, calmar_ratio)
return calmar_ratio