adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'freqtrade:develop' into develop

Browse Source
This commit is contained in:
Stefano Ariestasia
2023-01-06 08:04:33 +08:00
committed by GitHub
parent 3b9052247f 787d292ba0
commit 148338cdf8
105 changed files with 2626 additions and 1191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/__init__.py
View File

@@ -1,5 +1,5 @@
""" Freqtrade bot """
__version__ = '2022.12.dev'
__version__ = '2023.1.dev'
if 'dev' in __version__:
try:

7
freqtrade/configuration/config_validation.py
View File

@@ -355,6 +355,13 @@ def _validate_freqai_include_timeframes(conf: Dict[str, Any]) -> None:
f"Main timeframe of {main_tf} must be smaller or equal to FreqAI "
f"`include_timeframes`.Offending include-timeframes: {', '.join(offending_lines)}")
# Ensure that the base timeframe is included in the include_timeframes list
if main_tf not in freqai_include_timeframes:
feature_parameters = conf.get('freqai', {}).get('feature_parameters', {})
include_timeframes = [main_tf] + freqai_include_timeframes
conf.get('freqai', {}).get('feature_parameters', {}) \
.update({**feature_parameters, 'include_timeframes': include_timeframes})
def _validate_freqai_backtest(conf: Dict[str, Any]) -> None:
if conf.get('runmode', RunMode.OTHER) == RunMode.BACKTEST:

8
freqtrade/constants.py
View File

@@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'CalmarHyperOptLoss',
'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
'ProfitDrawDownHyperOptLoss']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList', 'RemotePairList',
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
@@ -61,6 +61,7 @@ USERPATH_FREQAIMODELS = 'freqaimodels'
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw']
FULL_DATAFRAME_THRESHOLD = 100
ENV_VAR_PREFIX = 'FREQTRADE__'
@@ -608,9 +609,8 @@ CONF_SCHEMA = {
"backtest_period_days",
"identifier",
"feature_parameters",
"data_split_parameters",
"model_training_parameters"
]
"data_split_parameters"
]
},
},
}

50
freqtrade/data/btanalysis.py
View File

@@ -20,8 +20,8 @@ from freqtrade.persistence import LocalTrade, Trade, init_db
logger = logging.getLogger(__name__)
# Newest format
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'open_rate', 'close_rate',
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'max_stake_amount', 'amount',
'open_date', 'close_date', 'open_rate', 'close_rate',
'fee_open', 'fee_close', 'trade_duration',
'profit_ratio', 'profit_abs', 'exit_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
@@ -241,6 +241,33 @@ def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, s
return results
def _load_backtest_data_df_compatibility(df: pd.DataFrame) -> pd.DataFrame:
"""
Compatibility support for older backtest data.
"""
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
# Compatibility support for pre short Columns
if 'is_short' not in df.columns:
df['is_short'] = False
if 'leverage' not in df.columns:
df['leverage'] = 1.0
if 'enter_tag' not in df.columns:
df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1)
if 'max_stake_amount' not in df.columns:
df['max_stake_amount'] = df['stake_amount']
if 'orders' not in df.columns:
df['orders'] = None
return df
def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame:
"""
Load backtest data file.
@@ -269,24 +296,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
data = data['strategy'][strategy]['trades']
df = pd.DataFrame(data)
if not df.empty:
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
# Compatibility support for pre short Columns
if 'is_short' not in df.columns:
df['is_short'] = 0
if 'leverage' not in df.columns:
df['leverage'] = 1.0
if 'enter_tag' not in df.columns:
df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1)
if 'orders' not in df.columns:
df['orders'] = None
df = _load_backtest_data_df_compatibility(df)
else:
# old format - only with lists.

89
freqtrade/data/dataprovider.py
View File

@@ -9,14 +9,16 @@ from collections import deque
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from pandas import DataFrame, to_timedelta
from freqtrade.configuration import TimeRange
from freqtrade.constants import Config, ListPairsWithTimeframes, PairWithTimeframe
from freqtrade.constants import (FULL_DATAFRAME_THRESHOLD, Config, ListPairsWithTimeframes,
PairWithTimeframe)
from freqtrade.data.history import load_pair_history
from freqtrade.enums import CandleType, RPCMessageType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange, timeframe_to_seconds
from freqtrade.misc import append_candles_to_dataframe
from freqtrade.rpc import RPCManager
from freqtrade.util import PeriodicCache
@@ -120,7 +122,7 @@ class DataProvider:
'type': RPCMessageType.ANALYZED_DF,
'data': {
'key': pair_key,
'df': dataframe,
'df': dataframe.tail(1),
'la': datetime.now(timezone.utc)
}
}
@@ -131,7 +133,7 @@ class DataProvider:
'data': pair_key,
})
def _add_external_df(
def _replace_external_df(
self,
pair: str,
dataframe: DataFrame,
@@ -157,6 +159,85 @@ class DataProvider:
self.__producer_pairs_df[producer_name][pair_key] = (dataframe, _last_analyzed)
logger.debug(f"External DataFrame for {pair_key} from {producer_name} added.")
def _add_external_df(
self,
pair: str,
dataframe: DataFrame,
last_analyzed: datetime,
timeframe: str,
candle_type: CandleType,
producer_name: str = "default"
) -> Tuple[bool, int]:
"""
Append a candle to the existing external dataframe. The incoming dataframe
must have at least 1 candle.
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param candle_type: Any of the enum CandleType (must match trading mode!)
:returns: False if the candle could not be appended, or the int number of missing candles.
"""
pair_key = (pair, timeframe, candle_type)
if dataframe.empty:
# The incoming dataframe must have at least 1 candle
return (False, 0)
if len(dataframe) >= FULL_DATAFRAME_THRESHOLD:
# This is likely a full dataframe
# Add the dataframe to the dataprovider
self._replace_external_df(
pair,
dataframe,
last_analyzed=last_analyzed,
timeframe=timeframe,
candle_type=candle_type,
producer_name=producer_name
)
return (True, 0)
if (producer_name not in self.__producer_pairs_df
or pair_key not in self.__producer_pairs_df[producer_name]):
# We don't have data from this producer yet,
# or we don't have data for this pair_key
# return False and 1000 for the full df
return (False, 1000)
existing_df, _ = self.__producer_pairs_df[producer_name][pair_key]
# CHECK FOR MISSING CANDLES
timeframe_delta = to_timedelta(timeframe) # Convert the timeframe to a timedelta for pandas
local_last = existing_df.iloc[-1]['date'] # We want the last date from our copy
incoming_first = dataframe.iloc[0]['date'] # We want the first date from the incoming
# Remove existing candles that are newer than the incoming first candle
existing_df1 = existing_df[existing_df['date'] < incoming_first]
candle_difference = (incoming_first - local_last) / timeframe_delta
# If the difference divided by the timeframe is 1, then this
# is the candle we want and the incoming data isn't missing any.
# If the candle_difference is more than 1, that means
# we missed some candles between our data and the incoming
# so return False and candle_difference.
if candle_difference > 1:
return (False, candle_difference)
if existing_df1.empty:
appended_df = dataframe
else:
appended_df = append_candles_to_dataframe(existing_df1, dataframe)
# Everything is good, we appended
self._replace_external_df(
pair,
appended_df,
last_analyzed=last_analyzed,
timeframe=timeframe,
candle_type=candle_type,
producer_name=producer_name
)
return (True, 0)
def get_producer_df(
self,
pair: str,

4
freqtrade/data/entryexitanalysis.py
View File

@@ -52,7 +52,7 @@ def _process_candles_and_indicators(pairlist, strategy_name, trades, signal_cand
return analysed_trades_dict
def _analyze_candles_and_indicators(pair, trades, signal_candles):
def _analyze_candles_and_indicators(pair, trades: pd.DataFrame, signal_candles: pd.DataFrame):
buyf = signal_candles
if len(buyf) > 0:
@@ -120,7 +120,7 @@ def _do_group_table_output(bigdf, glist):
else:
agg_mask = {'profit_abs': ['count', 'sum', 'median', 'mean'],
'profit_ratio': ['sum', 'median', 'mean']}
'profit_ratio': ['median', 'mean', 'sum']}
agg_cols = ['num_buys', 'profit_abs_sum', 'profit_abs_median',
'profit_abs_mean', 'median_profit_pct', 'mean_profit_pct',
'total_profit_pct']

118
freqtrade/data/metrics.py
View File

@@ -1,4 +1,6 @@
import logging
import math
from datetime import datetime
from typing import Dict, Tuple
import numpy as np
@@ -190,3 +192,119 @@ def calculate_cagr(days_passed: int, starting_balance: float, final_balance: flo
:return: CAGR
"""
return (final_balance / starting_balance) ** (1 / (days_passed / 365)) - 1
def calculate_expectancy(trades: pd.DataFrame) -> float:
"""
Calculate expectancy
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:return: expectancy
"""
if len(trades) == 0:
return 0
expectancy = 1
profit_sum = trades.loc[trades['profit_abs'] > 0, 'profit_abs'].sum()
loss_sum = abs(trades.loc[trades['profit_abs'] < 0, 'profit_abs'].sum())
nb_win_trades = len(trades.loc[trades['profit_abs'] > 0])
nb_loss_trades = len(trades.loc[trades['profit_abs'] < 0])
if (nb_win_trades > 0) and (nb_loss_trades > 0):
average_win = profit_sum / nb_win_trades
average_loss = loss_sum / nb_loss_trades
risk_reward_ratio = average_win / average_loss
winrate = nb_win_trades / len(trades)
expectancy = ((1 + risk_reward_ratio) * winrate) - 1
elif nb_win_trades == 0:
expectancy = 0
return expectancy
def calculate_sortino(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
starting_balance: float) -> float:
"""
Calculate sortino
:param trades: DataFrame containing trades (requires columns profit_abs)
:return: sortino
"""
if (len(trades) == 0) or (min_date is None) or (max_date is None) or (min_date == max_date):
return 0
total_profit = trades['profit_abs'] / starting_balance
days_period = max(1, (max_date - min_date).days)
expected_returns_mean = total_profit.sum() / days_period
down_stdev = np.std(trades.loc[trades['profit_abs'] < 0, 'profit_abs'] / starting_balance)
if down_stdev != 0 and not np.isnan(down_stdev):
sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365)
else:
# Define high (negative) sortino ratio to be clear that this is NOT optimal.
sortino_ratio = -100
# print(expected_returns_mean, down_stdev, sortino_ratio)
return sortino_ratio
def calculate_sharpe(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
starting_balance: float) -> float:
"""
Calculate sharpe
:param trades: DataFrame containing trades (requires column profit_abs)
:return: sharpe
"""
if (len(trades) == 0) or (min_date is None) or (max_date is None) or (min_date == max_date):
return 0
total_profit = trades['profit_abs'] / starting_balance
days_period = max(1, (max_date - min_date).days)
expected_returns_mean = total_profit.sum() / days_period
up_stdev = np.std(total_profit)
if up_stdev != 0:
sharp_ratio = expected_returns_mean / up_stdev * np.sqrt(365)
else:
# Define high (negative) sharpe ratio to be clear that this is NOT optimal.
sharp_ratio = -100
# print(expected_returns_mean, up_stdev, sharp_ratio)
return sharp_ratio
def calculate_calmar(trades: pd.DataFrame, min_date: datetime, max_date: datetime,
starting_balance: float) -> float:
"""
Calculate calmar
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:return: calmar
"""
if (len(trades) == 0) or (min_date is None) or (max_date is None) or (min_date == max_date):
return 0
total_profit = trades['profit_abs'].sum() / starting_balance
days_period = max(1, (max_date - min_date).days)
# adding slippage of 0.1% per trade
# total_profit = total_profit - 0.0005
expected_returns_mean = total_profit / days_period * 100
# calculate max drawdown
try:
_, _, _, _, _, max_drawdown = calculate_max_drawdown(
trades, value_col="profit_abs", starting_balance=starting_balance
)
except ValueError:
max_drawdown = 0
if max_drawdown != 0:
calmar_ratio = expected_returns_mean / max_drawdown * math.sqrt(365)
else:
# Define high (negative) calmar ratio to be clear that this is NOT optimal.
calmar_ratio = -100
# print(expected_returns_mean, max_drawdown, calmar_ratio)
return calmar_ratio

1
freqtrade/exchange/__init__.py
View File

@@ -3,7 +3,6 @@
from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS
from freqtrade.exchange.exchange import Exchange
# isort: on
from freqtrade.exchange.bibox import Bibox
from freqtrade.exchange.binance import Binance
from freqtrade.exchange.bitpanda import Bitpanda
from freqtrade.exchange.bittrex import Bittrex

28
freqtrade/exchange/bibox.py
View File

@@ -1,28 +0,0 @@
""" Bibox exchange subclass """
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Bibox(Exchange):
"""
Bibox exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
# fetchCurrencies API point requires authentication for Bibox,
# so switch it off for Freqtrade load_markets()
@property
def _ccxt_config(self) -> Dict:
# Parameters to add directly to ccxt sync/async initialization.
config = {"has": {"fetchCurrencies": False}}
config.update(super()._ccxt_config)
return config

6
freqtrade/exchange/binance.py
View File

@@ -11,7 +11,7 @@ from freqtrade.enums import CandleType, MarginMode, TradingMode
from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
from freqtrade.exchange.types import Tickers
from freqtrade.exchange.types import OHLCVResponse, Tickers
from freqtrade.misc import deep_merge_dicts, json_load
@@ -31,7 +31,7 @@ class Binance(Exchange):
"ccxt_futures_name": "future"
}
_ft_has_futures: Dict = {
"stoploss_order_types": {"limit": "limit", "market": "market"},
"stoploss_order_types": {"limit": "stop", "market": "stop_market"},
"tickers_have_price": False,
}
@@ -112,7 +112,7 @@ class Binance(Exchange):
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
until_ms: Optional[int] = None
) -> Tuple[str, str, str, List]:
) -> OHLCVResponse:
"""
Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
Does not work for other exchanges, which don't return the earliest data when called with "0"

87
freqtrade/exchange/exchange.py
View File

@@ -36,7 +36,7 @@ from freqtrade.exchange.exchange_utils import (CcxtModuleType, amount_to_contrac
price_to_precision, timeframe_to_minutes,
timeframe_to_msecs, timeframe_to_next_date,
timeframe_to_prev_date, timeframe_to_seconds)
from freqtrade.exchange.types import Ticker, Tickers
from freqtrade.exchange.types import OHLCVResponse, Ticker, Tickers
from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
safe_value_fallback2)
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -1813,32 +1813,18 @@ class Exchange:
:param candle_type: '', mark, index, premiumIndex, or funding_rate
:return: List with candle (OHLCV) data
"""
pair, _, _, data = self.loop.run_until_complete(
pair, _, _, data, _ = self.loop.run_until_complete(
self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms, until_ms=until_ms,
is_new_pair=is_new_pair, candle_type=candle_type))
logger.info(f"Downloaded data for {pair} with length {len(data)}.")
return data
def get_historic_ohlcv_as_df(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType) -> DataFrame:
"""
Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe
:param pair: Pair to download
:param timeframe: Timeframe to get data for
:param since_ms: Timestamp in milliseconds to get history from
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: OHLCV DataFrame
"""
ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, candle_type=candle_type)
return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
until_ms: Optional[int] = None
) -> Tuple[str, str, str, List]:
) -> OHLCVResponse:
"""
Download historic ohlcv
:param is_new_pair: used by binance subclass to allow "fast" new pair downloading
@@ -1869,15 +1855,16 @@ class Exchange:
continue
else:
# Deconstruct tuple if it's not an exception
p, _, c, new_data = res
p, _, c, new_data, _ = res
if p == pair and c == candle_type:
data.extend(new_data)
# Sort data again after extending the result - above calls return in "async order"
data = sorted(data, key=lambda x: x[0])
return pair, timeframe, candle_type, data
return pair, timeframe, candle_type, data, self._ohlcv_partial_candle
def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
since_ms: Optional[int], cache: bool) -> Coroutine:
def _build_coroutine(
self, pair: str, timeframe: str, candle_type: CandleType,
since_ms: Optional[int], cache: bool) -> Coroutine[Any, Any, OHLCVResponse]:
not_all_data = cache and self.required_candle_call_count > 1
if cache and (pair, timeframe, candle_type) in self._klines:
candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
@@ -1914,7 +1901,7 @@ class Exchange:
"""
Build Coroutines to execute as part of refresh_latest_ohlcv
"""
input_coroutines = []
input_coroutines: List[Coroutine[Any, Any, OHLCVResponse]] = []
cached_pairs = []
for pair, timeframe, candle_type in set(pair_list):
if (timeframe not in self.timeframes
@@ -1978,7 +1965,6 @@ class Exchange:
:return: Dict of [{(pair, timeframe): Dataframe}]
"""
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete
# Gather coroutines to run
input_coroutines, cached_pairs = self._build_ohlcv_dl_jobs(pair_list, since_ms, cache)
@@ -1996,8 +1982,9 @@ class Exchange:
if isinstance(res, Exception):
logger.warning(f"Async code raised an exception: {repr(res)}")
continue
# Deconstruct tuple (has 4 elements)
pair, timeframe, c_type, ticks = res
# Deconstruct tuple (has 5 elements)
pair, timeframe, c_type, ticks, drop_hint = res
drop_incomplete = drop_hint if drop_incomplete is None else drop_incomplete
ohlcv_df = self._process_ohlcv_df(
pair, timeframe, c_type, ticks, cache, drop_incomplete)
@@ -2025,7 +2012,7 @@ class Exchange:
timeframe: str,
candle_type: CandleType,
since_ms: Optional[int] = None,
) -> Tuple[str, str, str, List]:
) -> OHLCVResponse:
"""
Asynchronously get candle history data using fetch_ohlcv
:param candle_type: '', mark, index, premiumIndex, or funding_rate
@@ -2035,8 +2022,8 @@ class Exchange:
# Fetch OHLCV asynchronously
s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else ''
logger.debug(
"Fetching pair %s, interval %s, since %s %s...",
pair, timeframe, since_ms, s
"Fetching pair %s, %s, interval %s, since %s %s...",
pair, candle_type, timeframe, since_ms, s
)
params = deepcopy(self._ft_has.get('ohlcv_params', {}))
candle_limit = self.ohlcv_candle_limit(
@@ -2050,11 +2037,12 @@ class Exchange:
limit=candle_limit, params=params)
else:
# Funding rate
data = await self._api_async.fetch_funding_rate_history(
pair, since=since_ms,
limit=candle_limit)
# Convert funding rate to candle pattern
data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data]
data = await self._fetch_funding_rate_history(
pair=pair,
timeframe=timeframe,
limit=candle_limit,
since_ms=since_ms,
)
# Some exchanges sort OHLCV in ASC order and others in DESC.
# Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last)
# while GDAX returns the list of OHLCV in DESC order (newest first, oldest last)
@@ -2064,9 +2052,9 @@ class Exchange:
data = sorted(data, key=lambda x: x[0])
except IndexError:
logger.exception("Error loading %s. Result was %s.", pair, data)
return pair, timeframe, candle_type, []
return pair, timeframe, candle_type, [], self._ohlcv_partial_candle
logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe)
return pair, timeframe, candle_type, data
return pair, timeframe, candle_type, data, self._ohlcv_partial_candle
except ccxt.NotSupported as e:
raise OperationalException(
@@ -2082,6 +2070,24 @@ class Exchange:
raise OperationalException(f'Could not fetch historical candle (OHLCV) data '
f'for pair {pair}. Message: {e}') from e
async def _fetch_funding_rate_history(
self,
pair: str,
timeframe: str,
limit: int,
since_ms: Optional[int] = None,
) -> List[List]:
"""
Fetch funding rate history - used to selectively override this by subclasses.
"""
# Funding rate
data = await self._api_async.fetch_funding_rate_history(
pair, since=since_ms,
limit=limit)
# Convert funding rate to candle pattern
data = [[x['timestamp'], x['fundingRate'], 0, 0, 0, 0] for x in data]
return data
# Fetch historic trades
@retrier_async
@@ -2745,11 +2751,16 @@ class Exchange:
"""
Important: Must be fetching data from cached values as this is used by backtesting!
PERPETUAL:
gateio: https://www.gate.io/help/futures/perpetual/22160/calculation-of-liquidation-price
gateio: https://www.gate.io/help/futures/futures/27724/liquidation-price-bankruptcy-price
> Liquidation Price = (Entry Price ± Margin / Contract Multiplier / Size) /
[ 1 ± (Maintenance Margin Ratio + Taker Rate)]
Wherein, "+" or "-" depends on whether the contract goes long or short:
"-" for long, and "+" for short.
okex: https://www.okex.com/support/hc/en-us/articles/
360053909592-VI-Introduction-to-the-isolated-mode-of-Single-Multi-currency-Portfolio-margin
:param exchange_name:
:param pair: Pair to calculate liquidation price for
:param open_rate: Entry price of position
:param is_short: True if the trade is a short, false otherwise
:param amount: Absolute value of position size incl. leverage (in base currency)
@@ -2789,7 +2800,7 @@ class Exchange:
def get_maintenance_ratio_and_amt(
self,
pair: str,
nominal_value: float = 0.0,
nominal_value: float,
) -> Tuple[float, Optional[float]]:
"""
Important: Must be fetching data from cached values as this is used by backtesting!

7
freqtrade/exchange/types.py
View File

@@ -1,4 +1,6 @@
from typing import Dict, Optional, TypedDict
from typing import Dict, List, Optional, Tuple, TypedDict
from freqtrade.enums import CandleType
class Ticker(TypedDict):
@@ -14,3 +16,6 @@ class Ticker(TypedDict):
Tickers = Dict[str, Ticker]
# pair, timeframe, candleType, OHLCV, drop last?,
OHLCVResponse = Tuple[str, str, CandleType, List, bool]

125
freqtrade/freqai/RL/Base3ActionRLEnv.py Normal file
View File

@@ -0,0 +1,125 @@
import logging
from enum import Enum
from gym import spaces
from freqtrade.freqai.RL.BaseEnvironment import BaseEnvironment, Positions
logger = logging.getLogger(__name__)
class Actions(Enum):
Neutral = 0
Buy = 1
Sell = 2
class Base3ActionRLEnv(BaseEnvironment):
"""
Base class for a 3 action environment
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.actions = Actions
def set_action_space(self):
self.action_space = spaces.Discrete(len(Actions))
def step(self, action: int):
"""
Logic for a single step (incrementing one candle in time)
by the agent
:param: action: int = the action type that the agent plans
to take for the current step.
:returns:
observation = current state of environment
step_reward = the reward from `calculate_reward()`
_done = if the agent "died" or if the candles finished
info = dict passed back to openai gym lib
"""
self._done = False
self._current_tick += 1
if self._current_tick == self._end_tick:
self._done = True
self._update_unrealized_total_profit()
step_reward = self.calculate_reward(action)
self.total_reward += step_reward
self.tensorboard_log(self.actions._member_names_[action])
trade_type = None
if self.is_tradesignal(action):
if action == Actions.Buy.value:
if self._position == Positions.Short:
self._update_total_profit()
self._position = Positions.Long
trade_type = "long"
self._last_trade_tick = self._current_tick
elif action == Actions.Sell.value and self.can_short:
if self._position == Positions.Long:
self._update_total_profit()
self._position = Positions.Short
trade_type = "short"
self._last_trade_tick = self._current_tick
elif action == Actions.Sell.value and not self.can_short:
self._update_total_profit()
self._position = Positions.Neutral
trade_type = "neutral"
self._last_trade_tick = None
else:
print("case not defined")
if trade_type is not None:
self.trade_history.append(
{'price': self.current_price(), 'index': self._current_tick,
'type': trade_type})
if (self._total_profit < self.max_drawdown or
self._total_unrealized_profit < self.max_drawdown):
self._done = True
self._position_history.append(self._position)
info = dict(
tick=self._current_tick,
action=action,
total_reward=self.total_reward,
total_profit=self._total_profit,
position=self._position.value,
trade_duration=self.get_trade_duration(),
current_profit_pct=self.get_unrealized_profit()
)
observation = self._get_observation()
self._update_history(info)
return observation, step_reward, self._done, info
def is_tradesignal(self, action: int) -> bool:
"""
Determine if the signal is a trade signal
e.g.: agent wants a Actions.Buy while it is in a Positions.short
"""
return (
(action == Actions.Buy.value and self._position == Positions.Neutral)
or (action == Actions.Sell.value and self._position == Positions.Long)
or (action == Actions.Sell.value and self._position == Positions.Neutral
and self.can_short)
or (action == Actions.Buy.value and self._position == Positions.Short
and self.can_short)
)
def _is_valid(self, action: int) -> bool:
"""
Determine if the signal is valid.
e.g.: agent wants a Actions.Sell while it is in a Positions.Long
"""
if self.can_short:
return action in [Actions.Buy.value, Actions.Sell.value, Actions.Neutral.value]
else:
if action == Actions.Sell.value and self._position != Positions.Long:
return False
return True

5
freqtrade/freqai/RL/Base4ActionRLEnv.py
View File

@@ -46,9 +46,9 @@ class Base4ActionRLEnv(BaseEnvironment):
self._done = True
self._update_unrealized_total_profit()
step_reward = self.calculate_reward(action)
self.total_reward += step_reward
self.tensorboard_log(self.actions._member_names_[action])
trade_type = None
if self.is_tradesignal(action):
@@ -88,7 +88,8 @@ class Base4ActionRLEnv(BaseEnvironment):
{'price': self.current_price(), 'index': self._current_tick,
'type': trade_type})
if self._total_profit < 1 - self.rl_config.get('max_training_drawdown_pct', 0.8):
if (self._total_profit < self.max_drawdown or
self._total_unrealized_profit < self.max_drawdown):
self._done = True
self._position_history.append(self._position)

1
freqtrade/freqai/RL/Base5ActionRLEnv.py
View File

@@ -49,6 +49,7 @@ class Base5ActionRLEnv(BaseEnvironment):
self._update_unrealized_total_profit()
step_reward = self.calculate_reward(action)
self.total_reward += step_reward
self.tensorboard_log(self.actions._member_names_[action])
trade_type = None
if self.is_tradesignal(action):

68
freqtrade/freqai/RL/BaseEnvironment.py
View File

@@ -2,7 +2,7 @@ import logging
import random
from abc import abstractmethod
from enum import Enum
from typing import Optional, Type
from typing import Optional, Type, Union
import gym
import numpy as np
@@ -11,9 +11,6 @@ from gym import spaces
from gym.utils import seeding
from pandas import DataFrame
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import RunMode
logger = logging.getLogger(__name__)
@@ -47,8 +44,8 @@ class BaseEnvironment(gym.Env):
def __init__(self, df: DataFrame = DataFrame(), prices: DataFrame = DataFrame(),
reward_kwargs: dict = {}, window_size=10, starting_point=True,
id: str = 'baseenv-1', seed: int = 1, config: dict = {},
dp: Optional[DataProvider] = None):
id: str = 'baseenv-1', seed: int = 1, config: dict = {}, live: bool = False,
fee: float = 0.0015, can_short: bool = False):
"""
Initializes the training/eval environment.
:param df: dataframe of features
@@ -59,32 +56,31 @@ class BaseEnvironment(gym.Env):
:param id: string id of the environment (used in backend for multiprocessed env)
:param seed: Sets the seed of the environment higher in the gym.Env object
:param config: Typical user configuration file
:param dp: dataprovider from freqtrade
:param live: Whether or not this environment is active in dry/live/backtesting
:param fee: The fee to use for environmental interactions.
:param can_short: Whether or not the environment can short
"""
self.config = config
self.rl_config = config['freqai']['rl_config']
self.add_state_info = self.rl_config.get('add_state_info', False)
self.id = id
self.seed(seed)
self.reset_env(df, prices, window_size, reward_kwargs, starting_point)
self.max_drawdown = 1 - self.rl_config.get('max_training_drawdown_pct', 0.8)
self.compound_trades = config['stake_amount'] == 'unlimited'
if self.config.get('fee', None) is not None:
self.fee = self.config['fee']
elif dp is not None:
self.fee = dp._exchange.get_fee(symbol=dp.current_whitelist()[0]) # type: ignore
else:
self.fee = 0.0015
self.fee = fee
# set here to default 5Ac, but all children envs can override this
self.actions: Type[Enum] = BaseActions
self.custom_info: dict = {}
self.live: bool = False
if dp:
self.live = dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE)
self.tensorboard_metrics: dict = {}
self.can_short = can_short
self.live = live
if not self.live and self.add_state_info:
self.add_state_info = False
logger.warning("add_state_info is not available in backtesting. Deactivating.")
self.seed(seed)
self.reset_env(df, prices, window_size, reward_kwargs, starting_point)
def reset_env(self, df: DataFrame, prices: DataFrame, window_size: int,
reward_kwargs: dict, starting_point=True):
@@ -139,20 +135,38 @@ class BaseEnvironment(gym.Env):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def tensorboard_log(self, metric: str, value: Union[int, float] = 1, inc: bool = True):
"""
Function builds the tensorboard_metrics dictionary
to be parsed by the TensorboardCallback. This
function is designed for tracking incremented objects,
events, actions inside the training environment.
For example, a user can call this to track the
frequency of occurence of an `is_valid` call in
their `calculate_reward()`:
def calculate_reward(self, action: int) -> float:
if not self._is_valid(action):
self.tensorboard_log("is_valid")
return -2
:param metric: metric to be tracked and incremented
:param value: value to increment `metric` by
:param inc: sets whether the `value` is incremented or not
"""
if not inc or metric not in self.tensorboard_metrics:
self.tensorboard_metrics[metric] = value
else:
self.tensorboard_metrics[metric] += value
def reset_tensorboard_log(self):
self.tensorboard_metrics = {}
def reset(self):
"""
Reset is called at the beginning of every episode
"""
# custom_info is used for episodic reports and tensorboard logging
self.custom_info["Invalid"] = 0
self.custom_info["Hold"] = 0
self.custom_info["Unknown"] = 0
self.custom_info["pnl_factor"] = 0
self.custom_info["duration_factor"] = 0
self.custom_info["reward_exit"] = 0
self.custom_info["reward_hold"] = 0
for action in self.actions:
self.custom_info[f"{action.name}"] = 0
self.reset_tensorboard_log()
self._done = False
@@ -195,7 +209,7 @@ class BaseEnvironment(gym.Env):
"""
features_window = self.signal_features[(
self._current_tick - self.window_size):self._current_tick]
if self.add_state_info and self.live:
if self.add_state_info:
features_and_state = DataFrame(np.zeros((len(features_window), 3)),
columns=['current_profit_pct',
'position',

36
freqtrade/freqai/RL/BaseReinforcementLearningModel.py
View File

@@ -143,18 +143,14 @@ class BaseReinforcementLearningModel(IFreqaiModel):
train_df = data_dictionary["train_features"]
test_df = data_dictionary["test_features"]
env_info = self.pack_env_dict()
self.train_env = self.MyRLEnv(df=train_df,
prices=prices_train,
window_size=self.CONV_WIDTH,
reward_kwargs=self.reward_params,
config=self.config,
dp=self.data_provider)
**env_info)
self.eval_env = Monitor(self.MyRLEnv(df=test_df,
prices=prices_test,
window_size=self.CONV_WIDTH,
reward_kwargs=self.reward_params,
config=self.config,
dp=self.data_provider))
**env_info))
self.eval_callback = EvalCallback(self.eval_env, deterministic=True,
render=False, eval_freq=len(train_df),
best_model_save_path=str(dk.data_path))
@@ -162,6 +158,21 @@ class BaseReinforcementLearningModel(IFreqaiModel):
actions = self.train_env.get_actions()
self.tensorboard_callback = TensorboardCallback(verbose=1, actions=actions)
def pack_env_dict(self) -> Dict[str, Any]:
"""
Create dictionary of environment arguments
"""
env_info = {"window_size": self.CONV_WIDTH,
"reward_kwargs": self.reward_params,
"config": self.config,
"live": self.live,
"can_short": self.can_short}
if self.data_provider:
env_info["fee"] = self.data_provider._exchange \
.get_fee(symbol=self.data_provider.current_whitelist()[0]) # type: ignore
return env_info
@abstractmethod
def fit(self, data_dictionary: Dict[str, Any], dk: FreqaiDataKitchen, **kwargs):
"""
@@ -383,8 +394,8 @@ class BaseReinforcementLearningModel(IFreqaiModel):
def make_env(MyRLEnv: Type[gym.Env], env_id: str, rank: int,
seed: int, train_df: DataFrame, price: DataFrame,
reward_params: Dict[str, int], window_size: int, monitor: bool = False,
config: Dict[str, Any] = {}) -> Callable:
monitor: bool = False,
env_info: Dict[str, Any] = {}) -> Callable:
"""
Utility function for multiprocessed env.
@@ -392,13 +403,14 @@ def make_env(MyRLEnv: Type[gym.Env], env_id: str, rank: int,
:param num_env: (int) the number of environment you wish to have in subprocesses
:param seed: (int) the inital seed for RNG
:param rank: (int) index of the subprocess
:param env_info: (dict) all required arguments to instantiate the environment.
:return: (Callable)
"""
def _init() -> gym.Env:
env = MyRLEnv(df=train_df, prices=price, window_size=window_size,
reward_kwargs=reward_params, id=env_id, seed=seed + rank, config=config)
env = MyRLEnv(df=train_df, prices=price, id=env_id, seed=seed + rank,
**env_info)
if monitor:
env = Monitor(env)
return env

29
freqtrade/freqai/RL/TensorboardCallback.py
View File

@@ -42,19 +42,18 @@ class TensorboardCallback(BaseCallback):
)
def _on_step(self) -> bool:
custom_info = self.training_env.get_attr("custom_info")[0]
self.logger.record("_state/position", self.locals["infos"][0]["position"])
self.logger.record("_state/trade_duration", self.locals["infos"][0]["trade_duration"])
self.logger.record("_state/current_profit_pct", self.locals["infos"]
[0]["current_profit_pct"])
self.logger.record("_reward/total_profit", self.locals["infos"][0]["total_profit"])
self.logger.record("_reward/total_reward", self.locals["infos"][0]["total_reward"])
self.logger.record_mean("_reward/mean_trade_duration", self.locals["infos"]
[0]["trade_duration"])
self.logger.record("_actions/action", self.locals["infos"][0]["action"])
self.logger.record("_actions/_Invalid", custom_info["Invalid"])
self.logger.record("_actions/_Unknown", custom_info["Unknown"])
self.logger.record("_actions/Hold", custom_info["Hold"])
for action in self.actions:
self.logger.record(f"_actions/{action.name}", custom_info[action.name])
local_info = self.locals["infos"][0]
tensorboard_metrics = self.training_env.get_attr("tensorboard_metrics")[0]
for info in local_info:
if info not in ["episode", "terminal_observation"]:
self.logger.record(f"_info/{info}", local_info[info])
for info in tensorboard_metrics:
if info in [action.name for action in self.actions]:
self.logger.record(f"_actions/{info}", tensorboard_metrics[info])
else:
self.logger.record(f"_custom/{info}", tensorboard_metrics[info])
return True

5
freqtrade/freqai/base_models/BaseClassifierModel.py
View File

@@ -95,9 +95,14 @@ class BaseClassifierModel(IFreqaiModel):
self.data_cleaning_predict(dk)
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
if self.CONV_WIDTH == 1:
predictions = np.reshape(predictions, (-1, len(dk.label_list)))
pred_df = DataFrame(predictions, columns=dk.label_list)
predictions_prob = self.model.predict_proba(dk.data_dictionary["prediction_features"])
if self.CONV_WIDTH == 1:
predictions_prob = np.reshape(predictions_prob, (-1, len(self.model.classes_)))
pred_df_prob = DataFrame(predictions_prob, columns=self.model.classes_)
pred_df = pd.concat([pred_df, pred_df_prob], axis=1)

3
freqtrade/freqai/base_models/BaseRegressionModel.py
View File

@@ -95,6 +95,9 @@ class BaseRegressionModel(IFreqaiModel):
self.data_cleaning_predict(dk)
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
if self.CONV_WIDTH == 1:
predictions = np.reshape(predictions, (-1, len(dk.label_list)))
pred_df = DataFrame(predictions, columns=dk.label_list)
pred_df = dk.denormalize_labels_from_metadata(pred_df)

2
freqtrade/freqai/freqai_interface.py
View File

@@ -104,6 +104,7 @@ class IFreqaiModel(ABC):
self.metadata: Dict[str, Any] = self.dd.load_global_metadata_from_disk()
self.data_provider: Optional[DataProvider] = None
self.max_system_threads = max(int(psutil.cpu_count() * 2 - 2), 1)
self.can_short = True # overridden in start() with strategy.can_short
record_params(config, self.full_path)
@@ -133,6 +134,7 @@ class IFreqaiModel(ABC):
self.live = strategy.dp.runmode in (RunMode.DRY_RUN, RunMode.LIVE)
self.dd.set_pair_dict_info(metadata)
self.data_provider = strategy.dp
self.can_short = strategy.can_short
if self.live:
self.inference_timer('start')

11
freqtrade/freqai/prediction_models/ReinforcementLearner.py
View File

@@ -61,7 +61,7 @@ class ReinforcementLearner(BaseReinforcementLearningModel):
model = self.MODELCLASS(self.policy_type, self.train_env, policy_kwargs=policy_kwargs,
tensorboard_log=Path(
dk.full_path / "tensorboard" / dk.pair.split('/')[0]),
**self.freqai_info['model_training_parameters']
**self.freqai_info.get('model_training_parameters', {})
)
else:
logger.info('Continual training activated - starting training from previously '
@@ -100,7 +100,7 @@ class ReinforcementLearner(BaseReinforcementLearningModel):
"""
# first, penalize if the action is not valid
if not self._is_valid(action):
self.custom_info["Invalid"] += 1
self.tensorboard_log("is_valid")
return -2
pnl = self.get_unrealized_profit()
@@ -109,15 +109,12 @@ class ReinforcementLearner(BaseReinforcementLearningModel):
# reward agent for entering trades
if (action == Actions.Long_enter.value
and self._position == Positions.Neutral):
self.custom_info[f"{Actions.Long_enter.name}"] += 1
return 25
if (action == Actions.Short_enter.value
and self._position == Positions.Neutral):
self.custom_info[f"{Actions.Short_enter.name}"] += 1
return 25
# discourage agent from not entering trades
if action == Actions.Neutral.value and self._position == Positions.Neutral:
self.custom_info[f"{Actions.Neutral.name}"] += 1
return -1
max_trade_duration = self.rl_config.get('max_trade_duration_candles', 300)
@@ -131,22 +128,18 @@ class ReinforcementLearner(BaseReinforcementLearningModel):
# discourage sitting in position
if (self._position in (Positions.Short, Positions.Long) and
action == Actions.Neutral.value):
self.custom_info["Hold"] += 1
return -1 * trade_duration / max_trade_duration
# close long
if action == Actions.Long_exit.value and self._position == Positions.Long:
if pnl > self.profit_aim * self.rr:
factor *= self.rl_config['model_reward_parameters'].get('win_reward_factor', 2)
self.custom_info[f"{Actions.Long_exit.name}"] += 1
return float(pnl * factor)
# close short
if action == Actions.Short_exit.value and self._position == Positions.Short:
if pnl > self.profit_aim * self.rr:
factor *= self.rl_config['model_reward_parameters'].get('win_reward_factor', 2)
self.custom_info[f"{Actions.Short_exit.name}"] += 1
return float(pnl * factor)
self.custom_info["Unknown"] += 1
return 0.

13
freqtrade/freqai/prediction_models/ReinforcementLearner_multiproc.py
View File

@@ -34,17 +34,20 @@ class ReinforcementLearner_multiproc(ReinforcementLearner):
train_df = data_dictionary["train_features"]
test_df = data_dictionary["test_features"]
env_info = self.pack_env_dict()
env_id = "train_env"
self.train_env = SubprocVecEnv([make_env(self.MyRLEnv, env_id, i, 1, train_df, prices_train,
self.reward_params, self.CONV_WIDTH, monitor=True,
config=self.config) for i
self.train_env = SubprocVecEnv([make_env(self.MyRLEnv, env_id, i, 1,
train_df, prices_train,
monitor=True,
env_info=env_info) for i
in range(self.max_threads)])
eval_env_id = 'eval_env'
self.eval_env = SubprocVecEnv([make_env(self.MyRLEnv, eval_env_id, i, 1,
test_df, prices_test,
self.reward_params, self.CONV_WIDTH, monitor=True,
config=self.config) for i
monitor=True,
env_info=env_info) for i
in range(self.max_threads)])
self.eval_callback = EvalCallback(self.eval_env, deterministic=True,
render=False, eval_freq=len(train_df),

10
freqtrade/freqtradebot.py
View File

@@ -155,6 +155,8 @@ class FreqtradeBot(LoggingMixin):
self.cancel_all_open_orders()
self.check_for_open_trades()
except Exception as e:
logger.warning(f'Exception during cleanup: {e.__class__.__name__} {e}')
finally:
self.strategy.ft_bot_cleanup()
@@ -162,8 +164,13 @@ class FreqtradeBot(LoggingMixin):
self.rpc.cleanup()
if self.emc:
self.emc.shutdown()
Trade.commit()
self.exchange.close()
try:
Trade.commit()
except Exception:
# Exeptions here will be happening if the db disappeared.
# At which point we can no longer commit anyway.
pass
def startup(self) -> None:
"""
@@ -905,6 +912,7 @@ class FreqtradeBot(LoggingMixin):
stake_amount=stake_amount,
min_stake_amount=min_stake_amount,
max_stake_amount=max_stake_amount,
trade_amount=trade.stake_amount if trade else None,
)
return enter_limit_requested, stake_amount, leverage

18
freqtrade/misc.py
View File

@@ -301,3 +301,21 @@ def remove_entry_exit_signals(dataframe: pd.DataFrame):
dataframe[SignalTagType.EXIT_TAG.value] = None
return dataframe
def append_candles_to_dataframe(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
"""
Append the `right` dataframe to the `left` dataframe
:param left: The full dataframe you want appended to
:param right: The new dataframe containing the data you want appended
:returns: The dataframe with the right data in it
"""
if left.iloc[-1]['date'] != right.iloc[-1]['date']:
left = pd.concat([left, right])
# Only keep the last 1500 candles in memory
left = left[-1500:] if len(left) > 1500 else left
left.reset_index(drop=True, inplace=True)
return left

2
freqtrade/optimize/backtesting.py
View File

@@ -769,6 +769,7 @@ class Backtesting:
stake_amount=stake_amount,
min_stake_amount=min_stake_amount,
max_stake_amount=max_stake_amount,
trade_amount=trade.stake_amount if trade else None
)
return propose_rate, stake_amount_val, leverage, min_stake_amount
@@ -1176,6 +1177,7 @@ class Backtesting:
open_trade_count_start = self.backtest_loop(
row, pair, current_time, end_date, max_open_trades,
open_trade_count_start)
continue
detail_data.loc[:, 'enter_long'] = row[LONG_IDX]
detail_data.loc[:, 'exit_long'] = row[ELONG_IDX]
detail_data.loc[:, 'enter_short'] = row[SHORT_IDX]

14
freqtrade/optimize/optimize_reports.py
View File

@@ -9,8 +9,9 @@ from tabulate import tabulate
from freqtrade.constants import (DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT,
Config)
from freqtrade.data.metrics import (calculate_cagr, calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.data.metrics import (calculate_cagr, calculate_calmar, calculate_csum,
calculate_expectancy, calculate_market_change,
calculate_max_drawdown, calculate_sharpe, calculate_sortino)
from freqtrade.misc import decimals_per_coin, file_dump_joblib, file_dump_json, round_coin_value
from freqtrade.optimize.backtest_caching import get_backtest_metadata_filename
@@ -448,6 +449,10 @@ def generate_strategy_stats(pairlist: List[str],
'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
'cagr': calculate_cagr(backtest_days, start_balance, content['final_balance']),
'expectancy': calculate_expectancy(results),
'sortino': calculate_sortino(results, min_date, max_date, start_balance),
'sharpe': calculate_sharpe(results, min_date, max_date, start_balance),
'calmar': calculate_calmar(results, min_date, max_date, start_balance),
'profit_factor': profit_factor,
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
@@ -785,8 +790,13 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])),
('Total profit %', f"{strat_results['profit_total']:.2%}"),
('CAGR %', f"{strat_results['cagr']:.2%}" if 'cagr' in strat_results else 'N/A'),
('Sortino', f"{strat_results['sortino']:.2f}" if 'sortino' in strat_results else 'N/A'),
('Sharpe', f"{strat_results['sharpe']:.2f}" if 'sharpe' in strat_results else 'N/A'),
('Calmar', f"{strat_results['calmar']:.2f}" if 'calmar' in strat_results else 'N/A'),
('Profit factor', f'{strat_results["profit_factor"]:.2f}' if 'profit_factor'
in strat_results else 'N/A'),
('Expectancy', f"{strat_results['expectancy']:.2f}" if 'expectancy'
in strat_results else 'N/A'),
('Trades per day', strat_results['trades_per_day']),
('Avg. daily profit %',
f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),

15
freqtrade/persistence/migrations.py
View File

@@ -109,11 +109,10 @@ def migrate_trades_and_orders_table(
else:
is_short = get_column_def(cols, 'is_short', '0')
# Margin Properties
# Futures Properties
interest_rate = get_column_def(cols, 'interest_rate', '0.0')
# Futures properties
funding_fees = get_column_def(cols, 'funding_fees', '0.0')
max_stake_amount = get_column_def(cols, 'max_stake_amount', 'stake_amount')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
@@ -162,7 +161,8 @@ def migrate_trades_and_orders_table(
timeframe, open_trade_value, close_profit_abs,
trading_mode, leverage, liquidation_price, is_short,
interest_rate, funding_fees, realized_profit,
amount_precision, price_precision, precision_mode, contract_size
amount_precision, price_precision, precision_mode, contract_size,
max_stake_amount
)
select id, lower(exchange), pair, {base_currency} base_currency,
{stake_currency} stake_currency,
@@ -190,7 +190,8 @@ def migrate_trades_and_orders_table(
{is_short} is_short, {interest_rate} interest_rate,
{funding_fees} funding_fees, {realized_profit} realized_profit,
{amount_precision} amount_precision, {price_precision} price_precision,
{precision_mode} precision_mode, {contract_size} contract_size
{precision_mode} precision_mode, {contract_size} contract_size,
{max_stake_amount} max_stake_amount
from {trade_back_name}
"""))
@@ -310,8 +311,8 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
# if ('orders' not in previous_tables
# or not has_column(cols_orders, 'funding_fee')):
migrating = False
# if not has_column(cols_trades, 'contract_size'):
if not has_column(cols_orders, 'funding_fee'):
# if not has_column(cols_orders, 'funding_fee'):
if not has_column(cols_trades, 'max_stake_amount'):
migrating = True
logger.info(f"Running database migration for trades - "
f"backup: {table_back_name}, {order_table_bak_name}")

14
freqtrade/persistence/trade_model.py
View File

@@ -293,6 +293,7 @@ class LocalTrade():
close_profit: Optional[float] = None
close_profit_abs: Optional[float] = None
stake_amount: float = 0.0
max_stake_amount: float = 0.0
amount: float = 0.0
amount_requested: Optional[float] = None
open_date: datetime
@@ -397,12 +398,6 @@ class LocalTrade():
def close_date_utc(self):
return self.close_date.replace(tzinfo=timezone.utc)
@property
def enter_side(self) -> str:
""" DEPRECATED, please use entry_side instead"""
# TODO: Please remove me after 2022.5
return self.entry_side
@property
def entry_side(self) -> str:
if self.is_short:
@@ -475,8 +470,8 @@ class LocalTrade():
'amount': round(self.amount, 8),
'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
'stake_amount': round(self.stake_amount, 8),
'max_stake_amount': round(self.max_stake_amount, 8) if self.max_stake_amount else None,
'strategy': self.strategy,
'buy_tag': self.enter_tag,
'enter_tag': self.enter_tag,
'timeframe': self.timeframe,
@@ -513,7 +508,6 @@ class LocalTrade():
'profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None,
'profit_abs': self.close_profit_abs,
'sell_reason': self.exit_reason, # Deprecated
'exit_reason': self.exit_reason,
'exit_order_status': self.exit_order_status,
'stop_loss_abs': self.stop_loss,
@@ -882,6 +876,7 @@ class LocalTrade():
ZERO = FtPrecise(0.0)
current_amount = FtPrecise(0.0)
current_stake = FtPrecise(0.0)
max_stake_amount = FtPrecise(0.0)
total_stake = 0.0 # Total stake after all buy orders (does not subtract!)
avg_price = FtPrecise(0.0)
close_profit = 0.0
@@ -923,7 +918,9 @@ class LocalTrade():
exit_rate, amount=exit_amount, open_rate=avg_price)
else:
total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price)
max_stake_amount += (tmp_amount * price)
self.funding_fees = funding_fees
self.max_stake_amount = float(max_stake_amount)
if close_profit:
self.close_profit = close_profit
@@ -1175,6 +1172,7 @@ class Trade(_DECL_BASE, LocalTrade):
close_profit = Column(Float)
close_profit_abs = Column(Float)
stake_amount = Column(Float, nullable=False)
max_stake_amount = Column(Float)
amount = Column(Float)
amount_requested = Column(Float)
open_date = Column(DateTime, nullable=False, default=datetime.utcnow)

206
freqtrade/plugins/pairlist/RemotePairList.py Normal file
View File

@@ -0,0 +1,206 @@
"""
Remote PairList provider
Provides pair list fetched from a remote source
"""
import json
import logging
from pathlib import Path
from typing import Any, Dict, List, Tuple
import requests
from cachetools import TTLCache
from freqtrade import __version__
from freqtrade.constants import Config
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class RemotePairList(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Config, pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
if 'number_assets' not in self._pairlistconfig:
raise OperationalException(
'`number_assets` not specified. Please check your configuration '
'for "pairlist.config.number_assets"')
if 'pairlist_url' not in self._pairlistconfig:
raise OperationalException(
'`pairlist_url` not specified. Please check your configuration '
'for "pairlist.config.pairlist_url"')
self._number_pairs = self._pairlistconfig['number_assets']
self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
self._bearer_token = self._pairlistconfig.get('bearer_token', '')
self._init_done = False
self._last_pairlist: List[Any] = list()
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return False
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."
def process_json(self, jsonparse) -> List[str]:
pairlist = jsonparse.get('pairs', [])
remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))
if self._refresh_period < remote_refresh_period:
self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
self._refresh_period = remote_refresh_period
self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)
self._init_done = True
return pairlist
def return_last_pairlist(self) -> List[str]:
if self._keep_pairlist_on_failure:
pairlist = self._last_pairlist
self.log_once('Keeping last fetched pairlist', logger.info)
else:
pairlist = []
return pairlist
def fetch_pairlist(self) -> Tuple[List[str], float]:
headers = {
'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
}
if self._bearer_token:
headers['Authorization'] = f'Bearer {self._bearer_token}'
try:
response = requests.get(self._pairlist_url, headers=headers,
timeout=self._read_timeout)
content_type = response.headers.get('content-type')
time_elapsed = response.elapsed.total_seconds()
if "application/json" in str(content_type):
jsonparse = response.json()
try:
pairlist = self.process_json(jsonparse)
except Exception as e:
if self._init_done:
pairlist = self.return_last_pairlist()
logger.warning(f'Error while processing JSON data: {type(e)}')
else:
raise OperationalException(f'Error while processing JSON data: {type(e)}')
else:
if self._init_done:
self.log_once(f'Error: RemotePairList is not of type JSON: '
f' {self._pairlist_url}', logger.info)
pairlist = self.return_last_pairlist()
else:
raise OperationalException('RemotePairList is not of type JSON, abort.')
except requests.exceptions.RequestException:
self.log_once(f'Was not able to fetch pairlist from:'
f' {self._pairlist_url}', logger.info)
pairlist = self.return_last_pairlist()
time_elapsed = 0
return pairlist, time_elapsed
def gen_pairlist(self, tickers: Tickers) -> List[str]:
"""
Generate the pairlist
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: List of pairs
"""
if self._init_done:
pairlist = self._pair_cache.get('pairlist')
else:
pairlist = []
time_elapsed = 0.0
if pairlist:
# Item found - no refresh necessary
return pairlist.copy()
else:
if self._pairlist_url.startswith("file:///"):
filename = self._pairlist_url.split("file:///", 1)[1]
file_path = Path(filename)
if file_path.exists():
with open(filename) as json_file:
# Load the JSON data into a dictionary
jsonparse = json.load(json_file)
try:
pairlist = self.process_json(jsonparse)
except Exception as e:
if self._init_done:
pairlist = self.return_last_pairlist()
logger.warning(f'Error while processing JSON data: {type(e)}')
else:
raise OperationalException('Error while processing'
f'JSON data: {type(e)}')
else:
raise ValueError(f"{self._pairlist_url} does not exist.")
else:
# Fetch Pairlist from Remote URL
pairlist, time_elapsed = self.fetch_pairlist()
self.log_once(f"Fetched pairs: {pairlist}", logger.debug)
pairlist = self._whitelist_for_active_markets(pairlist)
pairlist = pairlist[:self._number_pairs]
self._pair_cache['pairlist'] = pairlist.copy()
if time_elapsed != 0.0:
self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
else:
self.log_once('Fetched Pairlist.', logger.info)
self._last_pairlist = list(pairlist)
return pairlist
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers). May be cached.
:return: new whitelist
"""
rpl_pairlist = self.gen_pairlist(tickers)
merged_list = pairlist + rpl_pairlist
merged_list = sorted(set(merged_list), key=merged_list.index)
return merged_list

4
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -135,7 +135,7 @@ class VolumePairList(IPairList):
filtered_tickers = [
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and (self._use_range or v[self._sort_key] is not None)
and (self._use_range or v.get(self._sort_key) is not None)
and v['symbol'] in _pairlist)]
pairlist = [s['symbol'] for s in filtered_tickers]
else:
@@ -218,7 +218,7 @@ class VolumePairList(IPairList):
else:
filtered_tickers[i]['quoteVolume'] = 0
else:
# Tickers mode - filter based on incomming pairlist.
# Tickers mode - filter based on incoming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
if self._min_value > 0:

8
freqtrade/rpc/api_server/api_backtest.py
View File

@@ -11,6 +11,7 @@ from freqtrade.configuration.config_validation import validate_config_consistenc
from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
from freqtrade.enums import BacktestState
from freqtrade.exceptions import DependencyException
from freqtrade.misc import deep_merge_dicts
from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest,
BacktestResponse)
from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode
@@ -37,10 +38,11 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
btconfig = deepcopy(config)
settings = dict(bt_settings)
if settings.get('freqai', None) is not None:
settings['freqai'] = dict(settings['freqai'])
# Pydantic models will contain all keys, but non-provided ones are None
for setting in settings.keys():
if settings[setting] is not None:
btconfig[setting] = settings[setting]
btconfig = deep_merge_dicts(settings, btconfig, allow_null_overrides=False)
try:
btconfig['stake_amount'] = float(btconfig['stake_amount'])
except ValueError:

14
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -217,8 +217,8 @@ class TradeSchema(BaseModel):
amount: float
amount_requested: float
stake_amount: float
max_stake_amount: Optional[float]
strategy: str
buy_tag: Optional[str] # Deprecated
enter_tag: Optional[str]
timeframe: int
fee_open: Optional[float]
@@ -243,7 +243,6 @@ class TradeSchema(BaseModel):
profit_pct: Optional[float]
profit_abs: Optional[float]
profit_fiat: Optional[float]
sell_reason: Optional[str] # Deprecated
exit_reason: Optional[str]
exit_order_status: Optional[str]
stop_loss_abs: Optional[float]
@@ -372,6 +371,10 @@ class StrategyListResponse(BaseModel):
strategies: List[str]
class FreqAIModelListResponse(BaseModel):
freqaimodels: List[str]
class StrategyResponse(BaseModel):
strategy: str
code: str
@@ -410,6 +413,10 @@ class PairHistory(BaseModel):
}
class BacktestFreqAIInputs(BaseModel):
identifier: str
class BacktestRequest(BaseModel):
strategy: str
timeframe: Optional[str]
@@ -419,6 +426,9 @@ class BacktestRequest(BaseModel):
stake_amount: Optional[str]
enable_protections: bool
dry_run_wallet: Optional[float]
backtest_cache: Optional[str]
freqaimodel: Optional[str]
freqai: Optional[BacktestFreqAIInputs]
class BacktestResponse(BaseModel):

26
freqtrade/rpc/api_server/api_v1.py
View File

@@ -13,12 +13,13 @@ from freqtrade.rpc import RPC
from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload,
BlacklistResponse, Count, Daily,
DeleteLockRequest, DeleteTrade, ForceEnterPayload,
ForceEnterResponse, ForceExitPayload, Health,
Locks, Logs, OpenTradeSchema, PairHistory,
PerformanceEntry, Ping, PlotConfig, Profit,
ResultMsg, ShowConfig, Stats, StatusMsg,
StrategyListResponse, StrategyResponse, SysInfo,
Version, WhitelistResponse)
ForceEnterResponse, ForceExitPayload,
FreqAIModelListResponse, Health, Locks, Logs,
OpenTradeSchema, PairHistory, PerformanceEntry,
Ping, PlotConfig, Profit, ResultMsg, ShowConfig,
Stats, StatusMsg, StrategyListResponse,
StrategyResponse, SysInfo, Version,
WhitelistResponse)
from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional
from freqtrade.rpc.rpc import RPCException
@@ -38,7 +39,8 @@ logger = logging.getLogger(__name__)
# 2.17: Forceentry - leverage, partial force_exit
# 2.20: Add websocket endpoints
# 2.21: Add new_candle messagetype
API_VERSION = 2.21
# 2.22: Add FreqAI to backtesting
API_VERSION = 2.22
# Public API, requires no auth.
router_public = APIRouter()
@@ -279,6 +281,16 @@ def get_strategy(strategy: str, config=Depends(get_config)):
}
@router.get('/freqaimodels', response_model=FreqAIModelListResponse, tags=['freqai'])
def list_freqaimodels(config=Depends(get_config)):
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
strategies = FreqaiModelResolver.search_all_objects(
config, False)
strategies = sorted(strategies, key=lambda x: x['name'])
return {'freqaimodels': [x['name'] for x in strategies]}
@router.get('/available_pairs', response_model=AvailablePairs, tags=['candle data'])
def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None,
candletype: Optional[CandleType] = None, config=Depends(get_config)):

3
freqtrade/rpc/api_server/api_ws.py
View File

@@ -91,9 +91,10 @@ async def _process_consumer_request(
elif type == RPCRequestType.ANALYZED_DF:
# Limit the amount of candles per dataframe to 'limit' or 1500
limit = min(data.get('limit', 1500), 1500) if data else None
pair = data.get('pair', None) if data else None
# For every pair in the generator, send a separate message
for message in rpc._ws_request_analyzed_df(limit):
for message in rpc._ws_request_analyzed_df(limit, pair):
# Format response
response = WSAnalyzedDFMessage(data=message)
await channel.send(response.dict(exclude_none=True))

7
freqtrade/rpc/api_server/ws/channel.py
View File

@@ -27,7 +27,8 @@ class WebSocketChannel:
self,
websocket: WebSocketType,
channel_id: Optional[str] = None,
serializer_cls: Type[WebSocketSerializer] = HybridJSONWebSocketSerializer
serializer_cls: Type[WebSocketSerializer] = HybridJSONWebSocketSerializer,
send_throttle: float = 0.01
):
self.channel_id = channel_id if channel_id else uuid4().hex[:8]
self._websocket = WebSocketProxy(websocket)
@@ -41,6 +42,7 @@ class WebSocketChannel:
self._send_times: Deque[float] = deque([], maxlen=10)
# High limit defaults to 3 to start
self._send_high_limit = 3
self._send_throttle = send_throttle
# The subscribed message types
self._subscriptions: List[str] = []
@@ -106,7 +108,8 @@ class WebSocketChannel:
# Explicitly give control back to event loop as
# websockets.send does not
await asyncio.sleep(0.01)
# Also throttles how fast we send
await asyncio.sleep(self._send_throttle)
async def recv(self):
"""

2
freqtrade/rpc/api_server/ws_schemas.py
View File

@@ -47,7 +47,7 @@ class WSWhitelistRequest(WSRequestSchema):
class WSAnalyzedDFRequest(WSRequestSchema):
type: RPCRequestType = RPCRequestType.ANALYZED_DF
data: Dict[str, Any] = {"limit": 1500}
data: Dict[str, Any] = {"limit": 1500, "pair": None}
# ------------------------------ MESSAGE SCHEMAS ----------------------------

120
freqtrade/rpc/external_message_consumer.py
View File

@@ -8,15 +8,17 @@ import asyncio
import logging
import socket
from threading import Thread
from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypedDict
from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypedDict, Union
import websockets
from pydantic import ValidationError
from freqtrade.constants import FULL_DATAFRAME_THRESHOLD
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import RPCMessageType
from freqtrade.misc import remove_entry_exit_signals
from freqtrade.rpc.api_server.ws import WebSocketChannel
from freqtrade.rpc.api_server.ws.channel import WebSocketChannel, create_channel
from freqtrade.rpc.api_server.ws.message_stream import MessageStream
from freqtrade.rpc.api_server.ws_schemas import (WSAnalyzedDFMessage, WSAnalyzedDFRequest,
WSMessageSchema, WSRequestSchema,
WSSubscribeRequest, WSWhitelistMessage,
@@ -38,6 +40,10 @@ class Producer(TypedDict):
logger = logging.getLogger(__name__)
def schema_to_dict(schema: Union[WSMessageSchema, WSRequestSchema]):
return schema.dict(exclude_none=True)
class ExternalMessageConsumer:
"""
The main controller class for consuming external messages from
@@ -92,6 +98,8 @@ class ExternalMessageConsumer:
RPCMessageType.ANALYZED_DF: self._consume_analyzed_df_message,
}
self._channel_streams: Dict[str, MessageStream] = {}
self.start()
def start(self):
@@ -118,6 +126,8 @@ class ExternalMessageConsumer:
logger.info("Stopping ExternalMessageConsumer")
self._running = False
self._channel_streams = {}
if self._sub_tasks:
# Cancel sub tasks
for task in self._sub_tasks:
@@ -175,7 +185,6 @@ class ExternalMessageConsumer:
:param producer: Dictionary containing producer info
:param lock: An asyncio Lock
"""
channel = None
while self._running:
try:
host, port = producer['host'], producer['port']
@@ -190,19 +199,21 @@ class ExternalMessageConsumer:
max_size=self.message_size_limit,
ping_interval=None
) as ws:
channel = WebSocketChannel(ws, channel_id=name)
async with create_channel(
ws,
channel_id=name,
send_throttle=0.5
) as channel:
logger.info(f"Producer connection success - {channel}")
# Create the message stream for this channel
self._channel_streams[name] = MessageStream()
# Now request the initial data from this Producer
for request in self._initial_requests:
await channel.send(
request.dict(exclude_none=True)
# Run the channel tasks while connected
await channel.run_channel_tasks(
self._receive_messages(channel, producer, lock),
self._send_requests(channel, self._channel_streams[name])
)
# Now receive data, if none is within the time limit, ping
await self._receive_messages(channel, producer, lock)
except (websockets.exceptions.InvalidURI, ValueError) as e:
logger.error(f"{ws_url} is an invalid WebSocket URL - {e}")
break
@@ -229,11 +240,19 @@ class ExternalMessageConsumer:
# An unforseen error has occurred, log and continue
logger.error("Unexpected error has occurred:")
logger.exception(e)
await asyncio.sleep(self.sleep_time)
continue
finally:
if channel:
await channel.close()
async def _send_requests(self, channel: WebSocketChannel, channel_stream: MessageStream):
# Send the initial requests
for init_request in self._initial_requests:
await channel.send(schema_to_dict(init_request))
# Now send any subsequent requests published to
# this channel's stream
async for request, _ in channel_stream:
logger.debug(f"Sending request to channel - {channel} - {request}")
await channel.send(request)
async def _receive_messages(
self,
@@ -270,19 +289,31 @@ class ExternalMessageConsumer:
latency = (await asyncio.wait_for(pong, timeout=self.ping_timeout) * 1000)
logger.info(f"Connection to {channel} still alive, latency: {latency}ms")
continue
except (websockets.exceptions.ConnectionClosed):
# Just eat the error and continue reconnecting
logger.warning(f"Disconnection in {channel} - retrying in {self.sleep_time}s")
await asyncio.sleep(self.sleep_time)
break
except Exception as e:
# Just eat the error and continue reconnecting
logger.warning(f"Ping error {channel} - {e} - retrying in {self.sleep_time}s")
logger.debug(e, exc_info=e)
await asyncio.sleep(self.sleep_time)
raise
break
def send_producer_request(
self,
producer_name: str,
request: Union[WSRequestSchema, Dict[str, Any]]
):
"""
Publish a message to the producer's message stream to be
sent by the channel task.
:param producer_name: The name of the producer to publish the message to
:param request: The request to send to the producer
"""
if isinstance(request, WSRequestSchema):
request = schema_to_dict(request)
if channel_stream := self._channel_streams.get(producer_name):
channel_stream.publish(request)
def handle_producer_message(self, producer: Producer, message: Dict[str, Any]):
"""
@@ -336,16 +367,45 @@ class ExternalMessageConsumer:
pair, timeframe, candle_type = key
if df.empty:
logger.debug(f"Received Empty Dataframe for {key}")
return
# If set, remove the Entry and Exit signals from the Producer
if self._emc_config.get('remove_entry_exit_signals', False):
df = remove_entry_exit_signals(df)
# Add the dataframe to the dataprovider
self._dp._add_external_df(pair, df,
last_analyzed=la,
timeframe=timeframe,
candle_type=candle_type,
producer_name=producer_name)
logger.debug(f"Received {len(df)} candle(s) for {key}")
did_append, n_missing = self._dp._add_external_df(
pair,
df,
last_analyzed=la,
timeframe=timeframe,
candle_type=candle_type,
producer_name=producer_name
)
if not did_append:
# We want an overlap in candles incase some data has changed
n_missing += 1
# Set to None for all candles if we missed a full df's worth of candles
n_missing = n_missing if n_missing < FULL_DATAFRAME_THRESHOLD else 1500
logger.warning(f"Holes in data or no existing df, requesting {n_missing} candles "
f"for {key} from `{producer_name}`")
self.send_producer_request(
producer_name,
WSAnalyzedDFRequest(
data={
"limit": n_missing,
"pair": pair
}
)
)
return
logger.debug(
f"Consumed message from `{producer_name}` of type `RPCMessageType.ANALYZED_DF`")
f"Consumed message from `{producer_name}` "
f"of type `RPCMessageType.ANALYZED_DF` for {key}")

73
freqtrade/rpc/rpc.py
View File

@@ -167,6 +167,7 @@ class RPC:
results = []
for trade in trades:
order: Optional[Order] = None
current_profit_fiat: Optional[float] = None
if trade.open_order_id:
order = trade.select_order_by_order_id(trade.open_order_id)
# calculate profit and send message to user
@@ -176,23 +177,26 @@ class RPC:
trade.pair, side='exit', is_short=trade.is_short, refresh=False)
except (ExchangeError, PricingError):
current_rate = NAN
if len(trade.select_filled_orders(trade.entry_side)) > 0:
current_profit = trade.calc_profit_ratio(
current_rate) if not isnan(current_rate) else NAN
current_profit_abs = trade.calc_profit(
current_rate) if not isnan(current_rate) else NAN
else:
current_profit = current_profit_abs = current_profit_fiat = 0.0
else:
# Closed trade ...
current_rate = trade.close_rate
if len(trade.select_filled_orders(trade.entry_side)) > 0:
current_profit = trade.calc_profit_ratio(
current_rate) if not isnan(current_rate) else NAN
current_profit_abs = trade.calc_profit(
current_rate) if not isnan(current_rate) else NAN
current_profit_fiat: Optional[float] = None
# Calculate fiat profit
if self._fiat_converter:
current_profit_fiat = self._fiat_converter.convert_amount(
current_profit_abs,
self._freqtrade.config['stake_currency'],
self._freqtrade.config['fiat_display_currency']
)
else:
current_profit = current_profit_abs = current_profit_fiat = 0.0
current_profit = trade.close_profit
current_profit_abs = trade.close_profit_abs
# Calculate fiat profit
if not isnan(current_profit_abs) and self._fiat_converter:
current_profit_fiat = self._fiat_converter.convert_amount(
current_profit_abs,
self._freqtrade.config['stake_currency'],
self._freqtrade.config['fiat_display_currency']
)
# Calculate guaranteed profit (in case of trailing stop)
stoploss_entry_dist = trade.calc_profit(trade.stop_loss)
@@ -1058,15 +1062,26 @@ class RPC:
return self._convert_dataframe_to_dict(self._freqtrade.config['strategy'],
pair, timeframe, _data, last_analyzed)
def __rpc_analysed_dataframe_raw(self, pair: str, timeframe: str,
limit: Optional[int]) -> Tuple[DataFrame, datetime]:
""" Get the dataframe and last analyze from the dataprovider """
def __rpc_analysed_dataframe_raw(
self,
pair: str,
timeframe: str,
limit: Optional[int]
) -> Tuple[DataFrame, datetime]:
"""
Get the dataframe and last analyze from the dataprovider
:param pair: The pair to get
:param timeframe: The timeframe of data to get
:param limit: The amount of candles in the dataframe
"""
_data, last_analyzed = self._freqtrade.dataprovider.get_analyzed_dataframe(
pair, timeframe)
_data = _data.copy()
if limit:
_data = _data.iloc[-limit:]
return _data, last_analyzed
def _ws_all_analysed_dataframes(
@@ -1074,7 +1089,16 @@ class RPC:
pairlist: List[str],
limit: Optional[int]
) -> Generator[Dict[str, Any], None, None]:
""" Get the analysed dataframes of each pair in the pairlist """
"""
Get the analysed dataframes of each pair in the pairlist.
If specified, only return the most recent `limit` candles for
each dataframe.
:param pairlist: A list of pairs to get
:param limit: If an integer, limits the size of dataframe
If a list of string date times, only returns those candles
:returns: A generator of dictionaries with the key, dataframe, and last analyzed timestamp
"""
timeframe = self._freqtrade.config['timeframe']
candle_type = self._freqtrade.config.get('candle_type_def', CandleType.SPOT)
@@ -1087,10 +1111,15 @@ class RPC:
"la": last_analyzed
}
def _ws_request_analyzed_df(self, limit: Optional[int]):
def _ws_request_analyzed_df(
self,
limit: Optional[int] = None,
pair: Optional[str] = None
):
""" Historical Analyzed Dataframes for WebSocket """
whitelist = self._freqtrade.active_pair_whitelist
return self._ws_all_analysed_dataframes(whitelist, limit)
pairlist = [pair] if pair else self._freqtrade.active_pair_whitelist
return self._ws_all_analysed_dataframes(pairlist, limit)
def _ws_request_whitelist(self):
""" Whitelist data for WebSocket """

5
freqtrade/templates/FreqaiExampleStrategy.py
View File

@@ -28,7 +28,7 @@ class FreqaiExampleStrategy(IStrategy):
plot_config = {
"main_plot": {},
"subplots": {
"prediction": {"prediction": {"color": "blue"}},
"&-s_close": {"prediction": {"color": "blue"}},
"do_predict": {
"do_predict": {"color": "brown"},
},
@@ -140,7 +140,8 @@ class FreqaiExampleStrategy(IStrategy):
# If user wishes to use multiple targets, they can add more by
# appending more columns with '&'. User should keep in mind that multi targets
# requires a multioutput prediction model such as
# templates/CatboostPredictionMultiModel.py,
# freqai/prediction_models/CatboostRegressorMultiTarget.py,
# freqtrade trade --freqaimodel CatboostRegressorMultiTarget
# df["&-s_range"] = (
# df["close"]

48
freqtrade/templates/strategy_analysis_example.ipynb
View File

@@ -7,14 +7,17 @@
"# Strategy analysis example\n",
"\n",
"Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data.\n",
"The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location."
"The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location.\n",
"Please follow the [documentation](https://www.freqtrade.io/en/stable/data-download/) for more details."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Setup"
"## Setup\n",
"\n",
"### Change Working directory to repository root"
]
},
{
@@ -23,7 +26,38 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from pathlib import Path\n",
"\n",
"# Change directory\n",
"# Modify this cell to insure that the output shows the correct path.\n",
"# Define all paths relative to the project root shown in the cell output\n",
"project_root = \"somedir/freqtrade\"\n",
"i=0\n",
"try:\n",
" os.chdirdir(project_root)\n",
" assert Path('LICENSE').is_file()\n",
"except:\n",
" while i<4 and (not Path('LICENSE').is_file()):\n",
" os.chdir(Path(Path.cwd(), '../'))\n",
" i+=1\n",
" project_root = Path.cwd()\n",
"print(Path.cwd())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Configure Freqtrade environment"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from freqtrade.configuration import Configuration\n",
"\n",
"# Customize these according to your needs.\n",
@@ -31,14 +65,14 @@
"# Initialize empty configuration object\n",
"config = Configuration.from_files([])\n",
"# Optionally (recommended), use existing configuration file\n",
"# config = Configuration.from_files([\"config.json\"])\n",
"# config = Configuration.from_files([\"user_data/config.json\"])\n",
"\n",
"# Define some constants\n",
"config[\"timeframe\"] = \"5m\"\n",
"# Name of the strategy class\n",
"config[\"strategy\"] = \"SampleStrategy\"\n",
"# Location of the data\n",
"data_location = config['datadir']\n",
"data_location = config[\"datadir\"]\n",
"# Pair to analyze - Only use one pair here\n",
"pair = \"BTC/USDT\""
]
@@ -56,12 +90,12 @@
"candles = load_pair_history(datadir=data_location,\n",
" timeframe=config[\"timeframe\"],\n",
" pair=pair,\n",
" data_format = \"hdf5\",\n",
" data_format = \"json\", # Make sure to update this to your data\n",
" candle_type=CandleType.SPOT,\n",
" )\n",
"\n",
"# Confirm success\n",
"print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {data_location}\")\n",
"print(f\"Loaded {len(candles)} rows of data for {pair} from {data_location}\")\n",
"candles.head()"
]
},
@@ -365,7 +399,7 @@
"metadata": {
"file_extension": ".py",
"kernelspec": {
"display_name": "Python 3.9.7 64-bit ('trade_397')",
"display_name": "Python 3.9.7 64-bit",
"language": "python",
"name": "python3"
},

7
freqtrade/wallets.py
View File

@@ -291,12 +291,17 @@ class Wallets:
return self._check_available_stake_amount(stake_amount, available_amount)
def validate_stake_amount(self, pair: str, stake_amount: Optional[float],
min_stake_amount: Optional[float], max_stake_amount: float):
min_stake_amount: Optional[float], max_stake_amount: float,
trade_amount: Optional[float]):
if not stake_amount:
logger.debug(f"Stake amount is {stake_amount}, ignoring possible trade for {pair}.")
return 0
max_stake_amount = min(max_stake_amount, self.get_available_stake_amount())
if trade_amount:
# if in a trade, then the resulting trade size cannot go beyond the max stake
# Otherwise we could no longer exit.
max_stake_amount = min(max_stake_amount, max_stake_amount - trade_amount)
if min_stake_amount is not None and min_stake_amount > max_stake_amount:
if self._log: