Merge remote-tracking branch 'origin/develop' into spice-rack
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
""" Freqtrade bot """
|
||||
__version__ = '2022.9.dev'
|
||||
__version__ = '2022.10.dev'
|
||||
|
||||
if 'dev' in __version__:
|
||||
try:
|
||||
|
@@ -1,6 +1,5 @@
|
||||
# flake8: noqa: F401
|
||||
|
||||
from freqtrade.configuration.check_exchange import check_exchange
|
||||
from freqtrade.configuration.config_setup import setup_utils_configuration
|
||||
from freqtrade.configuration.config_validation import validate_config_consistency
|
||||
from freqtrade.configuration.configuration import Configuration
|
||||
|
@@ -8,7 +8,6 @@ from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from freqtrade import constants
|
||||
from freqtrade.configuration.check_exchange import check_exchange
|
||||
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
|
||||
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
|
||||
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
|
||||
@@ -100,6 +99,9 @@ class Configuration:
|
||||
|
||||
self._process_freqai_options(config)
|
||||
|
||||
# Import check_exchange here to avoid import cycle problems
|
||||
from freqtrade.exchange.check_exchange import check_exchange
|
||||
|
||||
# Check if the exchange set by the user is supported
|
||||
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
|
||||
|
||||
|
@@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
|
||||
'CalmarHyperOptLoss',
|
||||
'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
|
||||
'ProfitDrawDownHyperOptLoss']
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
|
||||
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
|
||||
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
|
||||
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
|
||||
@@ -552,7 +552,7 @@ CONF_SCHEMA = {
|
||||
"weight_factor": {"type": "number", "default": 0},
|
||||
"principal_component_analysis": {"type": "boolean", "default": False},
|
||||
"use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
|
||||
"plot_feature_importance": {"type": "boolean", "default": False},
|
||||
"plot_feature_importances": {"type": "integer", "default": 0},
|
||||
"svm_params": {"type": "object",
|
||||
"properties": {
|
||||
"shuffle": {"type": "boolean", "default": False},
|
||||
@@ -567,6 +567,7 @@ CONF_SCHEMA = {
|
||||
"properties": {
|
||||
"test_size": {"type": "number"},
|
||||
"random_state": {"type": "integer"},
|
||||
"shuffle": {"type": "boolean", "default": False}
|
||||
},
|
||||
},
|
||||
"model_training_parameters": {
|
||||
|
@@ -284,7 +284,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
|
||||
df['enter_tag'] = df['buy_tag']
|
||||
df = df.drop(['buy_tag'], axis=1)
|
||||
if 'orders' not in df.columns:
|
||||
df.loc[:, 'orders'] = None
|
||||
df['orders'] = None
|
||||
|
||||
else:
|
||||
# old format - only with lists.
|
||||
@@ -341,9 +341,9 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame:
|
||||
"""
|
||||
df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS)
|
||||
if len(df) > 0:
|
||||
df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True)
|
||||
df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True)
|
||||
df.loc[:, 'close_rate'] = df['close_rate'].astype('float64')
|
||||
df['close_date'] = pd.to_datetime(df['close_date'], utc=True)
|
||||
df['open_date'] = pd.to_datetime(df['open_date'], utc=True)
|
||||
df['close_rate'] = df['close_rate'].astype('float64')
|
||||
return df
|
||||
|
||||
|
||||
|
@@ -47,8 +47,7 @@ def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *,
|
||||
|
||||
|
||||
def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
|
||||
fill_missing: bool = True,
|
||||
drop_incomplete: bool = True) -> DataFrame:
|
||||
fill_missing: bool, drop_incomplete: bool) -> DataFrame:
|
||||
"""
|
||||
Cleanse a OHLCV dataframe by
|
||||
* Grouping it by date (removes duplicate tics)
|
||||
|
@@ -26,7 +26,7 @@ def load_pair_history(pair: str,
|
||||
datadir: Path, *,
|
||||
timerange: Optional[TimeRange] = None,
|
||||
fill_up_missing: bool = True,
|
||||
drop_incomplete: bool = True,
|
||||
drop_incomplete: bool = False,
|
||||
startup_candles: int = 0,
|
||||
data_format: str = None,
|
||||
data_handler: IDataHandler = None,
|
||||
|
@@ -272,10 +272,10 @@ class IDataHandler(ABC):
|
||||
return res
|
||||
|
||||
def ohlcv_load(self, pair, timeframe: str,
|
||||
candle_type: CandleType,
|
||||
candle_type: CandleType, *,
|
||||
timerange: Optional[TimeRange] = None,
|
||||
fill_missing: bool = True,
|
||||
drop_incomplete: bool = True,
|
||||
drop_incomplete: bool = False,
|
||||
startup_candles: int = 0,
|
||||
warn_no_data: bool = True,
|
||||
) -> DataFrame:
|
||||
|
@@ -12,8 +12,8 @@ from freqtrade.exchange.coinbasepro import Coinbasepro
|
||||
from freqtrade.exchange.exchange import (amount_to_contract_precision, amount_to_contracts,
|
||||
amount_to_precision, available_exchanges, ccxt_exchanges,
|
||||
contracts_to_amount, date_minus_candles,
|
||||
is_exchange_known_ccxt, is_exchange_officially_supported,
|
||||
market_is_active, price_to_precision, timeframe_to_minutes,
|
||||
is_exchange_known_ccxt, market_is_active,
|
||||
price_to_precision, timeframe_to_minutes,
|
||||
timeframe_to_msecs, timeframe_to_next_date,
|
||||
timeframe_to_prev_date, timeframe_to_seconds,
|
||||
validate_exchange, validate_exchanges)
|
||||
|
@@ -68,6 +68,37 @@ class Binance(Exchange):
|
||||
tickers = deep_merge_dicts(bidsasks, tickers, allow_null_overrides=False)
|
||||
return tickers
|
||||
|
||||
@retrier
|
||||
def additional_exchange_init(self) -> None:
|
||||
"""
|
||||
Additional exchange initialization logic.
|
||||
.api will be available at this point.
|
||||
Must be overridden in child methods if required.
|
||||
"""
|
||||
try:
|
||||
if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
|
||||
position_side = self._api.fapiPrivateGetPositionsideDual()
|
||||
self._log_exchange_response('position_side_setting', position_side)
|
||||
assets_margin = self._api.fapiPrivateGetMultiAssetsMargin()
|
||||
self._log_exchange_response('multi_asset_margin', assets_margin)
|
||||
msg = ""
|
||||
if position_side.get('dualSidePosition') is True:
|
||||
msg += (
|
||||
"\nHedge Mode is not supported by freqtrade. "
|
||||
"Please change 'Position Mode' on your binance futures account.")
|
||||
if assets_margin.get('multiAssetsMargin') is True:
|
||||
msg += ("\nMulti-Asset Mode is not supported by freqtrade. "
|
||||
"Please change 'Asset Mode' on your binance futures account.")
|
||||
if msg:
|
||||
raise OperationalException(msg)
|
||||
except ccxt.DDoSProtection as e:
|
||||
raise DDosProtection(e) from e
|
||||
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
|
||||
raise TemporaryError(
|
||||
f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
|
||||
except ccxt.BaseError as e:
|
||||
raise OperationalException(e) from e
|
||||
|
||||
@retrier
|
||||
def _set_leverage(
|
||||
self,
|
||||
|
@@ -4485,6 +4485,120 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"BTCUSDT_221230": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 0.0,
|
||||
"maxNotional": 375000.0,
|
||||
"maintenanceMarginRate": 0.02,
|
||||
"maxLeverage": 25.0,
|
||||
"info": {
|
||||
"bracket": "1",
|
||||
"initialLeverage": "25",
|
||||
"notionalCap": "375000",
|
||||
"notionalFloor": "0",
|
||||
"maintMarginRatio": "0.02",
|
||||
"cum": "0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 2.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 375000.0,
|
||||
"maxNotional": 2000000.0,
|
||||
"maintenanceMarginRate": 0.05,
|
||||
"maxLeverage": 10.0,
|
||||
"info": {
|
||||
"bracket": "2",
|
||||
"initialLeverage": "10",
|
||||
"notionalCap": "2000000",
|
||||
"notionalFloor": "375000",
|
||||
"maintMarginRatio": "0.05",
|
||||
"cum": "11250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 3.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 2000000.0,
|
||||
"maxNotional": 4000000.0,
|
||||
"maintenanceMarginRate": 0.1,
|
||||
"maxLeverage": 5.0,
|
||||
"info": {
|
||||
"bracket": "3",
|
||||
"initialLeverage": "5",
|
||||
"notionalCap": "4000000",
|
||||
"notionalFloor": "2000000",
|
||||
"maintMarginRatio": "0.1",
|
||||
"cum": "111250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 4.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 4000000.0,
|
||||
"maxNotional": 10000000.0,
|
||||
"maintenanceMarginRate": 0.125,
|
||||
"maxLeverage": 4.0,
|
||||
"info": {
|
||||
"bracket": "4",
|
||||
"initialLeverage": "4",
|
||||
"notionalCap": "10000000",
|
||||
"notionalFloor": "4000000",
|
||||
"maintMarginRatio": "0.125",
|
||||
"cum": "211250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 5.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 10000000.0,
|
||||
"maxNotional": 20000000.0,
|
||||
"maintenanceMarginRate": 0.15,
|
||||
"maxLeverage": 3.0,
|
||||
"info": {
|
||||
"bracket": "5",
|
||||
"initialLeverage": "3",
|
||||
"notionalCap": "20000000",
|
||||
"notionalFloor": "10000000",
|
||||
"maintMarginRatio": "0.15",
|
||||
"cum": "461250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 6.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 20000000.0,
|
||||
"maxNotional": 40000000.0,
|
||||
"maintenanceMarginRate": 0.25,
|
||||
"maxLeverage": 2.0,
|
||||
"info": {
|
||||
"bracket": "6",
|
||||
"initialLeverage": "2",
|
||||
"notionalCap": "40000000",
|
||||
"notionalFloor": "20000000",
|
||||
"maintMarginRatio": "0.25",
|
||||
"cum": "2461250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 7.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 40000000.0,
|
||||
"maxNotional": 400000000.0,
|
||||
"maintenanceMarginRate": 0.5,
|
||||
"maxLeverage": 1.0,
|
||||
"info": {
|
||||
"bracket": "7",
|
||||
"initialLeverage": "1",
|
||||
"notionalCap": "400000000",
|
||||
"notionalFloor": "40000000",
|
||||
"maintMarginRatio": "0.5",
|
||||
"cum": "1.246125E7"
|
||||
}
|
||||
}
|
||||
],
|
||||
"BTS/USDT": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
@@ -5759,6 +5873,104 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"CVX/USDT": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 0.0,
|
||||
"maxNotional": 5000.0,
|
||||
"maintenanceMarginRate": 0.01,
|
||||
"maxLeverage": 25.0,
|
||||
"info": {
|
||||
"bracket": "1",
|
||||
"initialLeverage": "25",
|
||||
"notionalCap": "5000",
|
||||
"notionalFloor": "0",
|
||||
"maintMarginRatio": "0.01",
|
||||
"cum": "0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 2.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 5000.0,
|
||||
"maxNotional": 25000.0,
|
||||
"maintenanceMarginRate": 0.025,
|
||||
"maxLeverage": 20.0,
|
||||
"info": {
|
||||
"bracket": "2",
|
||||
"initialLeverage": "20",
|
||||
"notionalCap": "25000",
|
||||
"notionalFloor": "5000",
|
||||
"maintMarginRatio": "0.025",
|
||||
"cum": "75.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 3.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 25000.0,
|
||||
"maxNotional": 100000.0,
|
||||
"maintenanceMarginRate": 0.05,
|
||||
"maxLeverage": 10.0,
|
||||
"info": {
|
||||
"bracket": "3",
|
||||
"initialLeverage": "10",
|
||||
"notionalCap": "100000",
|
||||
"notionalFloor": "25000",
|
||||
"maintMarginRatio": "0.05",
|
||||
"cum": "700.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 4.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 100000.0,
|
||||
"maxNotional": 250000.0,
|
||||
"maintenanceMarginRate": 0.1,
|
||||
"maxLeverage": 5.0,
|
||||
"info": {
|
||||
"bracket": "4",
|
||||
"initialLeverage": "5",
|
||||
"notionalCap": "250000",
|
||||
"notionalFloor": "100000",
|
||||
"maintMarginRatio": "0.1",
|
||||
"cum": "5700.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 5.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 250000.0,
|
||||
"maxNotional": 1000000.0,
|
||||
"maintenanceMarginRate": 0.125,
|
||||
"maxLeverage": 2.0,
|
||||
"info": {
|
||||
"bracket": "5",
|
||||
"initialLeverage": "2",
|
||||
"notionalCap": "1000000",
|
||||
"notionalFloor": "250000",
|
||||
"maintMarginRatio": "0.125",
|
||||
"cum": "11950.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 6.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 1000000.0,
|
||||
"maxNotional": 5000000.0,
|
||||
"maintenanceMarginRate": 0.5,
|
||||
"maxLeverage": 1.0,
|
||||
"info": {
|
||||
"bracket": "6",
|
||||
"initialLeverage": "1",
|
||||
"notionalCap": "5000000",
|
||||
"notionalFloor": "1000000",
|
||||
"maintMarginRatio": "0.5",
|
||||
"cum": "386950.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"DAR/USDT": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
@@ -8105,6 +8317,120 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"ETHUSDT_221230": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 0.0,
|
||||
"maxNotional": 375000.0,
|
||||
"maintenanceMarginRate": 0.02,
|
||||
"maxLeverage": 25.0,
|
||||
"info": {
|
||||
"bracket": "1",
|
||||
"initialLeverage": "25",
|
||||
"notionalCap": "375000",
|
||||
"notionalFloor": "0",
|
||||
"maintMarginRatio": "0.02",
|
||||
"cum": "0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 2.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 375000.0,
|
||||
"maxNotional": 2000000.0,
|
||||
"maintenanceMarginRate": 0.05,
|
||||
"maxLeverage": 10.0,
|
||||
"info": {
|
||||
"bracket": "2",
|
||||
"initialLeverage": "10",
|
||||
"notionalCap": "2000000",
|
||||
"notionalFloor": "375000",
|
||||
"maintMarginRatio": "0.05",
|
||||
"cum": "11250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 3.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 2000000.0,
|
||||
"maxNotional": 4000000.0,
|
||||
"maintenanceMarginRate": 0.1,
|
||||
"maxLeverage": 5.0,
|
||||
"info": {
|
||||
"bracket": "3",
|
||||
"initialLeverage": "5",
|
||||
"notionalCap": "4000000",
|
||||
"notionalFloor": "2000000",
|
||||
"maintMarginRatio": "0.1",
|
||||
"cum": "111250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 4.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 4000000.0,
|
||||
"maxNotional": 10000000.0,
|
||||
"maintenanceMarginRate": 0.125,
|
||||
"maxLeverage": 4.0,
|
||||
"info": {
|
||||
"bracket": "4",
|
||||
"initialLeverage": "4",
|
||||
"notionalCap": "10000000",
|
||||
"notionalFloor": "4000000",
|
||||
"maintMarginRatio": "0.125",
|
||||
"cum": "211250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 5.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 10000000.0,
|
||||
"maxNotional": 20000000.0,
|
||||
"maintenanceMarginRate": 0.15,
|
||||
"maxLeverage": 3.0,
|
||||
"info": {
|
||||
"bracket": "5",
|
||||
"initialLeverage": "3",
|
||||
"notionalCap": "20000000",
|
||||
"notionalFloor": "10000000",
|
||||
"maintMarginRatio": "0.15",
|
||||
"cum": "461250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 6.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 20000000.0,
|
||||
"maxNotional": 40000000.0,
|
||||
"maintenanceMarginRate": 0.25,
|
||||
"maxLeverage": 2.0,
|
||||
"info": {
|
||||
"bracket": "6",
|
||||
"initialLeverage": "2",
|
||||
"notionalCap": "40000000",
|
||||
"notionalFloor": "20000000",
|
||||
"maintMarginRatio": "0.25",
|
||||
"cum": "2461250.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 7.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 40000000.0,
|
||||
"maxNotional": 400000000.0,
|
||||
"maintenanceMarginRate": 0.5,
|
||||
"maxLeverage": 1.0,
|
||||
"info": {
|
||||
"bracket": "7",
|
||||
"initialLeverage": "1",
|
||||
"notionalCap": "400000000",
|
||||
"notionalFloor": "40000000",
|
||||
"maintMarginRatio": "0.5",
|
||||
"cum": "1.246125E7"
|
||||
}
|
||||
}
|
||||
],
|
||||
"FIL/BUSD": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
@@ -10138,10 +10464,10 @@
|
||||
"minNotional": 0.0,
|
||||
"maxNotional": 5000.0,
|
||||
"maintenanceMarginRate": 0.01,
|
||||
"maxLeverage": 50.0,
|
||||
"maxLeverage": 25.0,
|
||||
"info": {
|
||||
"bracket": "1",
|
||||
"initialLeverage": "50",
|
||||
"initialLeverage": "25",
|
||||
"notionalCap": "5000",
|
||||
"notionalFloor": "0",
|
||||
"maintMarginRatio": "0.01",
|
||||
@@ -10216,13 +10542,13 @@
|
||||
"tier": 6.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 1000000.0,
|
||||
"maxNotional": 30000000.0,
|
||||
"maxNotional": 5000000.0,
|
||||
"maintenanceMarginRate": 0.5,
|
||||
"maxLeverage": 1.0,
|
||||
"info": {
|
||||
"bracket": "6",
|
||||
"initialLeverage": "1",
|
||||
"notionalCap": "30000000",
|
||||
"notionalCap": "5000000",
|
||||
"notionalFloor": "1000000",
|
||||
"maintMarginRatio": "0.5",
|
||||
"cum": "386950.0"
|
||||
@@ -11389,6 +11715,104 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"LDO/USDT": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 0.0,
|
||||
"maxNotional": 5000.0,
|
||||
"maintenanceMarginRate": 0.01,
|
||||
"maxLeverage": 25.0,
|
||||
"info": {
|
||||
"bracket": "1",
|
||||
"initialLeverage": "25",
|
||||
"notionalCap": "5000",
|
||||
"notionalFloor": "0",
|
||||
"maintMarginRatio": "0.01",
|
||||
"cum": "0.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 2.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 5000.0,
|
||||
"maxNotional": 25000.0,
|
||||
"maintenanceMarginRate": 0.025,
|
||||
"maxLeverage": 20.0,
|
||||
"info": {
|
||||
"bracket": "2",
|
||||
"initialLeverage": "20",
|
||||
"notionalCap": "25000",
|
||||
"notionalFloor": "5000",
|
||||
"maintMarginRatio": "0.025",
|
||||
"cum": "75.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 3.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 25000.0,
|
||||
"maxNotional": 100000.0,
|
||||
"maintenanceMarginRate": 0.05,
|
||||
"maxLeverage": 10.0,
|
||||
"info": {
|
||||
"bracket": "3",
|
||||
"initialLeverage": "10",
|
||||
"notionalCap": "100000",
|
||||
"notionalFloor": "25000",
|
||||
"maintMarginRatio": "0.05",
|
||||
"cum": "700.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 4.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 100000.0,
|
||||
"maxNotional": 250000.0,
|
||||
"maintenanceMarginRate": 0.1,
|
||||
"maxLeverage": 5.0,
|
||||
"info": {
|
||||
"bracket": "4",
|
||||
"initialLeverage": "5",
|
||||
"notionalCap": "250000",
|
||||
"notionalFloor": "100000",
|
||||
"maintMarginRatio": "0.1",
|
||||
"cum": "5700.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 5.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 250000.0,
|
||||
"maxNotional": 1000000.0,
|
||||
"maintenanceMarginRate": 0.125,
|
||||
"maxLeverage": 2.0,
|
||||
"info": {
|
||||
"bracket": "5",
|
||||
"initialLeverage": "2",
|
||||
"notionalCap": "1000000",
|
||||
"notionalFloor": "250000",
|
||||
"maintMarginRatio": "0.125",
|
||||
"cum": "11950.0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tier": 6.0,
|
||||
"currency": "USDT",
|
||||
"minNotional": 1000000.0,
|
||||
"maxNotional": 5000000.0,
|
||||
"maintenanceMarginRate": 0.5,
|
||||
"maxLeverage": 1.0,
|
||||
"info": {
|
||||
"bracket": "6",
|
||||
"initialLeverage": "1",
|
||||
"notionalCap": "5000000",
|
||||
"notionalFloor": "1000000",
|
||||
"maintMarginRatio": "0.5",
|
||||
"cum": "386950.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"LEVER/BUSD": [
|
||||
{
|
||||
"tier": 1.0,
|
||||
|
@@ -3,8 +3,8 @@ import logging
|
||||
from freqtrade.constants import Config
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
|
||||
is_exchange_officially_supported, validate_exchange)
|
||||
from freqtrade.exchange import available_exchanges, is_exchange_known_ccxt, validate_exchange
|
||||
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS, SUPPORTED_EXCHANGES
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -52,7 +52,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
|
||||
else:
|
||||
logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}')
|
||||
|
||||
if is_exchange_officially_supported(exchange):
|
||||
if MAP_EXCHANGE_CHILDCLASS.get(exchange, exchange) in SUPPORTED_EXCHANGES:
|
||||
logger.info(f'Exchange "{exchange}" is officially supported '
|
||||
f'by the Freqtrade development team.')
|
||||
else:
|
@@ -18,20 +18,19 @@ import ccxt.async_support as ccxt_async
|
||||
from cachetools import TTLCache
|
||||
from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision
|
||||
from dateutil import parser
|
||||
from pandas import DataFrame
|
||||
from pandas import DataFrame, concat
|
||||
|
||||
from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
|
||||
Config, EntryExit, ListPairsWithTimeframes, MakerTaker,
|
||||
PairWithTimeframe)
|
||||
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
|
||||
from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
|
||||
from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode
|
||||
from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
|
||||
InvalidOrderException, OperationalException, PricingError,
|
||||
RetryableOrderError, TemporaryError)
|
||||
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
|
||||
EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
|
||||
SUPPORTED_EXCHANGES, remove_credentials, retrier,
|
||||
retrier_async)
|
||||
remove_credentials, retrier, retrier_async)
|
||||
from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
|
||||
safe_value_fallback2)
|
||||
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
|
||||
@@ -185,8 +184,9 @@ class Exchange:
|
||||
# Initial markets load
|
||||
self._load_markets()
|
||||
self.validate_config(config)
|
||||
self._startup_candle_count: int = config.get('startup_candle_count', 0)
|
||||
self.required_candle_call_count = self.validate_required_startup_candles(
|
||||
config.get('startup_candle_count', 0), config.get('timeframe', ''))
|
||||
self._startup_candle_count, config.get('timeframe', ''))
|
||||
|
||||
# Converts the interval provided in minutes in config to seconds
|
||||
self.markets_refresh_interval: int = exchange_config.get(
|
||||
@@ -1292,7 +1292,14 @@ class Exchange:
|
||||
order = self.fetch_order(order_id, pair)
|
||||
except InvalidOrderException:
|
||||
logger.warning(f"Could not fetch cancelled order {order_id}.")
|
||||
order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
|
||||
order = {
|
||||
'id': order_id,
|
||||
'status': 'canceled',
|
||||
'amount': amount,
|
||||
'filled': 0.0,
|
||||
'fee': {},
|
||||
'info': {}
|
||||
}
|
||||
|
||||
return order
|
||||
|
||||
@@ -1844,10 +1851,22 @@ class Exchange:
|
||||
return pair, timeframe, candle_type, data
|
||||
|
||||
def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
|
||||
since_ms: Optional[int]) -> Coroutine:
|
||||
since_ms: Optional[int], cache: bool) -> Coroutine:
|
||||
not_all_data = self.required_candle_call_count > 1
|
||||
if cache and (pair, timeframe, candle_type) in self._klines:
|
||||
candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
|
||||
min_date = date_minus_candles(timeframe, candle_limit - 5).timestamp()
|
||||
# Check if 1 call can get us updated candles without hole in the data.
|
||||
if min_date < self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0):
|
||||
# Cache can be used - do one-off call.
|
||||
not_all_data = False
|
||||
else:
|
||||
# Time jump detected, evict cache
|
||||
logger.info(
|
||||
f"Time jump detected. Evicting cache for {pair}, {timeframe}, {candle_type}")
|
||||
del self._klines[(pair, timeframe, candle_type)]
|
||||
|
||||
if (not since_ms
|
||||
and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)):
|
||||
if (not since_ms and (self._ft_has["ohlcv_require_since"] or not_all_data)):
|
||||
# Multiple calls for one pair - to get more history
|
||||
one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(
|
||||
timeframe, candle_type, since_ms)
|
||||
@@ -1863,6 +1882,59 @@ class Exchange:
|
||||
return self._async_get_candle_history(
|
||||
pair, timeframe, since_ms=since_ms, candle_type=candle_type)
|
||||
|
||||
def _build_ohlcv_dl_jobs(
|
||||
self, pair_list: ListPairsWithTimeframes, since_ms: Optional[int],
|
||||
cache: bool) -> Tuple[List[Coroutine], List[Tuple[str, str, CandleType]]]:
|
||||
"""
|
||||
Build Coroutines to execute as part of refresh_latest_ohlcv
|
||||
"""
|
||||
input_coroutines = []
|
||||
cached_pairs = []
|
||||
for pair, timeframe, candle_type in set(pair_list):
|
||||
if (timeframe not in self.timeframes
|
||||
and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
|
||||
logger.warning(
|
||||
f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
|
||||
f"not available on {self.name}. Available timeframes are "
|
||||
f"{', '.join(self.timeframes)}.")
|
||||
continue
|
||||
|
||||
if ((pair, timeframe, candle_type) not in self._klines or not cache
|
||||
or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
|
||||
|
||||
input_coroutines.append(
|
||||
self._build_coroutine(pair, timeframe, candle_type, since_ms, cache))
|
||||
|
||||
else:
|
||||
logger.debug(
|
||||
f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
|
||||
)
|
||||
cached_pairs.append((pair, timeframe, candle_type))
|
||||
|
||||
return input_coroutines, cached_pairs
|
||||
|
||||
def _process_ohlcv_df(self, pair: str, timeframe: str, c_type: CandleType, ticks: List[List],
|
||||
cache: bool, drop_incomplete: bool) -> DataFrame:
|
||||
# keeping last candle time as last refreshed time of the pair
|
||||
if ticks and cache:
|
||||
self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
|
||||
# keeping parsed dataframe in cache
|
||||
ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
|
||||
drop_incomplete=drop_incomplete)
|
||||
if cache:
|
||||
if (pair, timeframe, c_type) in self._klines:
|
||||
old = self._klines[(pair, timeframe, c_type)]
|
||||
# Reassign so we return the updated, combined df
|
||||
ohlcv_df = clean_ohlcv_dataframe(concat([old, ohlcv_df], axis=0), timeframe, pair,
|
||||
fill_missing=True, drop_incomplete=False)
|
||||
candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'])
|
||||
# Age out old candles
|
||||
ohlcv_df = ohlcv_df.tail(candle_limit + self._startup_candle_count)
|
||||
self._klines[(pair, timeframe, c_type)] = ohlcv_df
|
||||
else:
|
||||
self._klines[(pair, timeframe, c_type)] = ohlcv_df
|
||||
return ohlcv_df
|
||||
|
||||
def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
|
||||
since_ms: Optional[int] = None, cache: bool = True,
|
||||
drop_incomplete: Optional[bool] = None
|
||||
@@ -1880,27 +1952,9 @@ class Exchange:
|
||||
"""
|
||||
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
|
||||
drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete
|
||||
input_coroutines = []
|
||||
cached_pairs = []
|
||||
# Gather coroutines to run
|
||||
for pair, timeframe, candle_type in set(pair_list):
|
||||
if (timeframe not in self.timeframes
|
||||
and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
|
||||
logger.warning(
|
||||
f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
|
||||
f"not available on {self.name}. Available timeframes are "
|
||||
f"{', '.join(self.timeframes)}.")
|
||||
continue
|
||||
if ((pair, timeframe, candle_type) not in self._klines or not cache
|
||||
or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
|
||||
input_coroutines.append(self._build_coroutine(
|
||||
pair, timeframe, candle_type=candle_type, since_ms=since_ms))
|
||||
|
||||
else:
|
||||
logger.debug(
|
||||
f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
|
||||
)
|
||||
cached_pairs.append((pair, timeframe, candle_type))
|
||||
# Gather coroutines to run
|
||||
input_coroutines, cached_pairs = self._build_ohlcv_dl_jobs(pair_list, since_ms, cache)
|
||||
|
||||
results_df = {}
|
||||
# Chunk requests into batches of 100 to avoid overwelming ccxt Throttling
|
||||
@@ -1917,16 +1971,11 @@ class Exchange:
|
||||
continue
|
||||
# Deconstruct tuple (has 4 elements)
|
||||
pair, timeframe, c_type, ticks = res
|
||||
# keeping last candle time as last refreshed time of the pair
|
||||
if ticks:
|
||||
self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
|
||||
# keeping parsed dataframe in cache
|
||||
ohlcv_df = ohlcv_to_dataframe(
|
||||
ticks, timeframe, pair=pair, fill_missing=True,
|
||||
drop_incomplete=drop_incomplete)
|
||||
ohlcv_df = self._process_ohlcv_df(
|
||||
pair, timeframe, c_type, ticks, cache, drop_incomplete)
|
||||
|
||||
results_df[(pair, timeframe, c_type)] = ohlcv_df
|
||||
if cache:
|
||||
self._klines[(pair, timeframe, c_type)] = ohlcv_df
|
||||
|
||||
# Return cached klines
|
||||
for pair, timeframe, c_type in cached_pairs:
|
||||
results_df[(pair, timeframe, c_type)] = self.klines(
|
||||
@@ -1941,10 +1990,8 @@ class Exchange:
|
||||
interval_in_sec = timeframe_to_seconds(timeframe)
|
||||
|
||||
return not (
|
||||
(self._pairs_last_refresh_time.get(
|
||||
(pair, timeframe, candle_type),
|
||||
0
|
||||
) + interval_in_sec) >= arrow.utcnow().int_timestamp
|
||||
(self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0)
|
||||
+ interval_in_sec) >= arrow.utcnow().int_timestamp
|
||||
)
|
||||
|
||||
@retrier_async
|
||||
@@ -2754,10 +2801,6 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non
|
||||
return exchange_name in ccxt_exchanges(ccxt_module)
|
||||
|
||||
|
||||
def is_exchange_officially_supported(exchange_name: str) -> bool:
|
||||
return exchange_name in SUPPORTED_EXCHANGES
|
||||
|
||||
|
||||
def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
|
||||
"""
|
||||
Return the list of all exchanges known to ccxt
|
||||
|
@@ -78,7 +78,8 @@ class Okx(Exchange):
|
||||
raise DDosProtection(e) from e
|
||||
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
|
||||
raise TemporaryError(
|
||||
f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
|
||||
f'Error in additional_exchange_init due to {e.__class__.__name__}. Message: {e}'
|
||||
) from e
|
||||
except ccxt.BaseError as e:
|
||||
raise OperationalException(e) from e
|
||||
|
||||
|
@@ -92,7 +92,7 @@ class BaseClassifierModel(IFreqaiModel):
|
||||
filtered_df = dk.normalize_data_from_metadata(filtered_df)
|
||||
dk.data_dictionary["prediction_features"] = filtered_df
|
||||
|
||||
self.data_cleaning_predict(dk, filtered_df)
|
||||
self.data_cleaning_predict(dk)
|
||||
|
||||
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
|
||||
pred_df = DataFrame(predictions, columns=dk.label_list)
|
||||
|
@@ -92,7 +92,7 @@ class BaseRegressionModel(IFreqaiModel):
|
||||
dk.data_dictionary["prediction_features"] = filtered_df
|
||||
|
||||
# optional additional data cleaning/analysis
|
||||
self.data_cleaning_predict(dk, filtered_df)
|
||||
self.data_cleaning_predict(dk)
|
||||
|
||||
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
|
||||
pred_df = DataFrame(predictions, columns=dk.label_list)
|
||||
|
@@ -257,7 +257,7 @@ class FreqaiDataDrawer:
|
||||
|
||||
def append_model_predictions(self, pair: str, predictions: DataFrame,
|
||||
do_preds: NDArray[np.int_],
|
||||
dk: FreqaiDataKitchen, len_df: int) -> None:
|
||||
dk: FreqaiDataKitchen, strat_df: DataFrame) -> None:
|
||||
"""
|
||||
Append model predictions to historic predictions dataframe, then set the
|
||||
strategy return dataframe to the tail of the historic predictions. The length of
|
||||
@@ -266,6 +266,7 @@ class FreqaiDataDrawer:
|
||||
historic predictions.
|
||||
"""
|
||||
|
||||
len_df = len(strat_df)
|
||||
index = self.historic_predictions[pair].index[-1:]
|
||||
columns = self.historic_predictions[pair].columns
|
||||
|
||||
@@ -293,6 +294,15 @@ class FreqaiDataDrawer:
|
||||
for return_str in rets:
|
||||
df[return_str].iloc[-1] = rets[return_str]
|
||||
|
||||
# this logic carries users between version without needing to
|
||||
# change their identifier
|
||||
if 'close_price' not in df.columns:
|
||||
df['close_price'] = np.nan
|
||||
df['date_pred'] = np.nan
|
||||
|
||||
df['close_price'].iloc[-1] = strat_df['close'].iloc[-1]
|
||||
df['date_pred'].iloc[-1] = strat_df['date'].iloc[-1]
|
||||
|
||||
self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
|
||||
|
||||
def attach_return_values_to_return_dataframe(
|
||||
@@ -313,6 +323,7 @@ class FreqaiDataDrawer:
|
||||
"""
|
||||
|
||||
dk.find_features(dataframe)
|
||||
dk.find_labels(dataframe)
|
||||
|
||||
full_labels = dk.label_list + dk.unique_class_list
|
||||
|
||||
@@ -376,7 +387,27 @@ class FreqaiDataDrawer:
|
||||
if self.config.get("freqai", {}).get("purge_old_models", False):
|
||||
self.purge_old_models()
|
||||
|
||||
# Functions pulled back from FreqaiDataKitchen because they relied on DataDrawer
|
||||
def save_metadata(self, dk: FreqaiDataKitchen) -> None:
|
||||
"""
|
||||
Saves only metadata for backtesting studies if user prefers
|
||||
not to save model data. This saves tremendous amounts of space
|
||||
for users generating huge studies.
|
||||
This is only active when `save_backtest_models`: false (not default)
|
||||
"""
|
||||
if not dk.data_path.is_dir():
|
||||
dk.data_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
save_path = Path(dk.data_path)
|
||||
|
||||
dk.data["data_path"] = str(dk.data_path)
|
||||
dk.data["model_filename"] = str(dk.model_filename)
|
||||
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
|
||||
dk.data["label_list"] = dk.label_list
|
||||
|
||||
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
|
||||
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
|
||||
|
||||
return
|
||||
|
||||
def save_data(self, model: Any, coin: str, dk: FreqaiDataKitchen) -> None:
|
||||
"""
|
||||
@@ -402,7 +433,7 @@ class FreqaiDataDrawer:
|
||||
|
||||
dk.data["data_path"] = str(dk.data_path)
|
||||
dk.data["model_filename"] = str(dk.model_filename)
|
||||
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
|
||||
dk.data["training_features_list"] = dk.training_features_list
|
||||
dk.data["label_list"] = dk.label_list
|
||||
# store the metadata
|
||||
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
|
||||
@@ -586,7 +617,8 @@ class FreqaiDataDrawer:
|
||||
"include_corr_pairlist", []
|
||||
)
|
||||
for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
|
||||
base_dataframes[tf] = dk.slice_dataframe(timerange, historic_data[pair][tf])
|
||||
base_dataframes[tf] = dk.slice_dataframe(
|
||||
timerange, historic_data[pair][tf]).reset_index(drop=True)
|
||||
if pairs:
|
||||
for p in pairs:
|
||||
if pair in p:
|
||||
@@ -595,7 +627,7 @@ class FreqaiDataDrawer:
|
||||
corr_dataframes[p] = {}
|
||||
corr_dataframes[p][tf] = dk.slice_dataframe(
|
||||
timerange, historic_data[p][tf]
|
||||
)
|
||||
).reset_index(drop=True)
|
||||
|
||||
return corr_dataframes, base_dataframes
|
||||
|
||||
|
@@ -135,20 +135,15 @@ class FreqaiDataKitchen:
|
||||
"""
|
||||
feat_dict = self.freqai_config["feature_parameters"]
|
||||
|
||||
if 'shuffle' not in self.freqai_config['data_split_parameters']:
|
||||
self.freqai_config["data_split_parameters"].update({'shuffle': False})
|
||||
|
||||
weights: npt.ArrayLike
|
||||
if feat_dict.get("weight_factor", 0) > 0:
|
||||
weights = self.set_weights_higher_recent(len(filtered_dataframe))
|
||||
else:
|
||||
weights = np.ones(len(filtered_dataframe))
|
||||
|
||||
if feat_dict.get("stratify_training_data", 0) > 0:
|
||||
stratification = np.zeros(len(filtered_dataframe))
|
||||
for i in range(1, len(stratification)):
|
||||
if i % feat_dict.get("stratify_training_data", 0) == 0:
|
||||
stratification[i] = 1
|
||||
else:
|
||||
stratification = None
|
||||
|
||||
if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0:
|
||||
(
|
||||
train_features,
|
||||
@@ -161,7 +156,6 @@ class FreqaiDataKitchen:
|
||||
filtered_dataframe[: filtered_dataframe.shape[0]],
|
||||
labels,
|
||||
weights,
|
||||
stratify=stratification,
|
||||
**self.config["freqai"]["data_split_parameters"],
|
||||
)
|
||||
else:
|
||||
@@ -211,7 +205,7 @@ class FreqaiDataKitchen:
|
||||
filtered_df = unfiltered_df.filter(training_feature_list, axis=1)
|
||||
filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
|
||||
|
||||
drop_index = pd.isnull(filtered_df).any(1) # get the rows that have NaNs,
|
||||
drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs,
|
||||
drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement.
|
||||
if (training_filter):
|
||||
const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index)
|
||||
@@ -222,7 +216,7 @@ class FreqaiDataKitchen:
|
||||
# about removing any row with NaNs
|
||||
# if labels has multiple columns (user wants to train multiple modelEs), we detect here
|
||||
labels = unfiltered_df.filter(label_list, axis=1)
|
||||
drop_index_labels = pd.isnull(labels).any(1)
|
||||
drop_index_labels = pd.isnull(labels).any(axis=1)
|
||||
drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0)
|
||||
dates = unfiltered_df['date']
|
||||
filtered_df = filtered_df[
|
||||
@@ -250,7 +244,7 @@ class FreqaiDataKitchen:
|
||||
else:
|
||||
# we are backtesting so we need to preserve row number to send back to strategy,
|
||||
# so now we use do_predict to avoid any prediction based on a NaN
|
||||
drop_index = pd.isnull(filtered_df).any(1)
|
||||
drop_index = pd.isnull(filtered_df).any(axis=1)
|
||||
self.data["filter_drop_index_prediction"] = drop_index
|
||||
filtered_df.fillna(0, inplace=True)
|
||||
# replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction
|
||||
@@ -809,7 +803,7 @@ class FreqaiDataKitchen:
|
||||
:, :no_prev_pts
|
||||
]
|
||||
distances = distances.replace([np.inf, -np.inf], np.nan)
|
||||
drop_index = pd.isnull(distances).any(1)
|
||||
drop_index = pd.isnull(distances).any(axis=1)
|
||||
distances = distances[drop_index == 0]
|
||||
|
||||
inliers = pd.DataFrame(index=distances.index)
|
||||
@@ -832,7 +826,7 @@ class FreqaiDataKitchen:
|
||||
|
||||
inlier_metric = pd.DataFrame(
|
||||
data=inliers.sum(axis=1) / no_prev_pts,
|
||||
columns=['inlier_metric'],
|
||||
columns=['%-inlier_metric'],
|
||||
index=compute_df.index
|
||||
)
|
||||
|
||||
@@ -882,11 +876,15 @@ class FreqaiDataKitchen:
|
||||
"""
|
||||
column_names = dataframe.columns
|
||||
features = [c for c in column_names if "%" in c]
|
||||
labels = [c for c in column_names if "&" in c]
|
||||
|
||||
if not features:
|
||||
raise OperationalException("Could not find any features!")
|
||||
|
||||
self.training_features_list = features
|
||||
|
||||
def find_labels(self, dataframe: DataFrame) -> None:
|
||||
column_names = dataframe.columns
|
||||
labels = [c for c in column_names if "&" in c]
|
||||
self.label_list = labels
|
||||
|
||||
def check_if_pred_in_training_spaces(self) -> None:
|
||||
@@ -1207,7 +1205,8 @@ class FreqaiDataKitchen:
|
||||
|
||||
def get_unique_classes_from_labels(self, dataframe: DataFrame) -> None:
|
||||
|
||||
self.find_features(dataframe)
|
||||
# self.find_features(dataframe)
|
||||
self.find_labels(dataframe)
|
||||
|
||||
for key in self.label_list:
|
||||
if dataframe[key].dtype == object:
|
||||
|
@@ -92,6 +92,7 @@ class IFreqaiModel(ABC):
|
||||
self.begin_time_train: float = 0
|
||||
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
|
||||
self.continual_learning = self.freqai_info.get('continual_learning', False)
|
||||
self.plot_features = self.ft_params.get("plot_feature_importances", 0)
|
||||
self.spice_rack_open: bool = False
|
||||
self._threads: List[threading.Thread] = []
|
||||
self._stop_event = threading.Event()
|
||||
@@ -210,7 +211,8 @@ class IFreqaiModel(ABC):
|
||||
new_trained_timerange, pair, strategy, dk, data_load_timerange
|
||||
)
|
||||
except Exception as msg:
|
||||
logger.warning(f'Training {pair} raised exception {msg}, skipping.')
|
||||
logger.warning(f"Training {pair} raised exception {msg.__class__.__name__}. "
|
||||
f"Message: {msg}, skipping.")
|
||||
|
||||
self.train_timer('stop')
|
||||
|
||||
@@ -274,26 +276,28 @@ class IFreqaiModel(ABC):
|
||||
|
||||
if dk.check_if_backtest_prediction_exists():
|
||||
self.dd.load_metadata(dk)
|
||||
self.check_if_feature_list_matches_strategy(dataframe_train, dk)
|
||||
dk.find_features(dataframe_train)
|
||||
self.check_if_feature_list_matches_strategy(dk)
|
||||
append_df = dk.get_backtesting_prediction()
|
||||
dk.append_predictions(append_df)
|
||||
else:
|
||||
if not self.model_exists(
|
||||
pair, dk, trained_timestamp=trained_timestamp_int
|
||||
):
|
||||
if not self.model_exists(dk):
|
||||
dk.find_features(dataframe_train)
|
||||
dk.find_labels(dataframe_train)
|
||||
self.model = self.train(dataframe_train, pair, dk)
|
||||
self.dd.pair_dict[pair]["trained_timestamp"] = int(
|
||||
trained_timestamp.stopts)
|
||||
|
||||
if self.plot_features:
|
||||
plot_feature_importance(self.model, pair, dk, self.plot_features)
|
||||
if self.save_backtest_models:
|
||||
logger.info('Saving backtest model to disk.')
|
||||
self.dd.save_data(self.model, pair, dk)
|
||||
else:
|
||||
logger.info('Saving metadata to disk.')
|
||||
self.dd.save_metadata(dk)
|
||||
else:
|
||||
self.model = self.dd.load_data(pair, dk)
|
||||
|
||||
self.check_if_feature_list_matches_strategy(dataframe_train, dk)
|
||||
|
||||
pred_df, do_preds = self.predict(dataframe_backtest, dk)
|
||||
append_df = dk.get_predictions_to_append(pred_df, do_preds)
|
||||
dk.append_predictions(append_df)
|
||||
@@ -372,8 +376,7 @@ class IFreqaiModel(ABC):
|
||||
self.dd.return_null_values_to_strategy(dataframe, dk)
|
||||
return dk
|
||||
|
||||
# ensure user is feeding the correct indicators to the model
|
||||
self.check_if_feature_list_matches_strategy(dataframe, dk)
|
||||
dk.find_labels(dataframe)
|
||||
|
||||
self.build_strategy_return_arrays(dataframe, dk, metadata["pair"], trained_timestamp)
|
||||
|
||||
@@ -391,7 +394,7 @@ class IFreqaiModel(ABC):
|
||||
# allows FreqUI to show full return values.
|
||||
pred_df, do_preds = self.predict(dataframe, dk)
|
||||
if pair not in self.dd.historic_predictions:
|
||||
self.set_initial_historic_predictions(pred_df, dk, pair)
|
||||
self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
|
||||
self.dd.set_initial_return_values(pair, pred_df)
|
||||
|
||||
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
|
||||
@@ -412,13 +415,13 @@ class IFreqaiModel(ABC):
|
||||
|
||||
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
|
||||
self.fit_live_predictions(dk, pair)
|
||||
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, len(dataframe))
|
||||
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, dataframe)
|
||||
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
|
||||
|
||||
return
|
||||
|
||||
def check_if_feature_list_matches_strategy(
|
||||
self, dataframe: DataFrame, dk: FreqaiDataKitchen
|
||||
self, dk: FreqaiDataKitchen
|
||||
) -> None:
|
||||
"""
|
||||
Ensure user is passing the proper feature set if they are reusing an `identifier` pointing
|
||||
@@ -427,11 +430,12 @@ class IFreqaiModel(ABC):
|
||||
:param dk: FreqaiDataKitchen = non-persistent data container/analyzer for
|
||||
current coin/bot loop
|
||||
"""
|
||||
dk.find_features(dataframe)
|
||||
|
||||
if "training_features_list_raw" in dk.data:
|
||||
feature_list = dk.data["training_features_list_raw"]
|
||||
else:
|
||||
feature_list = dk.data['training_features_list']
|
||||
|
||||
if dk.training_features_list != feature_list:
|
||||
raise OperationalException(
|
||||
"Trying to access pretrained model with `identifier` "
|
||||
@@ -479,20 +483,23 @@ class IFreqaiModel(ABC):
|
||||
if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0):
|
||||
dk.add_noise_to_training_features()
|
||||
|
||||
def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None:
|
||||
def data_cleaning_predict(self, dk: FreqaiDataKitchen) -> None:
|
||||
"""
|
||||
Base data cleaning method for predict.
|
||||
Functions here are complementary to the functions of data_cleaning_train.
|
||||
"""
|
||||
ft_params = self.freqai_info["feature_parameters"]
|
||||
|
||||
# ensure user is feeding the correct indicators to the model
|
||||
self.check_if_feature_list_matches_strategy(dk)
|
||||
|
||||
if ft_params.get('inlier_metric_window', 0):
|
||||
dk.compute_inlier_metric(set_='predict')
|
||||
|
||||
if ft_params.get(
|
||||
"principal_component_analysis", False
|
||||
):
|
||||
dk.pca_transform(self.dk.data_dictionary['prediction_features'])
|
||||
dk.pca_transform(dk.data_dictionary['prediction_features'])
|
||||
|
||||
if ft_params.get("use_SVM_to_remove_outliers", False):
|
||||
dk.use_SVM_to_remove_outliers(predict=True)
|
||||
@@ -503,14 +510,7 @@ class IFreqaiModel(ABC):
|
||||
if ft_params.get("use_DBSCAN_to_remove_outliers", False):
|
||||
dk.use_DBSCAN_to_remove_outliers(predict=True)
|
||||
|
||||
def model_exists(
|
||||
self,
|
||||
pair: str,
|
||||
dk: FreqaiDataKitchen,
|
||||
trained_timestamp: int = None,
|
||||
model_filename: str = "",
|
||||
scanning: bool = False,
|
||||
) -> bool:
|
||||
def model_exists(self, dk: FreqaiDataKitchen) -> bool:
|
||||
"""
|
||||
Given a pair and path, check if a model already exists
|
||||
:param pair: pair e.g. BTC/USD
|
||||
@@ -518,11 +518,11 @@ class IFreqaiModel(ABC):
|
||||
:return:
|
||||
:boolean: whether the model file exists or not.
|
||||
"""
|
||||
path_to_modelfile = Path(dk.data_path / f"{model_filename}_model.joblib")
|
||||
path_to_modelfile = Path(dk.data_path / f"{dk.model_filename}_model.joblib")
|
||||
file_exists = path_to_modelfile.is_file()
|
||||
if file_exists and not scanning:
|
||||
if file_exists:
|
||||
logger.info("Found model at %s", dk.data_path / dk.model_filename)
|
||||
elif not scanning:
|
||||
else:
|
||||
logger.info("Could not find model at %s", dk.data_path / dk.model_filename)
|
||||
return file_exists
|
||||
|
||||
@@ -569,6 +569,7 @@ class IFreqaiModel(ABC):
|
||||
|
||||
# find the features indicated by strategy and store in datakitchen
|
||||
dk.find_features(unfiltered_dataframe)
|
||||
dk.find_labels(unfiltered_dataframe)
|
||||
|
||||
model = self.train(unfiltered_dataframe, pair, dk)
|
||||
|
||||
@@ -576,14 +577,14 @@ class IFreqaiModel(ABC):
|
||||
dk.set_new_model_names(pair, new_trained_timerange)
|
||||
self.dd.save_data(model, pair, dk)
|
||||
|
||||
if self.freqai_info["feature_parameters"].get("plot_feature_importance", False):
|
||||
plot_feature_importance(model, pair, dk)
|
||||
if self.plot_features:
|
||||
plot_feature_importance(model, pair, dk, self.plot_features)
|
||||
|
||||
if self.freqai_info.get("purge_old_models", False):
|
||||
self.dd.purge_old_models()
|
||||
|
||||
def set_initial_historic_predictions(
|
||||
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str
|
||||
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame
|
||||
) -> None:
|
||||
"""
|
||||
This function is called only if the datadrawer failed to load an
|
||||
@@ -626,6 +627,9 @@ class IFreqaiModel(ABC):
|
||||
for return_str in dk.data['extra_returns_per_train']:
|
||||
hist_preds_df[return_str] = 0
|
||||
|
||||
hist_preds_df['close_price'] = strat_df['close']
|
||||
hist_preds_df['date_pred'] = strat_df['date']
|
||||
|
||||
# # for keras type models, the conv_window needs to be prepended so
|
||||
# # viewing is correct in frequi
|
||||
if self.freqai_info.get('keras', False) or self.ft_params.get('inlier_metric_window', 0):
|
||||
|
@@ -306,7 +306,7 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
|
||||
|
||||
# Data preparation
|
||||
fi_df = pd.DataFrame({
|
||||
"feature_names": np.array(dk.training_features_list),
|
||||
"feature_names": np.array(dk.data_dictionary['train_features'].columns),
|
||||
"feature_importance": np.array(feature_importance)
|
||||
})
|
||||
fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1]
|
||||
|
@@ -82,7 +82,10 @@ class FreqtradeBot(LoggingMixin):
|
||||
# Keep this at the end of this initialization method.
|
||||
self.rpc: RPCManager = RPCManager(self)
|
||||
|
||||
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc)
|
||||
self.dataprovider = DataProvider(self.config, self.exchange, rpc=self.rpc)
|
||||
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
|
||||
|
||||
self.dataprovider.add_pairlisthandler(self.pairlists)
|
||||
|
||||
# Attach Dataprovider to strategy instance
|
||||
self.strategy.dp = self.dataprovider
|
||||
@@ -597,7 +600,7 @@ class FreqtradeBot(LoggingMixin):
|
||||
# We should decrease our position
|
||||
amount = self.exchange.amount_to_contract_precision(
|
||||
trade.pair,
|
||||
abs(float(FtPrecise(stake_amount) / FtPrecise(current_exit_rate))))
|
||||
abs(float(FtPrecise(stake_amount * trade.leverage) / FtPrecise(current_exit_rate))))
|
||||
if amount > trade.amount:
|
||||
# This is currently ineffective as remaining would become < min tradable
|
||||
# Fixing this would require checking for 0.0 there -
|
||||
@@ -1308,7 +1311,7 @@ class FreqtradeBot(LoggingMixin):
|
||||
# place new order only if new price is supplied
|
||||
self.execute_entry(
|
||||
pair=trade.pair,
|
||||
stake_amount=(order_obj.remaining * order_obj.price),
|
||||
stake_amount=(order_obj.remaining * order_obj.price / trade.leverage),
|
||||
price=adjusted_entry_price,
|
||||
trade=trade,
|
||||
is_short=trade.is_short,
|
||||
@@ -1340,11 +1343,12 @@ class FreqtradeBot(LoggingMixin):
|
||||
replacing: Optional[bool] = False
|
||||
) -> bool:
|
||||
"""
|
||||
Buy cancel - cancel order
|
||||
entry cancel - cancel order
|
||||
:param replacing: Replacing order - prevent trade deletion.
|
||||
:return: True if order was fully cancelled
|
||||
:return: True if trade was fully cancelled
|
||||
"""
|
||||
was_trade_fully_canceled = False
|
||||
side = trade.entry_side.capitalize()
|
||||
|
||||
# Cancelled orders may have the status of 'canceled' or 'closed'
|
||||
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
|
||||
@@ -1371,7 +1375,6 @@ class FreqtradeBot(LoggingMixin):
|
||||
corder = order
|
||||
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
|
||||
|
||||
side = trade.entry_side.capitalize()
|
||||
logger.info('%s order %s for %s.', side, reason, trade)
|
||||
|
||||
# Using filled to determine the filled amount
|
||||
@@ -1385,24 +1388,15 @@ class FreqtradeBot(LoggingMixin):
|
||||
was_trade_fully_canceled = True
|
||||
reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
|
||||
else:
|
||||
# FIXME TODO: This could possibly reworked to not duplicate the code 15 lines below.
|
||||
self.update_trade_state(trade, trade.open_order_id, corder)
|
||||
trade.open_order_id = None
|
||||
logger.info(f'{side} Order timeout for {trade}.')
|
||||
else:
|
||||
# if trade is partially complete, edit the stake details for the trade
|
||||
# and close the order
|
||||
# cancel_order may not contain the full order dict, so we need to fallback
|
||||
# to the order dict acquired before cancelling.
|
||||
# we need to fall back to the values from order if corder does not contain these keys.
|
||||
trade.amount = filled_amount
|
||||
# * Check edge cases, we don't want to make leverage > 1.0 if we don't have to
|
||||
# * (for leverage modes which aren't isolated futures)
|
||||
|
||||
trade.stake_amount = trade.amount * trade.open_rate / trade.leverage
|
||||
# update_trade_state (and subsequently recalc_trade_from_orders) will handle updates
|
||||
# to the trade object
|
||||
self.update_trade_state(trade, trade.open_order_id, corder)
|
||||
|
||||
trade.open_order_id = None
|
||||
|
||||
logger.info(f'Partial {trade.entry_side} order timeout for {trade}.')
|
||||
reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
|
||||
|
||||
@@ -1417,49 +1411,63 @@ class FreqtradeBot(LoggingMixin):
|
||||
:return: True if exit order was cancelled, false otherwise
|
||||
"""
|
||||
cancelled = False
|
||||
# if trade is not partially completed, just cancel the order
|
||||
if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
|
||||
if not self.exchange.check_order_canceled_empty(order):
|
||||
try:
|
||||
# if trade is not partially completed, just delete the order
|
||||
co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
|
||||
trade.amount)
|
||||
trade.update_order(co)
|
||||
except InvalidOrderException:
|
||||
logger.exception(
|
||||
f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
|
||||
return False
|
||||
logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
|
||||
else:
|
||||
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
|
||||
logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
|
||||
trade.update_order(order)
|
||||
# Cancelled orders may have the status of 'canceled' or 'closed'
|
||||
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
|
||||
filled_val: float = order.get('filled', 0.0) or 0.0
|
||||
filled_rem_stake = trade.stake_amount - filled_val * trade.open_rate
|
||||
minstake = self.exchange.get_min_pair_stake_amount(
|
||||
trade.pair, trade.open_rate, self.strategy.stoploss)
|
||||
# Double-check remaining amount
|
||||
if filled_val > 0:
|
||||
reason = constants.CANCEL_REASON['PARTIALLY_FILLED']
|
||||
if minstake and filled_rem_stake < minstake:
|
||||
logger.warning(
|
||||
f"Order {trade.open_order_id} for {trade.pair} not cancelled, as "
|
||||
f"the filled amount of {filled_val} would result in an unexitable trade.")
|
||||
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
|
||||
|
||||
self._notify_exit_cancel(
|
||||
trade,
|
||||
order_type=self.strategy.order_types['exit'],
|
||||
reason=reason, order_id=order['id'],
|
||||
sub_trade=trade.amount != order['amount']
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
|
||||
trade.amount)
|
||||
except InvalidOrderException:
|
||||
logger.exception(
|
||||
f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
|
||||
return False
|
||||
trade.close_rate = None
|
||||
trade.close_rate_requested = None
|
||||
trade.close_profit = None
|
||||
trade.close_profit_abs = None
|
||||
trade.close_date = None
|
||||
trade.is_open = True
|
||||
trade.open_order_id = None
|
||||
trade.exit_reason = None
|
||||
# Set exit_reason for fill message
|
||||
exit_reason_prev = trade.exit_reason
|
||||
trade.exit_reason = trade.exit_reason + f", {reason}" if trade.exit_reason else reason
|
||||
self.update_trade_state(trade, trade.open_order_id, co)
|
||||
# Order might be filled above in odd timing issues.
|
||||
if co.get('status') in ('canceled', 'cancelled'):
|
||||
trade.exit_reason = None
|
||||
trade.open_order_id = None
|
||||
else:
|
||||
trade.exit_reason = exit_reason_prev
|
||||
|
||||
logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
|
||||
cancelled = True
|
||||
self.wallets.update()
|
||||
else:
|
||||
# TODO: figure out how to handle partially complete sell orders
|
||||
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
|
||||
cancelled = False
|
||||
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
|
||||
logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
|
||||
self.update_trade_state(trade, trade.open_order_id, order)
|
||||
trade.open_order_id = None
|
||||
|
||||
order_obj = trade.select_order_by_order_id(order['id'])
|
||||
if not order_obj:
|
||||
raise DependencyException(
|
||||
f"Order_obj not found for {order['id']}. This should not have happened.")
|
||||
|
||||
sub_trade = order_obj.amount != trade.amount
|
||||
self._notify_exit_cancel(
|
||||
trade,
|
||||
order_type=self.strategy.order_types['exit'],
|
||||
reason=reason, order=order_obj, sub_trade=sub_trade
|
||||
reason=reason, order_id=order['id'], sub_trade=trade.amount != order['amount']
|
||||
)
|
||||
return cancelled
|
||||
|
||||
@@ -1656,7 +1664,7 @@ class FreqtradeBot(LoggingMixin):
|
||||
self.rpc.send_msg(msg)
|
||||
|
||||
def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str,
|
||||
order: Order, sub_trade: bool = False) -> None:
|
||||
order_id: str, sub_trade: bool = False) -> None:
|
||||
"""
|
||||
Sends rpc notification when a sell cancel occurred.
|
||||
"""
|
||||
@@ -1665,6 +1673,11 @@ class FreqtradeBot(LoggingMixin):
|
||||
else:
|
||||
trade.exit_order_status = reason
|
||||
|
||||
order = trade.select_order_by_order_id(order_id)
|
||||
if not order:
|
||||
raise DependencyException(
|
||||
f"Order_obj not found for {order_id}. This should not have happened.")
|
||||
|
||||
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
|
||||
profit_trade = trade.calc_profit(rate=profit_rate)
|
||||
current_rate = self.exchange.get_rate(
|
||||
@@ -1700,11 +1713,6 @@ class FreqtradeBot(LoggingMixin):
|
||||
'stake_amount': trade.stake_amount,
|
||||
}
|
||||
|
||||
if 'fiat_display_currency' in self.config:
|
||||
msg.update({
|
||||
'fiat_currency': self.config['fiat_display_currency'],
|
||||
})
|
||||
|
||||
# Send the message
|
||||
self.rpc.send_msg(msg)
|
||||
|
||||
|
@@ -114,10 +114,10 @@ class Backtesting:
|
||||
self.timeframe = str(self.config.get('timeframe'))
|
||||
self.timeframe_min = timeframe_to_minutes(self.timeframe)
|
||||
self.init_backtest_detail()
|
||||
self.pairlists = PairListManager(self.exchange, self.config)
|
||||
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
|
||||
if 'VolumePairList' in self.pairlists.name_list:
|
||||
raise OperationalException("VolumePairList not allowed for backtesting. "
|
||||
"Please use StaticPairlist instead.")
|
||||
"Please use StaticPairList instead.")
|
||||
if 'PerformanceFilter' in self.pairlists.name_list:
|
||||
raise OperationalException("PerformanceFilter not allowed for backtesting.")
|
||||
|
||||
@@ -158,9 +158,6 @@ class Backtesting:
|
||||
|
||||
self.init_backtest()
|
||||
|
||||
def __del__(self):
|
||||
self.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def cleanup():
|
||||
LoggingMixin.show_output = True
|
||||
@@ -377,10 +374,10 @@ class Backtesting:
|
||||
for col in HEADERS[5:]:
|
||||
tag_col = col in ('enter_tag', 'exit_tag')
|
||||
if col in df_analyzed.columns:
|
||||
df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace(
|
||||
df_analyzed[col] = df_analyzed.loc[:, col].replace(
|
||||
[nan], [0 if not tag_col else None]).shift(1)
|
||||
elif not df_analyzed.empty:
|
||||
df_analyzed.loc[:, col] = 0 if not tag_col else None
|
||||
df_analyzed[col] = 0 if not tag_col else None
|
||||
|
||||
df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
|
||||
|
||||
@@ -547,7 +544,7 @@ class Backtesting:
|
||||
|
||||
if stake_amount is not None and stake_amount < 0.0:
|
||||
amount = amount_to_contract_precision(
|
||||
abs(stake_amount) / current_rate, trade.amount_precision,
|
||||
abs(stake_amount * trade.leverage) / current_rate, trade.amount_precision,
|
||||
self.precision_mode, trade.contract_size)
|
||||
if amount == 0.0:
|
||||
return trade
|
||||
@@ -1052,7 +1049,7 @@ class Backtesting:
|
||||
if requested_rate:
|
||||
self._enter_trade(pair=trade.pair, row=row, trade=trade,
|
||||
requested_rate=requested_rate,
|
||||
requested_stake=(order.remaining * order.price),
|
||||
requested_stake=(order.remaining * order.price / trade.leverage),
|
||||
direction='short' if trade.is_short else 'long')
|
||||
self.replaced_entry_orders += 1
|
||||
else:
|
||||
|
@@ -24,6 +24,7 @@ from pandas import DataFrame
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
|
||||
from freqtrade.data.converter import trim_dataframes
|
||||
from freqtrade.data.history import get_timerange
|
||||
from freqtrade.data.metrics import calculate_market_change
|
||||
from freqtrade.enums import HyperoptState
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
|
||||
@@ -61,7 +62,7 @@ class Hyperopt:
|
||||
"""
|
||||
Hyperopt class, this class contains all the logic to run a hyperopt simulation
|
||||
|
||||
To run a backtest:
|
||||
To start a hyperopt run:
|
||||
hyperopt = Hyperopt(config)
|
||||
hyperopt.start()
|
||||
"""
|
||||
@@ -111,6 +112,7 @@ class Hyperopt:
|
||||
|
||||
self.clean_hyperopt()
|
||||
|
||||
self.market_change = 0.0
|
||||
self.num_epochs_saved = 0
|
||||
self.current_best_epoch: Optional[Dict[str, Any]] = None
|
||||
|
||||
@@ -357,7 +359,7 @@ class Hyperopt:
|
||||
|
||||
strat_stats = generate_strategy_stats(
|
||||
self.pairlist, self.backtesting.strategy.get_strategy_name(),
|
||||
backtesting_results, min_date, max_date, market_change=0
|
||||
backtesting_results, min_date, max_date, market_change=self.market_change
|
||||
)
|
||||
results_explanation = HyperoptTools.format_results_explanation_string(
|
||||
strat_stats, self.config['stake_currency'])
|
||||
@@ -425,6 +427,9 @@ class Hyperopt:
|
||||
# Trim startup period from analyzed dataframe to get correct dates for output.
|
||||
trimmed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup)
|
||||
self.min_date, self.max_date = get_timerange(trimmed)
|
||||
if not self.market_change:
|
||||
self.market_change = calculate_market_change(trimmed, 'close')
|
||||
|
||||
# Real trimming will happen as part of backtesting.
|
||||
return preprocessed
|
||||
|
||||
|
@@ -173,7 +173,7 @@ def generate_tag_metrics(tag_type: str,
|
||||
tabular_data = []
|
||||
|
||||
if tag_type in results.columns:
|
||||
for tag, count in results[tag_type].value_counts().iteritems():
|
||||
for tag, count in results[tag_type].value_counts().items():
|
||||
result = results[results[tag_type] == tag]
|
||||
if skip_nan and result['profit_abs'].isnull().all():
|
||||
continue
|
||||
@@ -199,7 +199,7 @@ def generate_exit_reason_stats(max_open_trades: int, results: DataFrame) -> List
|
||||
"""
|
||||
tabular_data = []
|
||||
|
||||
for reason, count in results['exit_reason'].value_counts().iteritems():
|
||||
for reason, count in results['exit_reason'].value_counts().items():
|
||||
result = results.loc[results['exit_reason'] == reason]
|
||||
|
||||
profit_mean = result['profit_ratio'].mean()
|
||||
@@ -361,7 +361,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
|
||||
winning_days = sum(daily_profit > 0)
|
||||
draw_days = sum(daily_profit == 0)
|
||||
losing_days = sum(daily_profit < 0)
|
||||
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
|
||||
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.items()]
|
||||
|
||||
return {
|
||||
'backtest_best_day': best_rel,
|
||||
|
90
freqtrade/plugins/pairlist/ProducerPairList.py
Normal file
90
freqtrade/plugins/pairlist/ProducerPairList.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
External Pair List provider
|
||||
|
||||
Provides pair list from Leader data
|
||||
"""
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ProducerPairList(IPairList):
|
||||
"""
|
||||
PairList plugin for use with external_message_consumer.
|
||||
Will use pairs given from leader data.
|
||||
|
||||
Usage:
|
||||
"pairlists": [
|
||||
{
|
||||
"method": "ProducerPairList",
|
||||
"number_assets": 5,
|
||||
"producer_name": "default",
|
||||
}
|
||||
],
|
||||
"""
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
self._num_assets: int = self._pairlistconfig.get('number_assets', 0)
|
||||
self._producer_name = self._pairlistconfig.get('producer_name', 'default')
|
||||
if not config.get('external_message_consumer', {}).get('enabled'):
|
||||
raise OperationalException(
|
||||
"ProducerPairList requires external_message_consumer to be enabled.")
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
Boolean property defining if tickers are necessary.
|
||||
If no Pairlist requires tickers, an empty Dict is passed
|
||||
as tickers argument to filter_pairlist
|
||||
"""
|
||||
return False
|
||||
|
||||
def short_desc(self) -> str:
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
-> Please overwrite in subclasses
|
||||
"""
|
||||
return f"{self.name} - {self._producer_name}"
|
||||
|
||||
def _filter_pairlist(self, pairlist: Optional[List[str]]):
|
||||
upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(
|
||||
self._producer_name)
|
||||
|
||||
if pairlist is None:
|
||||
pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(self._producer_name)
|
||||
|
||||
pairs = list(dict.fromkeys(pairlist + upstream_pairlist))
|
||||
if self._num_assets:
|
||||
pairs = pairs[:self._num_assets]
|
||||
|
||||
return pairs
|
||||
|
||||
def gen_pairlist(self, tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Generate the pairlist
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: List of pairs
|
||||
"""
|
||||
pairs = self._filter_pairlist(None)
|
||||
self.log_once(f"Received pairs: {pairs}", logger.debug)
|
||||
pairs = self._whitelist_for_active_markets(self.verify_whitelist(pairs, logger.info))
|
||||
return pairs
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Filters and sorts pairlist and returns the whitelist again.
|
||||
Called on each bot iteration - please use internal caching if necessary
|
||||
:param pairlist: pairlist to filter or sort
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: new whitelist
|
||||
"""
|
||||
return self._filter_pairlist(pairlist)
|
@@ -232,6 +232,4 @@ class VolumePairList(IPairList):
|
||||
# Limit pairlist to the requested number of pairs
|
||||
pairs = pairs[:self._number_pairs]
|
||||
|
||||
self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
|
||||
|
||||
return pairs
|
||||
|
@@ -3,11 +3,12 @@ PairList manager class
|
||||
"""
|
||||
import logging
|
||||
from functools import partial
|
||||
from typing import Dict, List
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
from freqtrade.constants import Config, ListPairsWithTimeframes
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.enums import CandleType
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.mixins import LoggingMixin
|
||||
@@ -21,13 +22,14 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class PairListManager(LoggingMixin):
|
||||
|
||||
def __init__(self, exchange, config: Config) -> None:
|
||||
def __init__(self, exchange, config: Config, dataprovider: DataProvider = None) -> None:
|
||||
self._exchange = exchange
|
||||
self._config = config
|
||||
self._whitelist = self._config['exchange'].get('pair_whitelist')
|
||||
self._blacklist = self._config['exchange'].get('pair_blacklist', [])
|
||||
self._pairlist_handlers: List[IPairList] = []
|
||||
self._tickers_needed = False
|
||||
self._dataprovider: Optional[DataProvider] = dataprovider
|
||||
for pairlist_handler_config in self._config.get('pairlists', []):
|
||||
pairlist_handler = PairListResolver.load_pairlist(
|
||||
pairlist_handler_config['method'],
|
||||
@@ -96,6 +98,8 @@ class PairListManager(LoggingMixin):
|
||||
# to ensure blacklist is respected.
|
||||
pairlist = self.verify_blacklist(pairlist, logger.warning)
|
||||
|
||||
self.log_once(f"Whitelist with {len(pairlist)} pairs: {pairlist}", logger.info)
|
||||
|
||||
self._whitelist = pairlist
|
||||
|
||||
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
|
||||
|
@@ -5,6 +5,7 @@ from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from fastapi.exceptions import HTTPException
|
||||
|
||||
from freqtrade.configuration.config_validation import validate_config_consistency
|
||||
from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
|
||||
@@ -31,6 +32,9 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
|
||||
if ApiServer._bgtask_running:
|
||||
raise RPCException('Bot Background task already running')
|
||||
|
||||
if ':' in bt_settings.strategy:
|
||||
raise HTTPException(status_code=500, detail="base64 encoded strategies are not allowed.")
|
||||
|
||||
btconfig = deepcopy(config)
|
||||
settings = dict(bt_settings)
|
||||
# Pydantic models will contain all keys, but non-provided ones are None
|
||||
|
@@ -265,6 +265,8 @@ def list_strategies(config=Depends(get_config)):
|
||||
|
||||
@router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
|
||||
def get_strategy(strategy: str, config=Depends(get_config)):
|
||||
if ":" in strategy:
|
||||
raise HTTPException(status_code=500, detail="base64 encoded strategies are not allowed.")
|
||||
|
||||
config_ = deepcopy(config)
|
||||
from freqtrade.resolvers.strategy_resolver import StrategyResolver
|
||||
|
@@ -198,8 +198,10 @@ class ApiServer(RPCHandler):
|
||||
logger.debug(f"Found message of type: {message.get('type')}")
|
||||
# Broadcast it
|
||||
await self._ws_channel_manager.broadcast(message)
|
||||
# Sleep, make this configurable?
|
||||
await asyncio.sleep(0.1)
|
||||
# Limit messages per sec.
|
||||
# Could cause problems with queue size if too low, and
|
||||
# problems with network traffik if too high.
|
||||
await asyncio.sleep(0.001)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
@@ -140,7 +140,7 @@ class ChannelManager:
|
||||
Disconnect all Channels
|
||||
"""
|
||||
with self._lock:
|
||||
for websocket, channel in self.channels.items():
|
||||
for websocket, channel in self.channels.copy().items():
|
||||
if not channel.is_closed():
|
||||
await channel.close()
|
||||
|
||||
@@ -154,7 +154,7 @@ class ChannelManager:
|
||||
"""
|
||||
with self._lock:
|
||||
message_type = data.get('type')
|
||||
for websocket, channel in self.channels.items():
|
||||
for websocket, channel in self.channels.copy().items():
|
||||
try:
|
||||
if channel.subscribed_to(message_type):
|
||||
await channel.send(data)
|
||||
|
@@ -30,9 +30,9 @@ class Discord(Webhook):
|
||||
pass
|
||||
|
||||
def send_msg(self, msg) -> None:
|
||||
logger.info(f"Sending discord message: {msg}")
|
||||
|
||||
if msg['type'].value in self.config['discord']:
|
||||
logger.info(f"Sending discord message: {msg}")
|
||||
|
||||
msg['strategy'] = self.strategy
|
||||
msg['timeframe'] = self.timeframe
|
||||
|
@@ -284,7 +284,7 @@ class ExternalMessageConsumer:
|
||||
logger.error(f"Empty message received from `{producer_name}`")
|
||||
return
|
||||
|
||||
logger.info(f"Received message of type `{producer_message.type}` from `{producer_name}`")
|
||||
logger.debug(f"Received message of type `{producer_message.type}` from `{producer_name}`")
|
||||
|
||||
message_handler = self._message_handlers.get(producer_message.type)
|
||||
|
||||
|
@@ -3,8 +3,8 @@ Module that define classes to convert Crypto-currency to FIAT
|
||||
e.g BTC to USD
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, List
|
||||
|
||||
from cachetools import TTLCache
|
||||
@@ -46,7 +46,9 @@ class CryptoToFiatConverter(LoggingMixin):
|
||||
if CryptoToFiatConverter.__instance is None:
|
||||
CryptoToFiatConverter.__instance = object.__new__(cls)
|
||||
try:
|
||||
CryptoToFiatConverter._coingekko = CoinGeckoAPI()
|
||||
# Limit retires to 1 (0 and 1)
|
||||
# otherwise we risk bot impact if coingecko is down.
|
||||
CryptoToFiatConverter._coingekko = CoinGeckoAPI(retries=1)
|
||||
except BaseException:
|
||||
CryptoToFiatConverter._coingekko = None
|
||||
return CryptoToFiatConverter.__instance
|
||||
@@ -67,7 +69,7 @@ class CryptoToFiatConverter(LoggingMixin):
|
||||
logger.warning(
|
||||
"Too many requests for CoinGecko API, backing off and trying again later.")
|
||||
# Set backoff timestamp to 60 seconds in the future
|
||||
self._backoff = datetime.datetime.now().timestamp() + 60
|
||||
self._backoff = datetime.now().timestamp() + 60
|
||||
return
|
||||
# If the request is not a 429 error we want to raise the normal error
|
||||
logger.error(
|
||||
@@ -81,7 +83,7 @@ class CryptoToFiatConverter(LoggingMixin):
|
||||
|
||||
def _get_gekko_id(self, crypto_symbol):
|
||||
if not self._coinlistings:
|
||||
if self._backoff <= datetime.datetime.now().timestamp():
|
||||
if self._backoff <= datetime.now().timestamp():
|
||||
self._load_cryptomap()
|
||||
# Still not loaded.
|
||||
if not self._coinlistings:
|
||||
|
@@ -25,7 +25,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
|
||||
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
|
||||
from freqtrade.loggers import bufferHandler
|
||||
from freqtrade.misc import decimals_per_coin, shorten_date
|
||||
from freqtrade.persistence import PairLocks, Trade
|
||||
from freqtrade.persistence import Order, PairLocks, Trade
|
||||
from freqtrade.persistence.models import PairLock
|
||||
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
|
||||
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
|
||||
@@ -166,9 +166,9 @@ class RPC:
|
||||
else:
|
||||
results = []
|
||||
for trade in trades:
|
||||
order = None
|
||||
order: Optional[Order] = None
|
||||
if trade.open_order_id:
|
||||
order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
|
||||
order = trade.select_order_by_order_id(trade.open_order_id)
|
||||
# calculate profit and send message to user
|
||||
if trade.is_open:
|
||||
try:
|
||||
@@ -219,7 +219,7 @@ class RPC:
|
||||
stoploss_entry_dist=stoploss_entry_dist,
|
||||
stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
|
||||
open_order='({} {} rem={:.8f})'.format(
|
||||
order['type'], order['side'], order['remaining']
|
||||
order.order_type, order.side, order.remaining
|
||||
) if order else None,
|
||||
))
|
||||
results.append(trade_dict)
|
||||
@@ -773,6 +773,9 @@ class RPC:
|
||||
is_short = trade.is_short
|
||||
if not self._freqtrade.strategy.position_adjustment_enable:
|
||||
raise RPCException(f'position for {pair} already open - id: {trade.id}')
|
||||
else:
|
||||
if Trade.get_open_trade_count() >= self._config['max_open_trades']:
|
||||
raise RPCException("Maximum number of trades is reached.")
|
||||
|
||||
if not stake_amount:
|
||||
# gen stake amount
|
||||
|
@@ -67,7 +67,7 @@ class RPCManager:
|
||||
'status': 'stopping bot'
|
||||
}
|
||||
"""
|
||||
if msg.get('type') is not RPCMessageType.ANALYZED_DF:
|
||||
if msg.get('type') not in (RPCMessageType.ANALYZED_DF, RPCMessageType.WHITELIST):
|
||||
logger.info('Sending rpc message: %s', msg)
|
||||
if 'pair' in msg:
|
||||
msg.update({
|
||||
|
@@ -61,6 +61,14 @@ class Webhook(RPCHandler):
|
||||
RPCMessageType.STARTUP,
|
||||
RPCMessageType.WARNING):
|
||||
valuedict = whconfig.get('webhookstatus')
|
||||
elif msg['type'] in (
|
||||
RPCMessageType.PROTECTION_TRIGGER,
|
||||
RPCMessageType.PROTECTION_TRIGGER_GLOBAL,
|
||||
RPCMessageType.WHITELIST,
|
||||
RPCMessageType.ANALYZED_DF,
|
||||
RPCMessageType.STRATEGY_MSG):
|
||||
# Don't fail for non-implemented types
|
||||
return
|
||||
else:
|
||||
raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
|
||||
if not valuedict:
|
||||
|
@@ -5,6 +5,7 @@
|
||||
import numpy as np # noqa
|
||||
import pandas as pd # noqa
|
||||
from pandas import DataFrame
|
||||
from typing import Optional, Union
|
||||
|
||||
from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter,
|
||||
IStrategy, IntParameter)
|
||||
|
Reference in New Issue
Block a user