Merge remote-tracking branch 'origin/develop' into spice-rack

This commit is contained in:
robcaulk
2022-10-08 12:25:46 +02:00
99 changed files with 2734 additions and 1313 deletions

View File

@@ -1,5 +1,5 @@
""" Freqtrade bot """
__version__ = '2022.9.dev'
__version__ = '2022.10.dev'
if 'dev' in __version__:
try:

View File

@@ -1,6 +1,5 @@
# flake8: noqa: F401
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration

View File

@@ -8,7 +8,6 @@ from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
@@ -100,6 +99,9 @@ class Configuration:
self._process_freqai_options(config)
# Import check_exchange here to avoid import cycle problems
from freqtrade.exchange.check_exchange import check_exchange
# Check if the exchange set by the user is supported
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))

View File

@@ -31,7 +31,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'CalmarHyperOptLoss',
'MaxDrawDownHyperOptLoss', 'MaxDrawDownRelativeHyperOptLoss',
'ProfitDrawDownHyperOptLoss']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'ProducerPairList',
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
@@ -552,7 +552,7 @@ CONF_SCHEMA = {
"weight_factor": {"type": "number", "default": 0},
"principal_component_analysis": {"type": "boolean", "default": False},
"use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
"plot_feature_importance": {"type": "boolean", "default": False},
"plot_feature_importances": {"type": "integer", "default": 0},
"svm_params": {"type": "object",
"properties": {
"shuffle": {"type": "boolean", "default": False},
@@ -567,6 +567,7 @@ CONF_SCHEMA = {
"properties": {
"test_size": {"type": "number"},
"random_state": {"type": "integer"},
"shuffle": {"type": "boolean", "default": False}
},
},
"model_training_parameters": {

View File

@@ -284,7 +284,7 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1)
if 'orders' not in df.columns:
df.loc[:, 'orders'] = None
df['orders'] = None
else:
# old format - only with lists.
@@ -341,9 +341,9 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame:
"""
df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS)
if len(df) > 0:
df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True)
df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True)
df.loc[:, 'close_rate'] = df['close_rate'].astype('float64')
df['close_date'] = pd.to_datetime(df['close_date'], utc=True)
df['open_date'] = pd.to_datetime(df['open_date'], utc=True)
df['close_rate'] = df['close_rate'].astype('float64')
return df

View File

@@ -47,8 +47,7 @@ def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *,
def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
fill_missing: bool = True,
drop_incomplete: bool = True) -> DataFrame:
fill_missing: bool, drop_incomplete: bool) -> DataFrame:
"""
Cleanse a OHLCV dataframe by
* Grouping it by date (removes duplicate tics)

View File

@@ -26,7 +26,7 @@ def load_pair_history(pair: str,
datadir: Path, *,
timerange: Optional[TimeRange] = None,
fill_up_missing: bool = True,
drop_incomplete: bool = True,
drop_incomplete: bool = False,
startup_candles: int = 0,
data_format: str = None,
data_handler: IDataHandler = None,

View File

@@ -272,10 +272,10 @@ class IDataHandler(ABC):
return res
def ohlcv_load(self, pair, timeframe: str,
candle_type: CandleType,
candle_type: CandleType, *,
timerange: Optional[TimeRange] = None,
fill_missing: bool = True,
drop_incomplete: bool = True,
drop_incomplete: bool = False,
startup_candles: int = 0,
warn_no_data: bool = True,
) -> DataFrame:

View File

@@ -12,8 +12,8 @@ from freqtrade.exchange.coinbasepro import Coinbasepro
from freqtrade.exchange.exchange import (amount_to_contract_precision, amount_to_contracts,
amount_to_precision, available_exchanges, ccxt_exchanges,
contracts_to_amount, date_minus_candles,
is_exchange_known_ccxt, is_exchange_officially_supported,
market_is_active, price_to_precision, timeframe_to_minutes,
is_exchange_known_ccxt, market_is_active,
price_to_precision, timeframe_to_minutes,
timeframe_to_msecs, timeframe_to_next_date,
timeframe_to_prev_date, timeframe_to_seconds,
validate_exchange, validate_exchanges)

View File

@@ -68,6 +68,37 @@ class Binance(Exchange):
tickers = deep_merge_dicts(bidsasks, tickers, allow_null_overrides=False)
return tickers
@retrier
def additional_exchange_init(self) -> None:
"""
Additional exchange initialization logic.
.api will be available at this point.
Must be overridden in child methods if required.
"""
try:
if self.trading_mode == TradingMode.FUTURES and not self._config['dry_run']:
position_side = self._api.fapiPrivateGetPositionsideDual()
self._log_exchange_response('position_side_setting', position_side)
assets_margin = self._api.fapiPrivateGetMultiAssetsMargin()
self._log_exchange_response('multi_asset_margin', assets_margin)
msg = ""
if position_side.get('dualSidePosition') is True:
msg += (
"\nHedge Mode is not supported by freqtrade. "
"Please change 'Position Mode' on your binance futures account.")
if assets_margin.get('multiAssetsMargin') is True:
msg += ("\nMulti-Asset Mode is not supported by freqtrade. "
"Please change 'Asset Mode' on your binance futures account.")
if msg:
raise OperationalException(msg)
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
@retrier
def _set_leverage(
self,

View File

@@ -4485,6 +4485,120 @@
}
}
],
"BTCUSDT_221230": [
{
"tier": 1.0,
"currency": "USDT",
"minNotional": 0.0,
"maxNotional": 375000.0,
"maintenanceMarginRate": 0.02,
"maxLeverage": 25.0,
"info": {
"bracket": "1",
"initialLeverage": "25",
"notionalCap": "375000",
"notionalFloor": "0",
"maintMarginRatio": "0.02",
"cum": "0.0"
}
},
{
"tier": 2.0,
"currency": "USDT",
"minNotional": 375000.0,
"maxNotional": 2000000.0,
"maintenanceMarginRate": 0.05,
"maxLeverage": 10.0,
"info": {
"bracket": "2",
"initialLeverage": "10",
"notionalCap": "2000000",
"notionalFloor": "375000",
"maintMarginRatio": "0.05",
"cum": "11250.0"
}
},
{
"tier": 3.0,
"currency": "USDT",
"minNotional": 2000000.0,
"maxNotional": 4000000.0,
"maintenanceMarginRate": 0.1,
"maxLeverage": 5.0,
"info": {
"bracket": "3",
"initialLeverage": "5",
"notionalCap": "4000000",
"notionalFloor": "2000000",
"maintMarginRatio": "0.1",
"cum": "111250.0"
}
},
{
"tier": 4.0,
"currency": "USDT",
"minNotional": 4000000.0,
"maxNotional": 10000000.0,
"maintenanceMarginRate": 0.125,
"maxLeverage": 4.0,
"info": {
"bracket": "4",
"initialLeverage": "4",
"notionalCap": "10000000",
"notionalFloor": "4000000",
"maintMarginRatio": "0.125",
"cum": "211250.0"
}
},
{
"tier": 5.0,
"currency": "USDT",
"minNotional": 10000000.0,
"maxNotional": 20000000.0,
"maintenanceMarginRate": 0.15,
"maxLeverage": 3.0,
"info": {
"bracket": "5",
"initialLeverage": "3",
"notionalCap": "20000000",
"notionalFloor": "10000000",
"maintMarginRatio": "0.15",
"cum": "461250.0"
}
},
{
"tier": 6.0,
"currency": "USDT",
"minNotional": 20000000.0,
"maxNotional": 40000000.0,
"maintenanceMarginRate": 0.25,
"maxLeverage": 2.0,
"info": {
"bracket": "6",
"initialLeverage": "2",
"notionalCap": "40000000",
"notionalFloor": "20000000",
"maintMarginRatio": "0.25",
"cum": "2461250.0"
}
},
{
"tier": 7.0,
"currency": "USDT",
"minNotional": 40000000.0,
"maxNotional": 400000000.0,
"maintenanceMarginRate": 0.5,
"maxLeverage": 1.0,
"info": {
"bracket": "7",
"initialLeverage": "1",
"notionalCap": "400000000",
"notionalFloor": "40000000",
"maintMarginRatio": "0.5",
"cum": "1.246125E7"
}
}
],
"BTS/USDT": [
{
"tier": 1.0,
@@ -5759,6 +5873,104 @@
}
}
],
"CVX/USDT": [
{
"tier": 1.0,
"currency": "USDT",
"minNotional": 0.0,
"maxNotional": 5000.0,
"maintenanceMarginRate": 0.01,
"maxLeverage": 25.0,
"info": {
"bracket": "1",
"initialLeverage": "25",
"notionalCap": "5000",
"notionalFloor": "0",
"maintMarginRatio": "0.01",
"cum": "0.0"
}
},
{
"tier": 2.0,
"currency": "USDT",
"minNotional": 5000.0,
"maxNotional": 25000.0,
"maintenanceMarginRate": 0.025,
"maxLeverage": 20.0,
"info": {
"bracket": "2",
"initialLeverage": "20",
"notionalCap": "25000",
"notionalFloor": "5000",
"maintMarginRatio": "0.025",
"cum": "75.0"
}
},
{
"tier": 3.0,
"currency": "USDT",
"minNotional": 25000.0,
"maxNotional": 100000.0,
"maintenanceMarginRate": 0.05,
"maxLeverage": 10.0,
"info": {
"bracket": "3",
"initialLeverage": "10",
"notionalCap": "100000",
"notionalFloor": "25000",
"maintMarginRatio": "0.05",
"cum": "700.0"
}
},
{
"tier": 4.0,
"currency": "USDT",
"minNotional": 100000.0,
"maxNotional": 250000.0,
"maintenanceMarginRate": 0.1,
"maxLeverage": 5.0,
"info": {
"bracket": "4",
"initialLeverage": "5",
"notionalCap": "250000",
"notionalFloor": "100000",
"maintMarginRatio": "0.1",
"cum": "5700.0"
}
},
{
"tier": 5.0,
"currency": "USDT",
"minNotional": 250000.0,
"maxNotional": 1000000.0,
"maintenanceMarginRate": 0.125,
"maxLeverage": 2.0,
"info": {
"bracket": "5",
"initialLeverage": "2",
"notionalCap": "1000000",
"notionalFloor": "250000",
"maintMarginRatio": "0.125",
"cum": "11950.0"
}
},
{
"tier": 6.0,
"currency": "USDT",
"minNotional": 1000000.0,
"maxNotional": 5000000.0,
"maintenanceMarginRate": 0.5,
"maxLeverage": 1.0,
"info": {
"bracket": "6",
"initialLeverage": "1",
"notionalCap": "5000000",
"notionalFloor": "1000000",
"maintMarginRatio": "0.5",
"cum": "386950.0"
}
}
],
"DAR/USDT": [
{
"tier": 1.0,
@@ -8105,6 +8317,120 @@
}
}
],
"ETHUSDT_221230": [
{
"tier": 1.0,
"currency": "USDT",
"minNotional": 0.0,
"maxNotional": 375000.0,
"maintenanceMarginRate": 0.02,
"maxLeverage": 25.0,
"info": {
"bracket": "1",
"initialLeverage": "25",
"notionalCap": "375000",
"notionalFloor": "0",
"maintMarginRatio": "0.02",
"cum": "0.0"
}
},
{
"tier": 2.0,
"currency": "USDT",
"minNotional": 375000.0,
"maxNotional": 2000000.0,
"maintenanceMarginRate": 0.05,
"maxLeverage": 10.0,
"info": {
"bracket": "2",
"initialLeverage": "10",
"notionalCap": "2000000",
"notionalFloor": "375000",
"maintMarginRatio": "0.05",
"cum": "11250.0"
}
},
{
"tier": 3.0,
"currency": "USDT",
"minNotional": 2000000.0,
"maxNotional": 4000000.0,
"maintenanceMarginRate": 0.1,
"maxLeverage": 5.0,
"info": {
"bracket": "3",
"initialLeverage": "5",
"notionalCap": "4000000",
"notionalFloor": "2000000",
"maintMarginRatio": "0.1",
"cum": "111250.0"
}
},
{
"tier": 4.0,
"currency": "USDT",
"minNotional": 4000000.0,
"maxNotional": 10000000.0,
"maintenanceMarginRate": 0.125,
"maxLeverage": 4.0,
"info": {
"bracket": "4",
"initialLeverage": "4",
"notionalCap": "10000000",
"notionalFloor": "4000000",
"maintMarginRatio": "0.125",
"cum": "211250.0"
}
},
{
"tier": 5.0,
"currency": "USDT",
"minNotional": 10000000.0,
"maxNotional": 20000000.0,
"maintenanceMarginRate": 0.15,
"maxLeverage": 3.0,
"info": {
"bracket": "5",
"initialLeverage": "3",
"notionalCap": "20000000",
"notionalFloor": "10000000",
"maintMarginRatio": "0.15",
"cum": "461250.0"
}
},
{
"tier": 6.0,
"currency": "USDT",
"minNotional": 20000000.0,
"maxNotional": 40000000.0,
"maintenanceMarginRate": 0.25,
"maxLeverage": 2.0,
"info": {
"bracket": "6",
"initialLeverage": "2",
"notionalCap": "40000000",
"notionalFloor": "20000000",
"maintMarginRatio": "0.25",
"cum": "2461250.0"
}
},
{
"tier": 7.0,
"currency": "USDT",
"minNotional": 40000000.0,
"maxNotional": 400000000.0,
"maintenanceMarginRate": 0.5,
"maxLeverage": 1.0,
"info": {
"bracket": "7",
"initialLeverage": "1",
"notionalCap": "400000000",
"notionalFloor": "40000000",
"maintMarginRatio": "0.5",
"cum": "1.246125E7"
}
}
],
"FIL/BUSD": [
{
"tier": 1.0,
@@ -10138,10 +10464,10 @@
"minNotional": 0.0,
"maxNotional": 5000.0,
"maintenanceMarginRate": 0.01,
"maxLeverage": 50.0,
"maxLeverage": 25.0,
"info": {
"bracket": "1",
"initialLeverage": "50",
"initialLeverage": "25",
"notionalCap": "5000",
"notionalFloor": "0",
"maintMarginRatio": "0.01",
@@ -10216,13 +10542,13 @@
"tier": 6.0,
"currency": "USDT",
"minNotional": 1000000.0,
"maxNotional": 30000000.0,
"maxNotional": 5000000.0,
"maintenanceMarginRate": 0.5,
"maxLeverage": 1.0,
"info": {
"bracket": "6",
"initialLeverage": "1",
"notionalCap": "30000000",
"notionalCap": "5000000",
"notionalFloor": "1000000",
"maintMarginRatio": "0.5",
"cum": "386950.0"
@@ -11389,6 +11715,104 @@
}
}
],
"LDO/USDT": [
{
"tier": 1.0,
"currency": "USDT",
"minNotional": 0.0,
"maxNotional": 5000.0,
"maintenanceMarginRate": 0.01,
"maxLeverage": 25.0,
"info": {
"bracket": "1",
"initialLeverage": "25",
"notionalCap": "5000",
"notionalFloor": "0",
"maintMarginRatio": "0.01",
"cum": "0.0"
}
},
{
"tier": 2.0,
"currency": "USDT",
"minNotional": 5000.0,
"maxNotional": 25000.0,
"maintenanceMarginRate": 0.025,
"maxLeverage": 20.0,
"info": {
"bracket": "2",
"initialLeverage": "20",
"notionalCap": "25000",
"notionalFloor": "5000",
"maintMarginRatio": "0.025",
"cum": "75.0"
}
},
{
"tier": 3.0,
"currency": "USDT",
"minNotional": 25000.0,
"maxNotional": 100000.0,
"maintenanceMarginRate": 0.05,
"maxLeverage": 10.0,
"info": {
"bracket": "3",
"initialLeverage": "10",
"notionalCap": "100000",
"notionalFloor": "25000",
"maintMarginRatio": "0.05",
"cum": "700.0"
}
},
{
"tier": 4.0,
"currency": "USDT",
"minNotional": 100000.0,
"maxNotional": 250000.0,
"maintenanceMarginRate": 0.1,
"maxLeverage": 5.0,
"info": {
"bracket": "4",
"initialLeverage": "5",
"notionalCap": "250000",
"notionalFloor": "100000",
"maintMarginRatio": "0.1",
"cum": "5700.0"
}
},
{
"tier": 5.0,
"currency": "USDT",
"minNotional": 250000.0,
"maxNotional": 1000000.0,
"maintenanceMarginRate": 0.125,
"maxLeverage": 2.0,
"info": {
"bracket": "5",
"initialLeverage": "2",
"notionalCap": "1000000",
"notionalFloor": "250000",
"maintMarginRatio": "0.125",
"cum": "11950.0"
}
},
{
"tier": 6.0,
"currency": "USDT",
"minNotional": 1000000.0,
"maxNotional": 5000000.0,
"maintenanceMarginRate": 0.5,
"maxLeverage": 1.0,
"info": {
"bracket": "6",
"initialLeverage": "1",
"notionalCap": "5000000",
"notionalFloor": "1000000",
"maintMarginRatio": "0.5",
"cum": "386950.0"
}
}
],
"LEVER/BUSD": [
{
"tier": 1.0,

View File

@@ -3,8 +3,8 @@ import logging
from freqtrade.constants import Config
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
is_exchange_officially_supported, validate_exchange)
from freqtrade.exchange import available_exchanges, is_exchange_known_ccxt, validate_exchange
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS, SUPPORTED_EXCHANGES
logger = logging.getLogger(__name__)
@@ -52,7 +52,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
else:
logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}')
if is_exchange_officially_supported(exchange):
if MAP_EXCHANGE_CHILDCLASS.get(exchange, exchange) in SUPPORTED_EXCHANGES:
logger.info(f'Exchange "{exchange}" is officially supported '
f'by the Freqtrade development team.')
else:

View File

@@ -18,20 +18,19 @@ import ccxt.async_support as ccxt_async
from cachetools import TTLCache
from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision
from dateutil import parser
from pandas import DataFrame
from pandas import DataFrame, concat
from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
Config, EntryExit, ListPairsWithTimeframes, MakerTaker,
PairWithTimeframe)
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.data.converter import clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.enums import OPTIMIZE_MODES, CandleType, MarginMode, TradingMode
from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
InvalidOrderException, OperationalException, PricingError,
RetryableOrderError, TemporaryError)
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
SUPPORTED_EXCHANGES, remove_credentials, retrier,
retrier_async)
remove_credentials, retrier, retrier_async)
from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
safe_value_fallback2)
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -185,8 +184,9 @@ class Exchange:
# Initial markets load
self._load_markets()
self.validate_config(config)
self._startup_candle_count: int = config.get('startup_candle_count', 0)
self.required_candle_call_count = self.validate_required_startup_candles(
config.get('startup_candle_count', 0), config.get('timeframe', ''))
self._startup_candle_count, config.get('timeframe', ''))
# Converts the interval provided in minutes in config to seconds
self.markets_refresh_interval: int = exchange_config.get(
@@ -1292,7 +1292,14 @@ class Exchange:
order = self.fetch_order(order_id, pair)
except InvalidOrderException:
logger.warning(f"Could not fetch cancelled order {order_id}.")
order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
order = {
'id': order_id,
'status': 'canceled',
'amount': amount,
'filled': 0.0,
'fee': {},
'info': {}
}
return order
@@ -1844,10 +1851,22 @@ class Exchange:
return pair, timeframe, candle_type, data
def _build_coroutine(self, pair: str, timeframe: str, candle_type: CandleType,
since_ms: Optional[int]) -> Coroutine:
since_ms: Optional[int], cache: bool) -> Coroutine:
not_all_data = self.required_candle_call_count > 1
if cache and (pair, timeframe, candle_type) in self._klines:
candle_limit = self.ohlcv_candle_limit(timeframe, candle_type)
min_date = date_minus_candles(timeframe, candle_limit - 5).timestamp()
# Check if 1 call can get us updated candles without hole in the data.
if min_date < self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0):
# Cache can be used - do one-off call.
not_all_data = False
else:
# Time jump detected, evict cache
logger.info(
f"Time jump detected. Evicting cache for {pair}, {timeframe}, {candle_type}")
del self._klines[(pair, timeframe, candle_type)]
if (not since_ms
and (self._ft_has["ohlcv_require_since"] or self.required_candle_call_count > 1)):
if (not since_ms and (self._ft_has["ohlcv_require_since"] or not_all_data)):
# Multiple calls for one pair - to get more history
one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(
timeframe, candle_type, since_ms)
@@ -1863,6 +1882,59 @@ class Exchange:
return self._async_get_candle_history(
pair, timeframe, since_ms=since_ms, candle_type=candle_type)
def _build_ohlcv_dl_jobs(
self, pair_list: ListPairsWithTimeframes, since_ms: Optional[int],
cache: bool) -> Tuple[List[Coroutine], List[Tuple[str, str, CandleType]]]:
"""
Build Coroutines to execute as part of refresh_latest_ohlcv
"""
input_coroutines = []
cached_pairs = []
for pair, timeframe, candle_type in set(pair_list):
if (timeframe not in self.timeframes
and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
logger.warning(
f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
f"not available on {self.name}. Available timeframes are "
f"{', '.join(self.timeframes)}.")
continue
if ((pair, timeframe, candle_type) not in self._klines or not cache
or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
input_coroutines.append(
self._build_coroutine(pair, timeframe, candle_type, since_ms, cache))
else:
logger.debug(
f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
)
cached_pairs.append((pair, timeframe, candle_type))
return input_coroutines, cached_pairs
def _process_ohlcv_df(self, pair: str, timeframe: str, c_type: CandleType, ticks: List[List],
cache: bool, drop_incomplete: bool) -> DataFrame:
# keeping last candle time as last refreshed time of the pair
if ticks and cache:
self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
ohlcv_df = ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=drop_incomplete)
if cache:
if (pair, timeframe, c_type) in self._klines:
old = self._klines[(pair, timeframe, c_type)]
# Reassign so we return the updated, combined df
ohlcv_df = clean_ohlcv_dataframe(concat([old, ohlcv_df], axis=0), timeframe, pair,
fill_missing=True, drop_incomplete=False)
candle_limit = self.ohlcv_candle_limit(timeframe, self._config['candle_type_def'])
# Age out old candles
ohlcv_df = ohlcv_df.tail(candle_limit + self._startup_candle_count)
self._klines[(pair, timeframe, c_type)] = ohlcv_df
else:
self._klines[(pair, timeframe, c_type)] = ohlcv_df
return ohlcv_df
def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
since_ms: Optional[int] = None, cache: bool = True,
drop_incomplete: Optional[bool] = None
@@ -1880,27 +1952,9 @@ class Exchange:
"""
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete
input_coroutines = []
cached_pairs = []
# Gather coroutines to run
for pair, timeframe, candle_type in set(pair_list):
if (timeframe not in self.timeframes
and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
logger.warning(
f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
f"not available on {self.name}. Available timeframes are "
f"{', '.join(self.timeframes)}.")
continue
if ((pair, timeframe, candle_type) not in self._klines or not cache
or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
input_coroutines.append(self._build_coroutine(
pair, timeframe, candle_type=candle_type, since_ms=since_ms))
else:
logger.debug(
f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
)
cached_pairs.append((pair, timeframe, candle_type))
# Gather coroutines to run
input_coroutines, cached_pairs = self._build_ohlcv_dl_jobs(pair_list, since_ms, cache)
results_df = {}
# Chunk requests into batches of 100 to avoid overwelming ccxt Throttling
@@ -1917,16 +1971,11 @@ class Exchange:
continue
# Deconstruct tuple (has 4 elements)
pair, timeframe, c_type, ticks = res
# keeping last candle time as last refreshed time of the pair
if ticks:
self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
ohlcv_df = ohlcv_to_dataframe(
ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=drop_incomplete)
ohlcv_df = self._process_ohlcv_df(
pair, timeframe, c_type, ticks, cache, drop_incomplete)
results_df[(pair, timeframe, c_type)] = ohlcv_df
if cache:
self._klines[(pair, timeframe, c_type)] = ohlcv_df
# Return cached klines
for pair, timeframe, c_type in cached_pairs:
results_df[(pair, timeframe, c_type)] = self.klines(
@@ -1941,10 +1990,8 @@ class Exchange:
interval_in_sec = timeframe_to_seconds(timeframe)
return not (
(self._pairs_last_refresh_time.get(
(pair, timeframe, candle_type),
0
) + interval_in_sec) >= arrow.utcnow().int_timestamp
(self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0)
+ interval_in_sec) >= arrow.utcnow().int_timestamp
)
@retrier_async
@@ -2754,10 +2801,6 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non
return exchange_name in ccxt_exchanges(ccxt_module)
def is_exchange_officially_supported(exchange_name: str) -> bool:
return exchange_name in SUPPORTED_EXCHANGES
def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
"""
Return the list of all exchanges known to ccxt

View File

@@ -78,7 +78,8 @@ class Okx(Exchange):
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
f'Error in additional_exchange_init due to {e.__class__.__name__}. Message: {e}'
) from e
except ccxt.BaseError as e:
raise OperationalException(e) from e

View File

@@ -92,7 +92,7 @@ class BaseClassifierModel(IFreqaiModel):
filtered_df = dk.normalize_data_from_metadata(filtered_df)
dk.data_dictionary["prediction_features"] = filtered_df
self.data_cleaning_predict(dk, filtered_df)
self.data_cleaning_predict(dk)
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
pred_df = DataFrame(predictions, columns=dk.label_list)

View File

@@ -92,7 +92,7 @@ class BaseRegressionModel(IFreqaiModel):
dk.data_dictionary["prediction_features"] = filtered_df
# optional additional data cleaning/analysis
self.data_cleaning_predict(dk, filtered_df)
self.data_cleaning_predict(dk)
predictions = self.model.predict(dk.data_dictionary["prediction_features"])
pred_df = DataFrame(predictions, columns=dk.label_list)

View File

@@ -257,7 +257,7 @@ class FreqaiDataDrawer:
def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_],
dk: FreqaiDataKitchen, len_df: int) -> None:
dk: FreqaiDataKitchen, strat_df: DataFrame) -> None:
"""
Append model predictions to historic predictions dataframe, then set the
strategy return dataframe to the tail of the historic predictions. The length of
@@ -266,6 +266,7 @@ class FreqaiDataDrawer:
historic predictions.
"""
len_df = len(strat_df)
index = self.historic_predictions[pair].index[-1:]
columns = self.historic_predictions[pair].columns
@@ -293,6 +294,15 @@ class FreqaiDataDrawer:
for return_str in rets:
df[return_str].iloc[-1] = rets[return_str]
# this logic carries users between version without needing to
# change their identifier
if 'close_price' not in df.columns:
df['close_price'] = np.nan
df['date_pred'] = np.nan
df['close_price'].iloc[-1] = strat_df['close'].iloc[-1]
df['date_pred'].iloc[-1] = strat_df['date'].iloc[-1]
self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
def attach_return_values_to_return_dataframe(
@@ -313,6 +323,7 @@ class FreqaiDataDrawer:
"""
dk.find_features(dataframe)
dk.find_labels(dataframe)
full_labels = dk.label_list + dk.unique_class_list
@@ -376,7 +387,27 @@ class FreqaiDataDrawer:
if self.config.get("freqai", {}).get("purge_old_models", False):
self.purge_old_models()
# Functions pulled back from FreqaiDataKitchen because they relied on DataDrawer
def save_metadata(self, dk: FreqaiDataKitchen) -> None:
"""
Saves only metadata for backtesting studies if user prefers
not to save model data. This saves tremendous amounts of space
for users generating huge studies.
This is only active when `save_backtest_models`: false (not default)
"""
if not dk.data_path.is_dir():
dk.data_path.mkdir(parents=True, exist_ok=True)
save_path = Path(dk.data_path)
dk.data["data_path"] = str(dk.data_path)
dk.data["model_filename"] = str(dk.model_filename)
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
dk.data["label_list"] = dk.label_list
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
return
def save_data(self, model: Any, coin: str, dk: FreqaiDataKitchen) -> None:
"""
@@ -402,7 +433,7 @@ class FreqaiDataDrawer:
dk.data["data_path"] = str(dk.data_path)
dk.data["model_filename"] = str(dk.model_filename)
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
dk.data["training_features_list"] = dk.training_features_list
dk.data["label_list"] = dk.label_list
# store the metadata
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
@@ -586,7 +617,8 @@ class FreqaiDataDrawer:
"include_corr_pairlist", []
)
for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
base_dataframes[tf] = dk.slice_dataframe(timerange, historic_data[pair][tf])
base_dataframes[tf] = dk.slice_dataframe(
timerange, historic_data[pair][tf]).reset_index(drop=True)
if pairs:
for p in pairs:
if pair in p:
@@ -595,7 +627,7 @@ class FreqaiDataDrawer:
corr_dataframes[p] = {}
corr_dataframes[p][tf] = dk.slice_dataframe(
timerange, historic_data[p][tf]
)
).reset_index(drop=True)
return corr_dataframes, base_dataframes

View File

@@ -135,20 +135,15 @@ class FreqaiDataKitchen:
"""
feat_dict = self.freqai_config["feature_parameters"]
if 'shuffle' not in self.freqai_config['data_split_parameters']:
self.freqai_config["data_split_parameters"].update({'shuffle': False})
weights: npt.ArrayLike
if feat_dict.get("weight_factor", 0) > 0:
weights = self.set_weights_higher_recent(len(filtered_dataframe))
else:
weights = np.ones(len(filtered_dataframe))
if feat_dict.get("stratify_training_data", 0) > 0:
stratification = np.zeros(len(filtered_dataframe))
for i in range(1, len(stratification)):
if i % feat_dict.get("stratify_training_data", 0) == 0:
stratification[i] = 1
else:
stratification = None
if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0:
(
train_features,
@@ -161,7 +156,6 @@ class FreqaiDataKitchen:
filtered_dataframe[: filtered_dataframe.shape[0]],
labels,
weights,
stratify=stratification,
**self.config["freqai"]["data_split_parameters"],
)
else:
@@ -211,7 +205,7 @@ class FreqaiDataKitchen:
filtered_df = unfiltered_df.filter(training_feature_list, axis=1)
filtered_df = filtered_df.replace([np.inf, -np.inf], np.nan)
drop_index = pd.isnull(filtered_df).any(1) # get the rows that have NaNs,
drop_index = pd.isnull(filtered_df).any(axis=1) # get the rows that have NaNs,
drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement.
if (training_filter):
const_cols = list((filtered_df.nunique() == 1).loc[lambda x: x].index)
@@ -222,7 +216,7 @@ class FreqaiDataKitchen:
# about removing any row with NaNs
# if labels has multiple columns (user wants to train multiple modelEs), we detect here
labels = unfiltered_df.filter(label_list, axis=1)
drop_index_labels = pd.isnull(labels).any(1)
drop_index_labels = pd.isnull(labels).any(axis=1)
drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0)
dates = unfiltered_df['date']
filtered_df = filtered_df[
@@ -250,7 +244,7 @@ class FreqaiDataKitchen:
else:
# we are backtesting so we need to preserve row number to send back to strategy,
# so now we use do_predict to avoid any prediction based on a NaN
drop_index = pd.isnull(filtered_df).any(1)
drop_index = pd.isnull(filtered_df).any(axis=1)
self.data["filter_drop_index_prediction"] = drop_index
filtered_df.fillna(0, inplace=True)
# replacing all NaNs with zeros to avoid issues in 'prediction', but any prediction
@@ -809,7 +803,7 @@ class FreqaiDataKitchen:
:, :no_prev_pts
]
distances = distances.replace([np.inf, -np.inf], np.nan)
drop_index = pd.isnull(distances).any(1)
drop_index = pd.isnull(distances).any(axis=1)
distances = distances[drop_index == 0]
inliers = pd.DataFrame(index=distances.index)
@@ -832,7 +826,7 @@ class FreqaiDataKitchen:
inlier_metric = pd.DataFrame(
data=inliers.sum(axis=1) / no_prev_pts,
columns=['inlier_metric'],
columns=['%-inlier_metric'],
index=compute_df.index
)
@@ -882,11 +876,15 @@ class FreqaiDataKitchen:
"""
column_names = dataframe.columns
features = [c for c in column_names if "%" in c]
labels = [c for c in column_names if "&" in c]
if not features:
raise OperationalException("Could not find any features!")
self.training_features_list = features
def find_labels(self, dataframe: DataFrame) -> None:
column_names = dataframe.columns
labels = [c for c in column_names if "&" in c]
self.label_list = labels
def check_if_pred_in_training_spaces(self) -> None:
@@ -1207,7 +1205,8 @@ class FreqaiDataKitchen:
def get_unique_classes_from_labels(self, dataframe: DataFrame) -> None:
self.find_features(dataframe)
# self.find_features(dataframe)
self.find_labels(dataframe)
for key in self.label_list:
if dataframe[key].dtype == object:

View File

@@ -92,6 +92,7 @@ class IFreqaiModel(ABC):
self.begin_time_train: float = 0
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
self.continual_learning = self.freqai_info.get('continual_learning', False)
self.plot_features = self.ft_params.get("plot_feature_importances", 0)
self.spice_rack_open: bool = False
self._threads: List[threading.Thread] = []
self._stop_event = threading.Event()
@@ -210,7 +211,8 @@ class IFreqaiModel(ABC):
new_trained_timerange, pair, strategy, dk, data_load_timerange
)
except Exception as msg:
logger.warning(f'Training {pair} raised exception {msg}, skipping.')
logger.warning(f"Training {pair} raised exception {msg.__class__.__name__}. "
f"Message: {msg}, skipping.")
self.train_timer('stop')
@@ -274,26 +276,28 @@ class IFreqaiModel(ABC):
if dk.check_if_backtest_prediction_exists():
self.dd.load_metadata(dk)
self.check_if_feature_list_matches_strategy(dataframe_train, dk)
dk.find_features(dataframe_train)
self.check_if_feature_list_matches_strategy(dk)
append_df = dk.get_backtesting_prediction()
dk.append_predictions(append_df)
else:
if not self.model_exists(
pair, dk, trained_timestamp=trained_timestamp_int
):
if not self.model_exists(dk):
dk.find_features(dataframe_train)
dk.find_labels(dataframe_train)
self.model = self.train(dataframe_train, pair, dk)
self.dd.pair_dict[pair]["trained_timestamp"] = int(
trained_timestamp.stopts)
if self.plot_features:
plot_feature_importance(self.model, pair, dk, self.plot_features)
if self.save_backtest_models:
logger.info('Saving backtest model to disk.')
self.dd.save_data(self.model, pair, dk)
else:
logger.info('Saving metadata to disk.')
self.dd.save_metadata(dk)
else:
self.model = self.dd.load_data(pair, dk)
self.check_if_feature_list_matches_strategy(dataframe_train, dk)
pred_df, do_preds = self.predict(dataframe_backtest, dk)
append_df = dk.get_predictions_to_append(pred_df, do_preds)
dk.append_predictions(append_df)
@@ -372,8 +376,7 @@ class IFreqaiModel(ABC):
self.dd.return_null_values_to_strategy(dataframe, dk)
return dk
# ensure user is feeding the correct indicators to the model
self.check_if_feature_list_matches_strategy(dataframe, dk)
dk.find_labels(dataframe)
self.build_strategy_return_arrays(dataframe, dk, metadata["pair"], trained_timestamp)
@@ -391,7 +394,7 @@ class IFreqaiModel(ABC):
# allows FreqUI to show full return values.
pred_df, do_preds = self.predict(dataframe, dk)
if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(pred_df, dk, pair)
self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
self.dd.set_initial_return_values(pair, pred_df)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
@@ -412,13 +415,13 @@ class IFreqaiModel(ABC):
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
self.fit_live_predictions(dk, pair)
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, len(dataframe))
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, dataframe)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
return
def check_if_feature_list_matches_strategy(
self, dataframe: DataFrame, dk: FreqaiDataKitchen
self, dk: FreqaiDataKitchen
) -> None:
"""
Ensure user is passing the proper feature set if they are reusing an `identifier` pointing
@@ -427,11 +430,12 @@ class IFreqaiModel(ABC):
:param dk: FreqaiDataKitchen = non-persistent data container/analyzer for
current coin/bot loop
"""
dk.find_features(dataframe)
if "training_features_list_raw" in dk.data:
feature_list = dk.data["training_features_list_raw"]
else:
feature_list = dk.data['training_features_list']
if dk.training_features_list != feature_list:
raise OperationalException(
"Trying to access pretrained model with `identifier` "
@@ -479,20 +483,23 @@ class IFreqaiModel(ABC):
if self.freqai_info["feature_parameters"].get('noise_standard_deviation', 0):
dk.add_noise_to_training_features()
def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None:
def data_cleaning_predict(self, dk: FreqaiDataKitchen) -> None:
"""
Base data cleaning method for predict.
Functions here are complementary to the functions of data_cleaning_train.
"""
ft_params = self.freqai_info["feature_parameters"]
# ensure user is feeding the correct indicators to the model
self.check_if_feature_list_matches_strategy(dk)
if ft_params.get('inlier_metric_window', 0):
dk.compute_inlier_metric(set_='predict')
if ft_params.get(
"principal_component_analysis", False
):
dk.pca_transform(self.dk.data_dictionary['prediction_features'])
dk.pca_transform(dk.data_dictionary['prediction_features'])
if ft_params.get("use_SVM_to_remove_outliers", False):
dk.use_SVM_to_remove_outliers(predict=True)
@@ -503,14 +510,7 @@ class IFreqaiModel(ABC):
if ft_params.get("use_DBSCAN_to_remove_outliers", False):
dk.use_DBSCAN_to_remove_outliers(predict=True)
def model_exists(
self,
pair: str,
dk: FreqaiDataKitchen,
trained_timestamp: int = None,
model_filename: str = "",
scanning: bool = False,
) -> bool:
def model_exists(self, dk: FreqaiDataKitchen) -> bool:
"""
Given a pair and path, check if a model already exists
:param pair: pair e.g. BTC/USD
@@ -518,11 +518,11 @@ class IFreqaiModel(ABC):
:return:
:boolean: whether the model file exists or not.
"""
path_to_modelfile = Path(dk.data_path / f"{model_filename}_model.joblib")
path_to_modelfile = Path(dk.data_path / f"{dk.model_filename}_model.joblib")
file_exists = path_to_modelfile.is_file()
if file_exists and not scanning:
if file_exists:
logger.info("Found model at %s", dk.data_path / dk.model_filename)
elif not scanning:
else:
logger.info("Could not find model at %s", dk.data_path / dk.model_filename)
return file_exists
@@ -569,6 +569,7 @@ class IFreqaiModel(ABC):
# find the features indicated by strategy and store in datakitchen
dk.find_features(unfiltered_dataframe)
dk.find_labels(unfiltered_dataframe)
model = self.train(unfiltered_dataframe, pair, dk)
@@ -576,14 +577,14 @@ class IFreqaiModel(ABC):
dk.set_new_model_names(pair, new_trained_timerange)
self.dd.save_data(model, pair, dk)
if self.freqai_info["feature_parameters"].get("plot_feature_importance", False):
plot_feature_importance(model, pair, dk)
if self.plot_features:
plot_feature_importance(model, pair, dk, self.plot_features)
if self.freqai_info.get("purge_old_models", False):
self.dd.purge_old_models()
def set_initial_historic_predictions(
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame
) -> None:
"""
This function is called only if the datadrawer failed to load an
@@ -626,6 +627,9 @@ class IFreqaiModel(ABC):
for return_str in dk.data['extra_returns_per_train']:
hist_preds_df[return_str] = 0
hist_preds_df['close_price'] = strat_df['close']
hist_preds_df['date_pred'] = strat_df['date']
# # for keras type models, the conv_window needs to be prepended so
# # viewing is correct in frequi
if self.freqai_info.get('keras', False) or self.ft_params.get('inlier_metric_window', 0):

View File

@@ -306,7 +306,7 @@ def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
# Data preparation
fi_df = pd.DataFrame({
"feature_names": np.array(dk.training_features_list),
"feature_names": np.array(dk.data_dictionary['train_features'].columns),
"feature_importance": np.array(feature_importance)
})
fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1]

View File

@@ -82,7 +82,10 @@ class FreqtradeBot(LoggingMixin):
# Keep this at the end of this initialization method.
self.rpc: RPCManager = RPCManager(self)
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists, self.rpc)
self.dataprovider = DataProvider(self.config, self.exchange, rpc=self.rpc)
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
self.dataprovider.add_pairlisthandler(self.pairlists)
# Attach Dataprovider to strategy instance
self.strategy.dp = self.dataprovider
@@ -597,7 +600,7 @@ class FreqtradeBot(LoggingMixin):
# We should decrease our position
amount = self.exchange.amount_to_contract_precision(
trade.pair,
abs(float(FtPrecise(stake_amount) / FtPrecise(current_exit_rate))))
abs(float(FtPrecise(stake_amount * trade.leverage) / FtPrecise(current_exit_rate))))
if amount > trade.amount:
# This is currently ineffective as remaining would become < min tradable
# Fixing this would require checking for 0.0 there -
@@ -1308,7 +1311,7 @@ class FreqtradeBot(LoggingMixin):
# place new order only if new price is supplied
self.execute_entry(
pair=trade.pair,
stake_amount=(order_obj.remaining * order_obj.price),
stake_amount=(order_obj.remaining * order_obj.price / trade.leverage),
price=adjusted_entry_price,
trade=trade,
is_short=trade.is_short,
@@ -1340,11 +1343,12 @@ class FreqtradeBot(LoggingMixin):
replacing: Optional[bool] = False
) -> bool:
"""
Buy cancel - cancel order
entry cancel - cancel order
:param replacing: Replacing order - prevent trade deletion.
:return: True if order was fully cancelled
:return: True if trade was fully cancelled
"""
was_trade_fully_canceled = False
side = trade.entry_side.capitalize()
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
@@ -1371,7 +1375,6 @@ class FreqtradeBot(LoggingMixin):
corder = order
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
side = trade.entry_side.capitalize()
logger.info('%s order %s for %s.', side, reason, trade)
# Using filled to determine the filled amount
@@ -1385,24 +1388,15 @@ class FreqtradeBot(LoggingMixin):
was_trade_fully_canceled = True
reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
else:
# FIXME TODO: This could possibly reworked to not duplicate the code 15 lines below.
self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None
logger.info(f'{side} Order timeout for {trade}.')
else:
# if trade is partially complete, edit the stake details for the trade
# and close the order
# cancel_order may not contain the full order dict, so we need to fallback
# to the order dict acquired before cancelling.
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = filled_amount
# * Check edge cases, we don't want to make leverage > 1.0 if we don't have to
# * (for leverage modes which aren't isolated futures)
trade.stake_amount = trade.amount * trade.open_rate / trade.leverage
# update_trade_state (and subsequently recalc_trade_from_orders) will handle updates
# to the trade object
self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None
logger.info(f'Partial {trade.entry_side} order timeout for {trade}.')
reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
@@ -1417,49 +1411,63 @@ class FreqtradeBot(LoggingMixin):
:return: True if exit order was cancelled, false otherwise
"""
cancelled = False
# if trade is not partially completed, just cancel the order
if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
if not self.exchange.check_order_canceled_empty(order):
try:
# if trade is not partially completed, just delete the order
co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
trade.update_order(co)
except InvalidOrderException:
logger.exception(
f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
return False
logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
else:
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
trade.update_order(order)
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
filled_val: float = order.get('filled', 0.0) or 0.0
filled_rem_stake = trade.stake_amount - filled_val * trade.open_rate
minstake = self.exchange.get_min_pair_stake_amount(
trade.pair, trade.open_rate, self.strategy.stoploss)
# Double-check remaining amount
if filled_val > 0:
reason = constants.CANCEL_REASON['PARTIALLY_FILLED']
if minstake and filled_rem_stake < minstake:
logger.warning(
f"Order {trade.open_order_id} for {trade.pair} not cancelled, as "
f"the filled amount of {filled_val} would result in an unexitable trade.")
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
self._notify_exit_cancel(
trade,
order_type=self.strategy.order_types['exit'],
reason=reason, order_id=order['id'],
sub_trade=trade.amount != order['amount']
)
return False
try:
co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
except InvalidOrderException:
logger.exception(
f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
return False
trade.close_rate = None
trade.close_rate_requested = None
trade.close_profit = None
trade.close_profit_abs = None
trade.close_date = None
trade.is_open = True
trade.open_order_id = None
trade.exit_reason = None
# Set exit_reason for fill message
exit_reason_prev = trade.exit_reason
trade.exit_reason = trade.exit_reason + f", {reason}" if trade.exit_reason else reason
self.update_trade_state(trade, trade.open_order_id, co)
# Order might be filled above in odd timing issues.
if co.get('status') in ('canceled', 'cancelled'):
trade.exit_reason = None
trade.open_order_id = None
else:
trade.exit_reason = exit_reason_prev
logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
cancelled = True
self.wallets.update()
else:
# TODO: figure out how to handle partially complete sell orders
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
cancelled = False
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
self.update_trade_state(trade, trade.open_order_id, order)
trade.open_order_id = None
order_obj = trade.select_order_by_order_id(order['id'])
if not order_obj:
raise DependencyException(
f"Order_obj not found for {order['id']}. This should not have happened.")
sub_trade = order_obj.amount != trade.amount
self._notify_exit_cancel(
trade,
order_type=self.strategy.order_types['exit'],
reason=reason, order=order_obj, sub_trade=sub_trade
reason=reason, order_id=order['id'], sub_trade=trade.amount != order['amount']
)
return cancelled
@@ -1656,7 +1664,7 @@ class FreqtradeBot(LoggingMixin):
self.rpc.send_msg(msg)
def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str,
order: Order, sub_trade: bool = False) -> None:
order_id: str, sub_trade: bool = False) -> None:
"""
Sends rpc notification when a sell cancel occurred.
"""
@@ -1665,6 +1673,11 @@ class FreqtradeBot(LoggingMixin):
else:
trade.exit_order_status = reason
order = trade.select_order_by_order_id(order_id)
if not order:
raise DependencyException(
f"Order_obj not found for {order_id}. This should not have happened.")
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
current_rate = self.exchange.get_rate(
@@ -1700,11 +1713,6 @@ class FreqtradeBot(LoggingMixin):
'stake_amount': trade.stake_amount,
}
if 'fiat_display_currency' in self.config:
msg.update({
'fiat_currency': self.config['fiat_display_currency'],
})
# Send the message
self.rpc.send_msg(msg)

View File

@@ -114,10 +114,10 @@ class Backtesting:
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.init_backtest_detail()
self.pairlists = PairListManager(self.exchange, self.config)
self.pairlists = PairListManager(self.exchange, self.config, self.dataprovider)
if 'VolumePairList' in self.pairlists.name_list:
raise OperationalException("VolumePairList not allowed for backtesting. "
"Please use StaticPairlist instead.")
"Please use StaticPairList instead.")
if 'PerformanceFilter' in self.pairlists.name_list:
raise OperationalException("PerformanceFilter not allowed for backtesting.")
@@ -158,9 +158,6 @@ class Backtesting:
self.init_backtest()
def __del__(self):
self.cleanup()
@staticmethod
def cleanup():
LoggingMixin.show_output = True
@@ -377,10 +374,10 @@ class Backtesting:
for col in HEADERS[5:]:
tag_col = col in ('enter_tag', 'exit_tag')
if col in df_analyzed.columns:
df_analyzed.loc[:, col] = df_analyzed.loc[:, col].replace(
df_analyzed[col] = df_analyzed.loc[:, col].replace(
[nan], [0 if not tag_col else None]).shift(1)
elif not df_analyzed.empty:
df_analyzed.loc[:, col] = 0 if not tag_col else None
df_analyzed[col] = 0 if not tag_col else None
df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
@@ -547,7 +544,7 @@ class Backtesting:
if stake_amount is not None and stake_amount < 0.0:
amount = amount_to_contract_precision(
abs(stake_amount) / current_rate, trade.amount_precision,
abs(stake_amount * trade.leverage) / current_rate, trade.amount_precision,
self.precision_mode, trade.contract_size)
if amount == 0.0:
return trade
@@ -1052,7 +1049,7 @@ class Backtesting:
if requested_rate:
self._enter_trade(pair=trade.pair, row=row, trade=trade,
requested_rate=requested_rate,
requested_stake=(order.remaining * order.price),
requested_stake=(order.remaining * order.price / trade.leverage),
direction='short' if trade.is_short else 'long')
self.replaced_entry_orders += 1
else:

View File

@@ -24,6 +24,7 @@ from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.data.metrics import calculate_market_change
from freqtrade.enums import HyperoptState
from freqtrade.exceptions import OperationalException
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
@@ -61,7 +62,7 @@ class Hyperopt:
"""
Hyperopt class, this class contains all the logic to run a hyperopt simulation
To run a backtest:
To start a hyperopt run:
hyperopt = Hyperopt(config)
hyperopt.start()
"""
@@ -111,6 +112,7 @@ class Hyperopt:
self.clean_hyperopt()
self.market_change = 0.0
self.num_epochs_saved = 0
self.current_best_epoch: Optional[Dict[str, Any]] = None
@@ -357,7 +359,7 @@ class Hyperopt:
strat_stats = generate_strategy_stats(
self.pairlist, self.backtesting.strategy.get_strategy_name(),
backtesting_results, min_date, max_date, market_change=0
backtesting_results, min_date, max_date, market_change=self.market_change
)
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
@@ -425,6 +427,9 @@ class Hyperopt:
# Trim startup period from analyzed dataframe to get correct dates for output.
trimmed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup)
self.min_date, self.max_date = get_timerange(trimmed)
if not self.market_change:
self.market_change = calculate_market_change(trimmed, 'close')
# Real trimming will happen as part of backtesting.
return preprocessed

View File

@@ -173,7 +173,7 @@ def generate_tag_metrics(tag_type: str,
tabular_data = []
if tag_type in results.columns:
for tag, count in results[tag_type].value_counts().iteritems():
for tag, count in results[tag_type].value_counts().items():
result = results[results[tag_type] == tag]
if skip_nan and result['profit_abs'].isnull().all():
continue
@@ -199,7 +199,7 @@ def generate_exit_reason_stats(max_open_trades: int, results: DataFrame) -> List
"""
tabular_data = []
for reason, count in results['exit_reason'].value_counts().iteritems():
for reason, count in results['exit_reason'].value_counts().items():
result = results.loc[results['exit_reason'] == reason]
profit_mean = result['profit_ratio'].mean()
@@ -361,7 +361,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
winning_days = sum(daily_profit > 0)
draw_days = sum(daily_profit == 0)
losing_days = sum(daily_profit < 0)
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.items()]
return {
'backtest_best_day': best_rel,

View File

@@ -0,0 +1,90 @@
"""
External Pair List provider
Provides pair list from Leader data
"""
import logging
from typing import Any, Dict, List, Optional
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class ProducerPairList(IPairList):
"""
PairList plugin for use with external_message_consumer.
Will use pairs given from leader data.
Usage:
"pairlists": [
{
"method": "ProducerPairList",
"number_assets": 5,
"producer_name": "default",
}
],
"""
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._num_assets: int = self._pairlistconfig.get('number_assets', 0)
self._producer_name = self._pairlistconfig.get('producer_name', 'default')
if not config.get('external_message_consumer', {}).get('enabled'):
raise OperationalException(
"ProducerPairList requires external_message_consumer to be enabled.")
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return False
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
-> Please overwrite in subclasses
"""
return f"{self.name} - {self._producer_name}"
def _filter_pairlist(self, pairlist: Optional[List[str]]):
upstream_pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(
self._producer_name)
if pairlist is None:
pairlist = self._pairlistmanager._dataprovider.get_producer_pairs(self._producer_name)
pairs = list(dict.fromkeys(pairlist + upstream_pairlist))
if self._num_assets:
pairs = pairs[:self._num_assets]
return pairs
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""
pairs = self._filter_pairlist(None)
self.log_once(f"Received pairs: {pairs}", logger.debug)
pairs = self._whitelist_for_active_markets(self.verify_whitelist(pairs, logger.info))
return pairs
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
return self._filter_pairlist(pairlist)

View File

@@ -232,6 +232,4 @@ class VolumePairList(IPairList):
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]
self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info)
return pairs

View File

@@ -3,11 +3,12 @@ PairList manager class
"""
import logging
from functools import partial
from typing import Dict, List
from typing import Dict, List, Optional
from cachetools import TTLCache, cached
from freqtrade.constants import Config, ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.mixins import LoggingMixin
@@ -21,13 +22,14 @@ logger = logging.getLogger(__name__)
class PairListManager(LoggingMixin):
def __init__(self, exchange, config: Config) -> None:
def __init__(self, exchange, config: Config, dataprovider: DataProvider = None) -> None:
self._exchange = exchange
self._config = config
self._whitelist = self._config['exchange'].get('pair_whitelist')
self._blacklist = self._config['exchange'].get('pair_blacklist', [])
self._pairlist_handlers: List[IPairList] = []
self._tickers_needed = False
self._dataprovider: Optional[DataProvider] = dataprovider
for pairlist_handler_config in self._config.get('pairlists', []):
pairlist_handler = PairListResolver.load_pairlist(
pairlist_handler_config['method'],
@@ -96,6 +98,8 @@ class PairListManager(LoggingMixin):
# to ensure blacklist is respected.
pairlist = self.verify_blacklist(pairlist, logger.warning)
self.log_once(f"Whitelist with {len(pairlist)} pairs: {pairlist}", logger.info)
self._whitelist = pairlist
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:

View File

@@ -5,6 +5,7 @@ from datetime import datetime
from typing import Any, Dict, List
from fastapi import APIRouter, BackgroundTasks, Depends
from fastapi.exceptions import HTTPException
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
@@ -31,6 +32,9 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
if ApiServer._bgtask_running:
raise RPCException('Bot Background task already running')
if ':' in bt_settings.strategy:
raise HTTPException(status_code=500, detail="base64 encoded strategies are not allowed.")
btconfig = deepcopy(config)
settings = dict(bt_settings)
# Pydantic models will contain all keys, but non-provided ones are None

View File

@@ -265,6 +265,8 @@ def list_strategies(config=Depends(get_config)):
@router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
def get_strategy(strategy: str, config=Depends(get_config)):
if ":" in strategy:
raise HTTPException(status_code=500, detail="base64 encoded strategies are not allowed.")
config_ = deepcopy(config)
from freqtrade.resolvers.strategy_resolver import StrategyResolver

View File

@@ -198,8 +198,10 @@ class ApiServer(RPCHandler):
logger.debug(f"Found message of type: {message.get('type')}")
# Broadcast it
await self._ws_channel_manager.broadcast(message)
# Sleep, make this configurable?
await asyncio.sleep(0.1)
# Limit messages per sec.
# Could cause problems with queue size if too low, and
# problems with network traffik if too high.
await asyncio.sleep(0.001)
except asyncio.CancelledError:
pass

View File

@@ -140,7 +140,7 @@ class ChannelManager:
Disconnect all Channels
"""
with self._lock:
for websocket, channel in self.channels.items():
for websocket, channel in self.channels.copy().items():
if not channel.is_closed():
await channel.close()
@@ -154,7 +154,7 @@ class ChannelManager:
"""
with self._lock:
message_type = data.get('type')
for websocket, channel in self.channels.items():
for websocket, channel in self.channels.copy().items():
try:
if channel.subscribed_to(message_type):
await channel.send(data)

View File

@@ -30,9 +30,9 @@ class Discord(Webhook):
pass
def send_msg(self, msg) -> None:
logger.info(f"Sending discord message: {msg}")
if msg['type'].value in self.config['discord']:
logger.info(f"Sending discord message: {msg}")
msg['strategy'] = self.strategy
msg['timeframe'] = self.timeframe

View File

@@ -284,7 +284,7 @@ class ExternalMessageConsumer:
logger.error(f"Empty message received from `{producer_name}`")
return
logger.info(f"Received message of type `{producer_message.type}` from `{producer_name}`")
logger.debug(f"Received message of type `{producer_message.type}` from `{producer_name}`")
message_handler = self._message_handlers.get(producer_message.type)

View File

@@ -3,8 +3,8 @@ Module that define classes to convert Crypto-currency to FIAT
e.g BTC to USD
"""
import datetime
import logging
from datetime import datetime
from typing import Dict, List
from cachetools import TTLCache
@@ -46,7 +46,9 @@ class CryptoToFiatConverter(LoggingMixin):
if CryptoToFiatConverter.__instance is None:
CryptoToFiatConverter.__instance = object.__new__(cls)
try:
CryptoToFiatConverter._coingekko = CoinGeckoAPI()
# Limit retires to 1 (0 and 1)
# otherwise we risk bot impact if coingecko is down.
CryptoToFiatConverter._coingekko = CoinGeckoAPI(retries=1)
except BaseException:
CryptoToFiatConverter._coingekko = None
return CryptoToFiatConverter.__instance
@@ -67,7 +69,7 @@ class CryptoToFiatConverter(LoggingMixin):
logger.warning(
"Too many requests for CoinGecko API, backing off and trying again later.")
# Set backoff timestamp to 60 seconds in the future
self._backoff = datetime.datetime.now().timestamp() + 60
self._backoff = datetime.now().timestamp() + 60
return
# If the request is not a 429 error we want to raise the normal error
logger.error(
@@ -81,7 +83,7 @@ class CryptoToFiatConverter(LoggingMixin):
def _get_gekko_id(self, crypto_symbol):
if not self._coinlistings:
if self._backoff <= datetime.datetime.now().timestamp():
if self._backoff <= datetime.now().timestamp():
self._load_cryptomap()
# Still not loaded.
if not self._coinlistings:

View File

@@ -25,7 +25,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
from freqtrade.misc import decimals_per_coin, shorten_date
from freqtrade.persistence import PairLocks, Trade
from freqtrade.persistence import Order, PairLocks, Trade
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -166,9 +166,9 @@ class RPC:
else:
results = []
for trade in trades:
order = None
order: Optional[Order] = None
if trade.open_order_id:
order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
order = trade.select_order_by_order_id(trade.open_order_id)
# calculate profit and send message to user
if trade.is_open:
try:
@@ -219,7 +219,7 @@ class RPC:
stoploss_entry_dist=stoploss_entry_dist,
stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
open_order='({} {} rem={:.8f})'.format(
order['type'], order['side'], order['remaining']
order.order_type, order.side, order.remaining
) if order else None,
))
results.append(trade_dict)
@@ -773,6 +773,9 @@ class RPC:
is_short = trade.is_short
if not self._freqtrade.strategy.position_adjustment_enable:
raise RPCException(f'position for {pair} already open - id: {trade.id}')
else:
if Trade.get_open_trade_count() >= self._config['max_open_trades']:
raise RPCException("Maximum number of trades is reached.")
if not stake_amount:
# gen stake amount

View File

@@ -67,7 +67,7 @@ class RPCManager:
'status': 'stopping bot'
}
"""
if msg.get('type') is not RPCMessageType.ANALYZED_DF:
if msg.get('type') not in (RPCMessageType.ANALYZED_DF, RPCMessageType.WHITELIST):
logger.info('Sending rpc message: %s', msg)
if 'pair' in msg:
msg.update({

View File

@@ -61,6 +61,14 @@ class Webhook(RPCHandler):
RPCMessageType.STARTUP,
RPCMessageType.WARNING):
valuedict = whconfig.get('webhookstatus')
elif msg['type'] in (
RPCMessageType.PROTECTION_TRIGGER,
RPCMessageType.PROTECTION_TRIGGER_GLOBAL,
RPCMessageType.WHITELIST,
RPCMessageType.ANALYZED_DF,
RPCMessageType.STRATEGY_MSG):
# Don't fail for non-implemented types
return
else:
raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
if not valuedict:

View File

@@ -5,6 +5,7 @@
import numpy as np # noqa
import pandas as pd # noqa
from pandas import DataFrame
from typing import Optional, Union
from freqtrade.strategy import (BooleanParameter, CategoricalParameter, DecimalParameter,
IStrategy, IntParameter)