adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4 from freqtrade/feat/freqai

Browse Source 
Feat/freqai
This commit is contained in:
lolong
2022-08-13 10:48:07 +02:00
committed by GitHub
parent 263df87313 1ac6ec1446
commit 8f9e6efc5e
96 changed files with 3492 additions and 1676 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/__init__.py
View File

@@ -1,5 +1,5 @@
""" Freqtrade bot """
__version__ = 'develop'
__version__ = '2022.8.dev'
if 'dev' in __version__:
try:

6
freqtrade/commands/build_config_commands.py
View File

@@ -67,7 +67,7 @@ def ask_user_config() -> Dict[str, Any]:
"type": "text",
"name": "stake_amount",
"message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
"default": "100",
"default": "unlimited",
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
if val == UNLIMITED_STAKE_AMOUNT
@@ -164,7 +164,7 @@ def ask_user_config() -> Dict[str, Any]:
"when": lambda x: x['telegram']
},
{
"type": "text",
"type": "password",
"name": "telegram_chat_id",
"message": "Insert Telegram chat id",
"when": lambda x: x['telegram']
@@ -191,7 +191,7 @@ def ask_user_config() -> Dict[str, Any]:
"when": lambda x: x['api_server']
},
{
"type": "text",
"type": "password",
"name": "api_server_password",
"message": "Insert api-server password",
"when": lambda x: x['api_server']

1
freqtrade/configuration/__init__.py
View File

@@ -4,5 +4,4 @@ from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.PeriodicCache import PeriodicCache
from freqtrade.configuration.timerange import TimeRange

17
freqtrade/configuration/config_validation.py
View File

@@ -85,7 +85,6 @@ def validate_config_consistency(conf: Dict[str, Any], preliminary: bool = False)
_validate_unlimited_amount(conf)
_validate_ask_orderbook(conf)
validate_migrated_strategy_settings(conf)
_validate_freqai(conf)
# validate configuration before returning
logger.info('Validating configuration ...')
@@ -164,22 +163,6 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
)
def _validate_freqai(conf: Dict[str, Any]) -> None:
"""
Freqai param validator
"""
if not conf.get('freqai', {}):
return
for param in constants.SCHEMA_FREQAI_REQUIRED:
if param not in conf.get('freqai', {}):
if param not in conf.get('freqai', {}).get('feature_parameters', {}):
raise OperationalException(
f'{param} not found in Freqai config'
)
def _validate_whitelist(conf: Dict[str, Any]) -> None:
"""
Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does.

50
freqtrade/constants.py
View File

@@ -241,6 +241,7 @@ CONF_SCHEMA = {
},
'exchange': {'$ref': '#/definitions/exchange'},
'edge': {'$ref': '#/definitions/edge'},
'freqai': {'$ref': '#/definitions/freqai'},
'experimental': {
'type': 'object',
'properties': {
@@ -318,6 +319,10 @@ CONF_SCHEMA = {
'type': 'string',
'enum': ['off', 'ohlc'],
},
'strategy_msg': {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
},
}
},
'reload': {'type': 'boolean'},
@@ -481,21 +486,31 @@ CONF_SCHEMA = {
"freqai": {
"type": "object",
"properties": {
"enabled": {"type": "boolean", "default": False},
"keras": {"type": "boolean", "default": False},
"conv_width": {"type": "integer", "default": 2},
"train_period_days": {"type": "integer", "default": 0},
"backtest_period_days": {"type": "float", "default": 7},
"identifier": {"type": "str", "default": "example"},
"backtest_period_days": {"type": "number", "default": 7},
"identifier": {"type": "string", "default": "example"},
"feature_parameters": {
"type": "object",
"properties": {
"include_corr_pairlist": {"type": "list"},
"include_timeframes": {"type": "list"},
"include_corr_pairlist": {"type": "array"},
"include_timeframes": {"type": "array"},
"label_period_candles": {"type": "integer"},
"include_shifted_candles": {"type": "integer", "default": 0},
"DI_threshold": {"type": "float", "default": 0},
"DI_threshold": {"type": "number", "default": 0},
"weight_factor": {"type": "number", "default": 0},
"principal_component_analysis": {"type": "boolean", "default": False},
"use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
"svm_params": {"type": "object",
"properties": {
"shuffle": {"type": "boolean", "default": False},
"nu": {"type": "number", "default": 0.1}
},
}
},
"required": ["include_timeframes", "include_corr_pairlist", ]
},
"data_split_parameters": {
"type": "object",
@@ -507,13 +522,19 @@ CONF_SCHEMA = {
"model_training_parameters": {
"type": "object",
"properties": {
"n_estimators": {"type": "integer", "default": 2000},
"random_state": {"type": "integer", "default": 1},
"learning_rate": {"type": "number", "default": 0.02},
"task_type": {"type": "string", "default": "CPU"},
"n_estimators": {"type": "integer", "default": 1000}
},
},
},
"required": [
"enabled",
"train_period_days",
"backtest_period_days",
"identifier",
"feature_parameters",
"data_split_parameters",
"model_training_parameters"
]
},
},
}
@@ -558,17 +579,6 @@ SCHEMA_MINIMAL_REQUIRED = [
'dataformat_trades',
]
SCHEMA_FREQAI_REQUIRED = [
'include_timeframes',
'train_period_days',
'backtest_period_days',
'identifier',
'include_corr_pairlist',
'feature_parameters',
'data_split_parameters',
'model_training_parameters'
]
CANCEL_REASON = {
"TIMEOUT": "cancelled due to timeout",
"PARTIALLY_FILLED_KEEP_OPEN": "partially filled - keeping order open",

23
freqtrade/data/dataprovider.py
View File

@@ -5,6 +5,7 @@ including ticker and orderbook data, live and historical candle (OHLCV) data
Common Interface for bot and strategy to access data.
"""
import logging
from collections import deque
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Tuple
@@ -16,6 +17,7 @@ from freqtrade.data.history import load_pair_history
from freqtrade.enums import CandleType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange, timeframe_to_seconds
from freqtrade.util import PeriodicCache
logger = logging.getLogger(__name__)
@@ -33,6 +35,10 @@ class DataProvider:
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
self._msg_queue: deque = deque()
self.__msg_cache = PeriodicCache(
maxsize=1000, ttl=timeframe_to_seconds(self._config.get('timeframe', '1h')))
def _set_dataframe_max_index(self, limit_index: int):
"""
@@ -265,3 +271,20 @@ class DataProvider:
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
return self._exchange.fetch_l2_order_book(pair, maximum)
def send_msg(self, message: str, *, always_send: bool = False) -> None:
"""
Send custom RPC Notifications from your bot.
Will not send any bot in modes other than Dry-run or Live.
:param message: Message to be sent. Must be below 4096.
:param always_send: If False, will send the message only once per candle, and surpress
identical messages.
Careful as this can end up spaming your chat.
Defaults to False
"""
if self.runmode not in (RunMode.DRY_RUN, RunMode.LIVE):
return
if always_send or message not in self.__msg_cache:
self._msg_queue.append(message)
self.__msg_cache[message] = True

2
freqtrade/enums/exittype.py
View File

@@ -9,10 +9,12 @@ class ExitType(Enum):
STOP_LOSS = "stop_loss"
STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange"
TRAILING_STOP_LOSS = "trailing_stop_loss"
LIQUIDATION = "liquidation"
EXIT_SIGNAL = "exit_signal"
FORCE_EXIT = "force_exit"
EMERGENCY_EXIT = "emergency_exit"
CUSTOM_EXIT = "custom_exit"
PARTIAL_EXIT = "partial_exit"
NONE = ""
def __str__(self):

2
freqtrade/enums/rpcmessagetype.py
View File

@@ -17,6 +17,8 @@ class RPCMessageType(Enum):
PROTECTION_TRIGGER = 'protection_trigger'
PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global'
STRATEGY_MSG = 'strategy_msg'
def __repr__(self):
return self.value

87
freqtrade/exchange/exchange.py
View File

@@ -16,7 +16,7 @@ import arrow
import ccxt
import ccxt.async_support as ccxt_async
from cachetools import TTLCache
from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, Precise, decimal_to_precision
from ccxt import ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision
from pandas import DataFrame
from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, BuySell,
@@ -32,6 +32,7 @@ from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGE
retrier_async)
from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.util import FtPrecise
CcxtModuleType = Any
@@ -708,10 +709,10 @@ class Exchange:
# counting_mode=self.precisionMode,
# ))
if self.precisionMode == TICK_SIZE:
precision = Precise(str(self.markets[pair]['precision']['price']))
price_str = Precise(str(price))
precision = FtPrecise(self.markets[pair]['precision']['price'])
price_str = FtPrecise(price)
missing = price_str % precision
if not missing == Precise("0"):
if not missing == FtPrecise("0"):
price = round(float(str(price_str - missing + precision)), 14)
else:
symbol_prec = self.markets[pair]['precision']['price']
@@ -849,6 +850,7 @@ class Exchange:
dry_order.update({
'average': average,
'filled': _amount,
'remaining': 0.0,
'cost': (dry_order['amount'] * average) / leverage
})
# market orders will always incurr taker fees
@@ -1332,11 +1334,19 @@ class Exchange:
raise OperationalException(e) from e
@retrier
def fetch_positions(self) -> List[Dict]:
def fetch_positions(self, pair: str = None) -> List[Dict]:
"""
Fetch positions from the exchange.
If no pair is given, all positions are returned.
:param pair: Pair for the query
"""
if self._config['dry_run'] or self.trading_mode != TradingMode.FUTURES:
return []
try:
positions: List[Dict] = self._api.fetch_positions()
symbols = []
if pair:
symbols.append(pair)
positions: List[Dict] = self._api.fetch_positions(symbols)
self._log_exchange_response('fetch_positions', positions)
return positions
except ccxt.DDoSProtection as e:
@@ -1499,7 +1509,8 @@ class Exchange:
return price_side
def get_rate(self, pair: str, refresh: bool,
side: EntryExit, is_short: bool) -> float:
side: EntryExit, is_short: bool,
order_book: Optional[dict] = None, ticker: Optional[dict] = None) -> float:
"""
Calculates bid/ask target
bid rate - between current ask price and last price
@@ -1531,22 +1542,24 @@ class Exchange:
if conf_strategy.get('use_order_book', False):
order_book_top = conf_strategy.get('order_book_top', 1)
order_book = self.fetch_l2_order_book(pair, order_book_top)
if order_book is None:
order_book = self.fetch_l2_order_book(pair, order_book_top)
logger.debug('order_book %s', order_book)
# top 1 = index 0
try:
rate = order_book[f"{price_side}s"][order_book_top - 1][0]
except (IndexError, KeyError) as e:
logger.warning(
f"{name} Price at location {order_book_top} from orderbook could not be "
f"determined. Orderbook: {order_book}"
f"{pair} - {name} Price at location {order_book_top} from orderbook "
f"could not be determined. Orderbook: {order_book}"
)
raise PricingError from e
logger.debug(f"{name} price from orderbook {price_side_word}"
logger.debug(f"{pair} - {name} price from orderbook {price_side_word}"
f"side - top {order_book_top} order book {side} rate {rate:.8f}")
else:
logger.debug(f"Using Last {price_side_word} / Last Price")
ticker = self.fetch_ticker(pair)
if ticker is None:
ticker = self.fetch_ticker(pair)
ticker_rate = ticker[price_side]
if ticker['last'] and ticker_rate:
if side == 'entry' and ticker_rate > ticker['last']:
@@ -1563,6 +1576,33 @@ class Exchange:
return rate
def get_rates(self, pair: str, refresh: bool, is_short: bool) -> Tuple[float, float]:
entry_rate = None
exit_rate = None
if not refresh:
entry_rate = self._entry_rate_cache.get(pair)
exit_rate = self._exit_rate_cache.get(pair)
if entry_rate:
logger.debug(f"Using cached buy rate for {pair}.")
if exit_rate:
logger.debug(f"Using cached sell rate for {pair}.")
entry_pricing = self._config.get('entry_pricing', {})
exit_pricing = self._config.get('exit_pricing', {})
order_book = ticker = None
if not entry_rate and entry_pricing.get('use_order_book', False):
order_book_top = max(entry_pricing.get('order_book_top', 1),
exit_pricing.get('order_book_top', 1))
order_book = self.fetch_l2_order_book(pair, order_book_top)
entry_rate = self.get_rate(pair, refresh, 'entry', is_short, order_book=order_book)
elif not entry_rate:
ticker = self.fetch_ticker(pair)
entry_rate = self.get_rate(pair, refresh, 'entry', is_short, ticker=ticker)
if not exit_rate:
exit_rate = self.get_rate(pair, refresh, 'exit',
is_short, order_book=order_book, ticker=ticker)
return entry_rate, exit_rate
# Fee handling
@retrier
@@ -1981,7 +2021,7 @@ class Exchange:
else:
logger.debug(
"Fetching trades for pair %s, since %s %s...",
pair, since,
pair, since,
'(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else ''
)
trades = await self._api_async.fetch_trades(pair, since=since, limit=1000)
@@ -2539,7 +2579,6 @@ class Exchange:
else:
return 0.0
@retrier
def get_or_calculate_liquidation_price(
self,
pair: str,
@@ -2573,20 +2612,12 @@ class Exchange:
upnl_ex_1=upnl_ex_1
)
else:
try:
positions = self._api.fetch_positions([pair])
if len(positions) > 0:
pos = positions[0]
isolated_liq = pos['liquidationPrice']
else:
return None
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not set margin mode due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
positions = self.fetch_positions(pair)
if len(positions) > 0:
pos = positions[0]
isolated_liq = pos['liquidationPrice']
else:
return None
if isolated_liq:
buffer_amount = abs(open_rate - isolated_liq) * self.liquidation_buffer

14
freqtrade/exchange/ftx.py
View File

@@ -1,6 +1,6 @@
""" FTX exchange subclass """
import logging
from typing import Any, Dict, List, Tuple
from typing import Any, Dict, List, Optional, Tuple
import ccxt
@@ -116,9 +116,17 @@ class Ftx(Exchange):
if len(order) == 1:
if order[0].get('status') == 'closed':
# Trigger order was triggered ...
real_order_id = order[0].get('info', {}).get('orderId')
real_order_id: Optional[str] = order[0].get('info', {}).get('orderId')
# OrderId may be None for stoploss-market orders
# But contains "average" in these cases.
# So we need to get it through the endpoint
# /conditional_orders/{conditional_order_id}/triggers
if not real_order_id:
res = self._api.privateGetConditionalOrdersConditionalOrderIdTriggers(
params={'conditional_order_id': order_id})
self._log_exchange_response('fetch_stoploss_order2', res)
real_order_id = res['result'][0]['orderId'] if res.get(
'result', []) else None
if real_order_id:
order1 = self._api.fetch_order(real_order_id, pair)
self._log_exchange_response('fetch_stoploss_order1', order1)

178
freqtrade/freqai/data_drawer.py
View File

@@ -5,13 +5,14 @@ import re
import shutil
import threading
from pathlib import Path
from typing import Any, Dict, Tuple
from typing import Any, Dict, Tuple, TypedDict
import numpy as np
import pandas as pd
import rapidjson
from joblib import dump, load
from joblib.externals import cloudpickle
from numpy.typing import ArrayLike
from numpy.typing import ArrayLike, NDArray
from pandas import DataFrame
from freqtrade.configuration import TimeRange
@@ -24,6 +25,15 @@ from freqtrade.strategy.interface import IStrategy
logger = logging.getLogger(__name__)
class pair_info(TypedDict):
model_filename: str
first: bool
trained_timestamp: int
priority: int
data_path: str
extras: dict
class FreqaiDataDrawer:
"""
Class aimed at holding all pair models/info in memory for better inferencing/retrainig/saving
@@ -39,7 +49,7 @@ class FreqaiDataDrawer:
Robert Caulk @robcaulk
Theoretical brainstorming:
Elin Törnquist @thorntwig
Elin Törnquist @th0rntwig
Code review, software architecture brainstorming:
@xmatthias
@@ -54,14 +64,13 @@ class FreqaiDataDrawer:
self.config = config
self.freqai_info = config.get("freqai", {})
# dictionary holding all pair metadata necessary to load in from disk
self.pair_dict: Dict[str, Any] = {}
self.pair_dict: Dict[str, pair_info] = {}
# dictionary holding all actively inferenced models in memory given a model filename
self.model_dictionary: Dict[str, Any] = {}
self.model_return_values: Dict[str, Any] = {}
self.pair_data_dict: Dict[str, Any] = {}
self.historic_data: Dict[str, Any] = {}
self.historic_predictions: Dict[str, Any] = {}
self.follower_dict: Dict[str, Any] = {}
self.model_return_values: Dict[str, DataFrame] = {}
self.historic_data: Dict[str, Dict[str, DataFrame]] = {}
self.historic_predictions: Dict[str, DataFrame] = {}
self.follower_dict: Dict[str, pair_info] = {}
self.full_path = full_path
self.follower_name: str = self.config.get("bot_name", "follower1")
self.follower_dict_path = Path(
@@ -76,6 +85,10 @@ class FreqaiDataDrawer:
self.load_historic_predictions_from_disk()
self.training_queue: Dict[str, int] = {}
self.history_lock = threading.Lock()
self.old_DBSCAN_eps: Dict[str, float] = {}
self.empty_pair_dict: pair_info = {
"model_filename": "", "trained_timestamp": 0,
"priority": 1, "first": True, "data_path": "", "extras": {}}
def load_drawer_from_disk(self):
"""
@@ -132,15 +145,17 @@ class FreqaiDataDrawer:
"""
Save data drawer full of all pair model metadata in present model folder.
"""
with open(self.pair_dictionary_path, "w") as fp:
json.dump(self.pair_dict, fp, default=self.np_encoder)
with open(self.pair_dictionary_path, 'w') as fp:
rapidjson.dump(self.pair_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE)
def save_follower_dict_to_disk(self):
"""
Save follower dictionary to disk (used by strategy for persistent prediction targets)
"""
with open(self.follower_dict_path, "w") as fp:
json.dump(self.follower_dict, fp, default=self.np_encoder)
rapidjson.dump(self.follower_dict, fp, default=self.np_encoder,
number_mode=rapidjson.NM_NATIVE)
def create_follower_dict(self):
"""
@@ -174,18 +189,19 @@ class FreqaiDataDrawer:
trained_timestamp: int = the last time the coin was trained
return_null_array: bool = Follower could not find pair metadata
"""
pair_dict = self.pair_dict.get(pair)
data_path_set = self.pair_dict.get(pair, {}).get("data_path", None)
data_path_set = self.pair_dict.get(pair, self.empty_pair_dict).get("data_path", "")
return_null_array = False
if pair_dict:
model_filename = pair_dict["model_filename"]
trained_timestamp = pair_dict["trained_timestamp"]
elif not self.follow_mode:
pair_dict = self.pair_dict[pair] = {}
model_filename = pair_dict["model_filename"] = ""
trained_timestamp = pair_dict["trained_timestamp"] = 0
pair_dict["priority"] = len(self.pair_dict)
self.pair_dict[pair] = self.empty_pair_dict.copy()
model_filename = ""
trained_timestamp = 0
self.pair_dict[pair]["priority"] = len(self.pair_dict)
if not data_path_set and self.follow_mode:
logger.warning(
@@ -204,11 +220,9 @@ class FreqaiDataDrawer:
if pair_in_dict:
return
else:
self.pair_dict[metadata["pair"]] = {}
self.pair_dict[metadata["pair"]]["model_filename"] = ""
self.pair_dict[metadata["pair"]]["first"] = True
self.pair_dict[metadata["pair"]]["trained_timestamp"] = 0
self.pair_dict[metadata["pair"]] = self.empty_pair_dict.copy()
self.pair_dict[metadata["pair"]]["priority"] = len(self.pair_dict)
return
def pair_to_end_of_training_queue(self, pair: str) -> None:
@@ -225,25 +239,59 @@ class FreqaiDataDrawer:
historical candles, and also stores historical predictions despite retrainings (so stored
predictions are true predictions, not just inferencing on trained data)
"""
# dynamic df returned to strategy and plotted in frequi
mrv_df = self.model_return_values[pair] = pd.DataFrame()
for label in dk.label_list:
mrv_df[label] = pred_df[label]
mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
# if user reused `identifier` in config and has historical predictions collected, load them
# so that frequi remains uninterrupted after a crash
hist_df = self.historic_predictions
if pair in hist_df:
len_diff = len(hist_df[pair].index) - len(pred_df.index)
if len_diff < 0:
df_concat = pd.concat([pred_df.iloc[:abs(len_diff)], hist_df[pair]],
ignore_index=True, keys=hist_df[pair].keys())
else:
df_concat = hist_df[pair].tail(len(pred_df.index)).reset_index(drop=True)
df_concat = df_concat.fillna(0)
self.model_return_values[pair] = df_concat
logger.info(f'Setting initial FreqUI plots from historical data for {pair}.')
if self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0) > 0:
mrv_df["DI_values"] = dk.DI_values
else:
for label in pred_df.columns:
mrv_df[label] = pred_df[label]
if mrv_df[label].dtype == object:
continue
mrv_df[f"{label}_mean"] = dk.data["labels_mean"][label]
mrv_df[f"{label}_std"] = dk.data["labels_std"][label]
mrv_df["do_predict"] = do_preds
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
mrv_df["DI_values"] = dk.DI_values
def append_model_predictions(self, pair: str, predictions: DataFrame, do_preds: ArrayLike,
mrv_df["do_predict"] = do_preds
if dk.data['extra_returns_per_train']:
rets = dk.data['extra_returns_per_train']
for return_str in rets:
mrv_df[return_str] = rets[return_str]
# for keras type models, the conv_window needs to be prepended so
# viewing is correct in frequi
if self.freqai_info.get('keras', False):
n_lost_points = self.freqai_info.get('conv_width', 2)
zeros_df = DataFrame(np.zeros((n_lost_points, len(mrv_df.columns))),
columns=mrv_df.columns)
self.model_return_values[pair] = pd.concat(
[zeros_df, mrv_df], axis=0, ignore_index=True)
def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_],
dk: FreqaiDataKitchen, len_df: int) -> None:
# strat seems to feed us variable sized dataframes - and since we are trying to build our
# own return array in the same shape, we need to figure out how the size has changed
# and adapt our stored/returned info accordingly.
length_difference = len(self.model_return_values[pair]) - len_df
i = 0
@@ -262,19 +310,28 @@ class FreqaiDataDrawer:
hp_df = pd.concat([hp_df, nan_df], ignore_index=True, axis=0)
self.historic_predictions[pair] = hp_df[:-1]
for label in dk.label_list:
# incase user adds additional "predictions" e.g. predict_proba output:
for label in predictions.columns:
df[label].iloc[-1] = predictions[label].iloc[-1]
if df[label].dtype == object:
continue
df[f"{label}_mean"].iloc[-1] = dk.data["labels_mean"][label]
df[f"{label}_std"].iloc[-1] = dk.data["labels_std"][label]
# df['prediction'].iloc[-1] = predictions[-1]
df["do_predict"].iloc[-1] = do_preds[-1]
if self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0) > 0:
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
df["DI_values"].iloc[-1] = dk.DI_values[-1]
if dk.data['extra_returns_per_train']:
rets = dk.data['extra_returns_per_train']
for return_str in rets:
df[return_str].iloc[-1] = rets[return_str]
# append the new predictions to persistent storage
if pair in self.historic_predictions:
self.historic_predictions[pair].iloc[-1] = df[label].iloc[-1]
for key in df.keys():
self.historic_predictions[pair][key].iloc[-1] = df[key].iloc[-1]
if length_difference < 0:
prepend_df = pd.DataFrame(
@@ -301,16 +358,25 @@ class FreqaiDataDrawer:
dk.find_features(dataframe)
for label in dk.label_list:
if self.freqai_info.get('predict_proba', []):
full_labels = dk.label_list + self.freqai_info['predict_proba']
else:
full_labels = dk.label_list
for label in full_labels:
dataframe[label] = 0
dataframe[f"{label}_mean"] = 0
dataframe[f"{label}_std"] = 0
# dataframe['prediction'] = 0
dataframe["do_predict"] = 0
if self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0) > 0:
dataframe["DI_value"] = 0
if self.freqai_info["feature_parameters"].get("DI_threshold", 0) > 0:
dataframe["DI_values"] = 0
if dk.data['extra_returns_per_train']:
rets = dk.data['extra_returns_per_train']
for return_str in rets:
dataframe[return_str] = 0
dk.return_dataframe = dataframe
@@ -379,24 +445,28 @@ class FreqaiDataDrawer:
model.save(save_path / f"{dk.model_filename}_model.h5")
if dk.svm_model is not None:
dump(dk.svm_model, save_path / str(dk.model_filename + "_svm_model.joblib"))
dump(dk.svm_model, save_path / f"{dk.model_filename}_svm_model.joblib")
dk.data["data_path"] = str(dk.data_path)
dk.data["model_filename"] = str(dk.model_filename)
dk.data["training_features_list"] = list(dk.data_dictionary["train_features"].columns)
dk.data["label_list"] = dk.label_list
# store the metadata
with open(save_path / str(dk.model_filename + "_metadata.json"), "w") as fp:
json.dump(dk.data, fp, default=dk.np_encoder)
with open(save_path / f"{dk.model_filename}_metadata.json", "w") as fp:
rapidjson.dump(dk.data, fp, default=self.np_encoder, number_mode=rapidjson.NM_NATIVE)
# save the train data to file so we can check preds for area of applicability later
dk.data_dictionary["train_features"].to_pickle(
save_path / str(dk.model_filename + "_trained_df.pkl")
save_path / f"{dk.model_filename}_trained_df.pkl"
)
if self.freqai_info.get("feature_parameters", {}).get("principal_component_analysis"):
dk.data_dictionary["train_dates"].to_pickle(
save_path / f"{dk.model_filename}_trained_dates_df.pkl"
)
if self.freqai_info["feature_parameters"].get("principal_component_analysis"):
cloudpickle.dump(
dk.pca, open(dk.data_path / str(dk.model_filename + "_pca_object.pkl"), "wb")
dk.pca, open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "wb")
)
# if self.live:
@@ -429,27 +499,27 @@ class FreqaiDataDrawer:
/ dk.data_path.parts[-1]
)
with open(dk.data_path / str(dk.model_filename + "_metadata.json"), "r") as fp:
with open(dk.data_path / f"{dk.model_filename}_metadata.json", "r") as fp:
dk.data = json.load(fp)
dk.training_features_list = dk.data["training_features_list"]
dk.label_list = dk.data["label_list"]
dk.data_dictionary["train_features"] = pd.read_pickle(
dk.data_path / str(dk.model_filename + "_trained_df.pkl")
dk.data_path / f"{dk.model_filename}_trained_df.pkl"
)
# try to access model in memory instead of loading object from disk to save time
if dk.live and dk.model_filename in self.model_dictionary:
model = self.model_dictionary[dk.model_filename]
elif not dk.keras:
model = load(dk.data_path / str(dk.model_filename + "_model.joblib"))
model = load(dk.data_path / f"{dk.model_filename}_model.joblib")
else:
from tensorflow import keras
model = keras.models.load_model(dk.data_path / str(dk.model_filename + "_model.h5"))
model = keras.models.load_model(dk.data_path / f"{dk.model_filename}_model.h5")
if Path(dk.data_path / str(dk.model_filename + "_svm_model.joblib")).resolve().exists():
dk.svm_model = load(dk.data_path / str(dk.model_filename + "_svm_model.joblib"))
if Path(dk.data_path / f"{dk.model_filename}_svm_model.joblib").is_file():
dk.svm_model = load(dk.data_path / f"{dk.model_filename}_svm_model.joblib")
if not model:
raise OperationalException(
@@ -458,7 +528,7 @@ class FreqaiDataDrawer:
if self.config["freqai"]["feature_parameters"]["principal_component_analysis"]:
dk.pca = cloudpickle.load(
open(dk.data_path / str(dk.model_filename + "_pca_object.pkl"), "rb")
open(dk.data_path / f"{dk.model_filename}_pca_object.pkl", "rb")
)
return model
@@ -471,7 +541,7 @@ class FreqaiDataDrawer:
:params:
dataframe: DataFrame = strategy provided dataframe
"""
feat_params = self.freqai_info.get("feature_parameters", {})
feat_params = self.freqai_info["feature_parameters"]
with self.history_lock:
history_data = self.historic_data
@@ -524,7 +594,7 @@ class FreqaiDataDrawer:
for pair in dk.all_pairs:
if pair not in history_data:
history_data[pair] = {}
for tf in self.freqai_info.get("feature_parameters", {}).get("include_timeframes"):
for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
history_data[pair][tf] = load_pair_history(
datadir=self.config["datadir"],
timeframe=tf,
@@ -550,11 +620,11 @@ class FreqaiDataDrawer:
corr_dataframes: Dict[Any, Any] = {}
base_dataframes: Dict[Any, Any] = {}
historic_data = self.historic_data
pairs = self.freqai_info.get("feature_parameters", {}).get(
pairs = self.freqai_info["feature_parameters"].get(
"include_corr_pairlist", []
)
for tf in self.freqai_info.get("feature_parameters", {}).get("include_timeframes"):
for tf in self.freqai_info["feature_parameters"].get("include_timeframes"):
base_dataframes[tf] = dk.slice_dataframe(timerange, historic_data[pair][tf])
if pairs:
for p in pairs:

467
freqtrade/freqai/data_kitchen.py
View File

@@ -10,13 +10,16 @@ import numpy.typing as npt
import pandas as pd
from pandas import DataFrame
from sklearn import linear_model
from sklearn.cluster import DBSCAN
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.model_selection import train_test_split
from sklearn.neighbors import NearestNeighbors
from freqtrade.configuration import TimeRange
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
from freqtrade.exceptions import OperationalException
from freqtrade.resolvers import ExchangeResolver
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.strategy.interface import IStrategy
@@ -39,7 +42,7 @@ class FreqaiDataKitchen:
Robert Caulk @robcaulk
Theoretical brainstorming:
Elin Törnquist @thorntwig
Elin Törnquist @th0rntwig
Code review, software architecture brainstorming:
@xmatthias
@@ -55,10 +58,10 @@ class FreqaiDataKitchen:
live: bool = False,
pair: str = "",
):
self.data: Dict[Any, Any] = {}
self.data_dictionary: Dict[Any, Any] = {}
self.data: Dict[str, Any] = {}
self.data_dictionary: Dict[str, DataFrame] = {}
self.config = config
self.freqai_config = config["freqai"]
self.freqai_config: Dict[str, Any] = config["freqai"]
self.full_df: DataFrame = DataFrame()
self.append_df: DataFrame = DataFrame()
self.data_path = Path()
@@ -68,14 +71,14 @@ class FreqaiDataKitchen:
self.live = live
self.pair = pair
self.svm_model: linear_model.SGDOneClassSVM = None
self.keras = self.freqai_config.get("keras", False)
self.keras: bool = self.freqai_config.get("keras", False)
self.set_all_pairs()
if not self.live:
if not self.config["timerange"]:
raise OperationalException(
'Please pass --timerange if you intend to use FreqAI for backtesting.')
self.full_timerange = self.create_fulltimerange(
self.config["timerange"], self.freqai_config.get("train_period_days")
self.config["timerange"], self.freqai_config.get("train_period_days", 0)
)
(self.training_timeranges, self.backtesting_timeranges) = self.split_timerange(
@@ -84,6 +87,10 @@ class FreqaiDataKitchen:
config["freqai"]["backtest_period_days"],
)
self.data['extra_returns_per_train'] = self.freqai_config.get('extra_returns_per_train', {})
self.thread_count = self.freqai_config.get("data_kitchen_thread_count", -1)
self.train_dates: DataFrame = pd.DataFrame()
def set_paths(
self,
pair: str,
@@ -101,7 +108,7 @@ class FreqaiDataKitchen:
self.data_path = Path(
self.full_path
/ str("sub-train" + "-" + pair.split("/")[0] + "_" + str(trained_timestamp))
/ f"sub-train-{pair.split('/')[0]}_{trained_timestamp}"
)
return
@@ -116,7 +123,7 @@ class FreqaiDataKitchen:
:filtered_dataframe: cleaned dataframe ready to be split.
:labels: cleaned labels ready to be split.
"""
feat_dict = self.freqai_config.get("feature_parameters", {})
feat_dict = self.freqai_config["feature_parameters"]
weights: npt.ArrayLike
if feat_dict.get("weight_factor", 0) > 0:
@@ -188,20 +195,23 @@ class FreqaiDataKitchen:
drop_index = pd.isnull(filtered_dataframe).any(1) # get the rows that have NaNs,
drop_index = drop_index.replace(True, 1).replace(False, 0) # pep8 requirement.
if (
training_filter
): # we don't care about total row number (total no. datapoints) in training, we only care
if (training_filter):
# we don't care about total row number (total no. datapoints) in training, we only care
# about removing any row with NaNs
# if labels has multiple columns (user wants to train multiple models), we detect here
# if labels has multiple columns (user wants to train multiple modelEs), we detect here
labels = unfiltered_dataframe.filter(label_list, axis=1)
drop_index_labels = pd.isnull(labels).any(1)
drop_index_labels = drop_index_labels.replace(True, 1).replace(False, 0)
dates = unfiltered_dataframe['date']
filtered_dataframe = filtered_dataframe[
(drop_index == 0) & (drop_index_labels == 0)
] # dropping values
labels = labels[
(drop_index == 0) & (drop_index_labels == 0)
] # assuming the labels depend entirely on the dataframe here.
self.train_dates = dates[
(drop_index == 0) & (drop_index_labels == 0)
]
logger.info(
f"dropped {len(unfiltered_dataframe) - len(filtered_dataframe)} training points"
f" due to NaNs in populated dataset {len(unfiltered_dataframe)}."
@@ -252,6 +262,7 @@ class FreqaiDataKitchen:
"test_labels": test_labels,
"train_weights": train_weights,
"test_weights": test_weights,
"train_dates": self.train_dates
}
return self.data_dictionary
@@ -279,7 +290,7 @@ class FreqaiDataKitchen:
self.data[item + "_min"] = train_min[item]
for item in data_dictionary["train_labels"].keys():
if data_dictionary["train_labels"][item].dtype == str:
if data_dictionary["train_labels"][item].dtype == object:
continue
train_labels_max = data_dictionary["train_labels"][item].max()
train_labels_min = data_dictionary["train_labels"][item].min()
@@ -305,8 +316,7 @@ class FreqaiDataKitchen:
"""
Normalize a set of data using the mean and standard deviation from
the associated training data.
:params:
:df: Dataframe to be standardized
:param df: Dataframe to be standardized
"""
for item in df.keys():
@@ -323,12 +333,11 @@ class FreqaiDataKitchen:
"""
Normalize a set of data using the mean and standard deviation from
the associated training data.
:params:
:df: Dataframe of predictions to be denormalized
:param df: Dataframe of predictions to be denormalized
"""
for label in self.label_list:
if df[label].dtype == str:
for label in df.columns:
if df[label].dtype == object:
continue
df[label] = (
(df[label] + 1)
@@ -339,7 +348,7 @@ class FreqaiDataKitchen:
return df
def split_timerange(
self, tr: str, train_split: int = 28, bt_split: int = 7
self, tr: str, train_split: int = 28, bt_split: float = 7
) -> Tuple[list, list]:
"""
Function which takes a single time range (tr) and splits it
@@ -347,12 +356,12 @@ class FreqaiDataKitchen:
tr: str, full timerange to train on
train_split: the period length for the each training (days). Specified in user
configuration file
bt_split: the backtesting length (dats). Specified in user configuration file
bt_split: the backtesting length (days). Specified in user configuration file
"""
if not isinstance(train_split, int) or train_split < 1:
raise OperationalException(
"train_period_days must be an integer greater than 0. " f"Got {train_split}."
f"train_period_days must be an integer greater than 0. Got {train_split}."
)
train_period_days = train_split * SECONDS_IN_DAY
bt_period = bt_split * SECONDS_IN_DAY
@@ -374,7 +383,7 @@ class FreqaiDataKitchen:
while True:
if not first:
timerange_train.startts = timerange_train.startts + bt_period
timerange_train.startts = timerange_train.startts + int(bt_period)
timerange_train.stopts = timerange_train.startts + train_period_days
first = False
@@ -387,7 +396,7 @@ class FreqaiDataKitchen:
timerange_backtest.startts = timerange_train.stopts
timerange_backtest.stopts = timerange_backtest.startts + bt_period
timerange_backtest.stopts = timerange_backtest.startts + int(bt_period)
if timerange_backtest.stopts > config_timerange.stopts:
timerange_backtest.stopts = config_timerange.stopts
@@ -408,10 +417,9 @@ class FreqaiDataKitchen:
def slice_dataframe(self, timerange: TimeRange, df: DataFrame) -> DataFrame:
"""
Given a full dataframe, extract the user desired window
:params:
:tr: timerange string that we wish to extract from df
:df: Dataframe containing all candles to run the entire backtest. Here
it is sliced down to just the present training period.
:param tr: timerange string that we wish to extract from df
:param df: Dataframe containing all candles to run the entire backtest. Here
it is sliced down to just the present training period.
"""
start = datetime.datetime.fromtimestamp(timerange.startts, tz=datetime.timezone.utc)
@@ -489,11 +497,10 @@ class FreqaiDataKitchen:
point. This metric defines the neighborhood of trained data and is used
for prediction confidence in the Dissimilarity Index
"""
logger.info("computing average mean distance for all training points")
tc = self.freqai_config.get("model_training_parameters", {}).get("thread_count", -1)
pairwise = pairwise_distances(self.data_dictionary["train_features"], n_jobs=tc)
# logger.info("computing average mean distance for all training points")
pairwise = pairwise_distances(
self.data_dictionary["train_features"], n_jobs=self.thread_count)
avg_mean_dist = pairwise.mean(axis=1).mean()
logger.info(f"avg_mean_dist {avg_mean_dist:.2f}")
return avg_mean_dist
@@ -515,21 +522,22 @@ class FreqaiDataKitchen:
return
if predict:
assert self.svm_model, "No svm model available for outlier removal"
if not self.svm_model:
logger.warning("No svm model available for outlier removal")
return
y_pred = self.svm_model.predict(self.data_dictionary["prediction_features"])
do_predict = np.where(y_pred == -1, 0, y_pred)
if (len(do_predict) - do_predict.sum()) > 0:
logger.info(
f"svm_remove_outliers() tossed {len(do_predict) - do_predict.sum()} predictions"
)
logger.info(f"SVM tossed {len(do_predict) - do_predict.sum()} predictions.")
self.do_predict += do_predict
self.do_predict -= 1
else:
# use SGDOneClassSVM to increase speed?
nu = self.freqai_config.get("feature_parameters", {}).get("svm_nu", 0.2)
self.svm_model = linear_model.SGDOneClassSVM(nu=nu).fit(
svm_params = self.freqai_config["feature_parameters"].get(
"svm_params", {"shuffle": False, "nu": 0.1})
self.svm_model = linear_model.SGDOneClassSVM(**svm_params).fit(
self.data_dictionary["train_features"]
)
y_pred = self.svm_model.predict(self.data_dictionary["train_features"])
@@ -546,12 +554,14 @@ class FreqaiDataKitchen:
]
logger.info(
f"svm_remove_outliers() tossed {len(y_pred) - dropped_points.sum()}"
f" train points from {len(y_pred)}"
f"SVM tossed {len(y_pred) - dropped_points.sum()}"
f" train points from {len(y_pred)} total points."
)
# same for test data
if self.freqai_config.get('data_split_parameters', {}).get('test_size', 0.1) != 0:
# TODO: This (and the part above) could be refactored into a separate function
# to reduce code duplication
if self.freqai_config['data_split_parameters'].get('test_size', 0.1) != 0:
y_pred = self.svm_model.predict(self.data_dictionary["test_features"])
dropped_points = np.where(y_pred == -1, 0, y_pred)
self.data_dictionary["test_features"] = self.data_dictionary["test_features"][
@@ -564,8 +574,77 @@ class FreqaiDataKitchen:
]
logger.info(
f"svm_remove_outliers() tossed {len(y_pred) - dropped_points.sum()}"
f" test points from {len(y_pred)}"
f"SVM tossed {len(y_pred) - dropped_points.sum()}"
f" test points from {len(y_pred)} total points."
)
return
def use_DBSCAN_to_remove_outliers(self, predict: bool, eps=None) -> None:
"""
Use DBSCAN to cluster training data and remove "noisy" data (read outliers).
User controls this via the config param `DBSCAN_outlier_pct` which indicates the
pct of training data that they want to be considered outliers.
:params:
predict: bool = If False (training), iterate to find the best hyper parameters to match
user requested outlier percent target. If True (prediction), use the parameters
determined from the previous training to estimate if the current prediction point
is an outlier.
"""
if predict:
train_ft_df = self.data_dictionary['train_features']
pred_ft_df = self.data_dictionary['prediction_features']
num_preds = len(pred_ft_df)
df = pd.concat([train_ft_df, pred_ft_df], axis=0, ignore_index=True)
clustering = DBSCAN(eps=self.data['DBSCAN_eps'],
min_samples=self.data['DBSCAN_min_samples'],
n_jobs=self.thread_count
).fit(df)
do_predict = np.where(clustering.labels_[-num_preds:] == -1, 0, 1)
if (len(do_predict) - do_predict.sum()) > 0:
logger.info(f"DBSCAN tossed {len(do_predict) - do_predict.sum()} predictions")
self.do_predict += do_predict
self.do_predict -= 1
else:
MinPts = len(self.data_dictionary['train_features'].columns) * 2
# measure pairwise distances to train_features.shape[1]*2 nearest neighbours
neighbors = NearestNeighbors(
n_neighbors=MinPts, n_jobs=self.thread_count)
neighbors_fit = neighbors.fit(self.data_dictionary['train_features'])
distances, _ = neighbors_fit.kneighbors(self.data_dictionary['train_features'])
distances = np.sort(distances, axis=0)
index_ten_pct = int(len(distances[:, 1]) * 0.1)
distances = distances[index_ten_pct:, 1]
epsilon = distances[-1]
clustering = DBSCAN(eps=epsilon, min_samples=MinPts,
n_jobs=int(self.thread_count)).fit(
self.data_dictionary['train_features']
)
logger.info(f'DBSCAN found eps of {epsilon}.')
self.data['DBSCAN_eps'] = epsilon
self.data['DBSCAN_min_samples'] = MinPts
dropped_points = np.where(clustering.labels_ == -1, 1, 0)
self.data_dictionary['train_features'] = self.data_dictionary['train_features'][
(clustering.labels_ != -1)
]
self.data_dictionary["train_labels"] = self.data_dictionary["train_labels"][
(clustering.labels_ != -1)
]
self.data_dictionary["train_weights"] = self.data_dictionary["train_weights"][
(clustering.labels_ != -1)
]
logger.info(
f"DBSCAN tossed {dropped_points.sum()}"
f" train points from {len(clustering.labels_)}"
)
return
@@ -573,9 +652,8 @@ class FreqaiDataKitchen:
def find_features(self, dataframe: DataFrame) -> None:
"""
Find features in the strategy provided dataframe
:params:
dataframe: DataFrame = strategy provided dataframe
:returns:
:param dataframe: DataFrame = strategy provided dataframe
:return:
features: list = the features to be used for training/prediction
"""
column_names = dataframe.columns
@@ -586,7 +664,6 @@ class FreqaiDataKitchen:
self.training_features_list = features
self.label_list = labels
# return features, labels
def check_if_pred_in_training_spaces(self) -> None:
"""
@@ -599,13 +676,13 @@ class FreqaiDataKitchen:
distance = pairwise_distances(
self.data_dictionary["train_features"],
self.data_dictionary["prediction_features"],
n_jobs=-1,
n_jobs=self.thread_count,
)
self.DI_values = distance.min(axis=0) / self.data["avg_mean_dist"]
do_predict = np.where(
self.DI_values < self.freqai_config.get("feature_parameters", {}).get("DI_threshold"),
self.DI_values < self.freqai_config["feature_parameters"]["DI_threshold"],
1,
0,
)
@@ -628,25 +705,27 @@ class FreqaiDataKitchen:
weights = np.exp(-np.arange(num_weights) / (wfactor * num_weights))[::-1]
return weights
def append_predictions(self, predictions, do_predict, len_dataframe):
def append_predictions(self, predictions: DataFrame, do_predict: npt.ArrayLike) -> None:
"""
Append backtest prediction from current backtest period to all previous periods
"""
self.append_df = DataFrame()
for label in self.label_list:
self.append_df[label] = predictions[label]
self.append_df[f"{label}_mean"] = self.data["labels_mean"][label]
self.append_df[f"{label}_std"] = self.data["labels_std"][label]
append_df = DataFrame()
for label in predictions.columns:
append_df[label] = predictions[label]
if append_df[label].dtype == object:
continue
append_df[f"{label}_mean"] = self.data["labels_mean"][label]
append_df[f"{label}_std"] = self.data["labels_std"][label]
self.append_df["do_predict"] = do_predict
if self.freqai_config.get("feature_parameters", {}).get("DI_threshold", 0) > 0:
self.append_df["DI_values"] = self.DI_values
append_df["do_predict"] = do_predict
if self.freqai_config["feature_parameters"].get("DI_threshold", 0) > 0:
append_df["DI_values"] = self.DI_values
if self.full_df.empty:
self.full_df = self.append_df
self.full_df = append_df
else:
self.full_df = pd.concat([self.full_df, self.append_df], axis=0)
self.full_df = pd.concat([self.full_df, append_df], axis=0)
return
@@ -666,7 +745,6 @@ class FreqaiDataKitchen:
to_keep = [col for col in dataframe.columns if not col.startswith("&")]
self.return_dataframe = pd.concat([dataframe[to_keep], self.full_df], axis=1)
self.append_df = DataFrame()
self.full_df = DataFrame()
return
@@ -683,7 +761,7 @@ class FreqaiDataKitchen:
if backtest_timerange.stopts == 0:
# typically open ended time ranges do work, however, there are some edge cases where
# it does not. accomodating these kinds of edge cases just to allow open-ended
# it does not. accommodating these kinds of edge cases just to allow open-ended
# timerange is not high enough priority to warrant the effort. It is safer for now
# to simply ask user to add their end date
raise OperationalException("FreqAI backtesting does not allow open ended timeranges. "
@@ -701,7 +779,7 @@ class FreqaiDataKitchen:
full_timerange = start.strftime("%Y%m%d") + "-" + stop.strftime("%Y%m%d")
self.full_path = Path(
self.config["user_data_dir"] / "models" / str(self.freqai_config.get("identifier"))
self.config["user_data_dir"] / "models" / f"{self.freqai_config['identifier']}"
)
config_path = Path(self.config["config_files"][0])
@@ -719,10 +797,9 @@ class FreqaiDataKitchen:
"""
A model age checker to determine if the model is trustworthy based on user defined
`expiration_hours` in the configuration file.
:params:
trained_timestamp: int = The time of training for the most recent model.
:returns:
bool = If the model is expired or not.
:param trained_timestamp: int = The time of training for the most recent model.
:return:
bool = If the model is expired or not.
"""
time = datetime.datetime.now(tz=datetime.timezone.utc).timestamp()
elapsed_time = (time - trained_timestamp) / 3600 # hours
@@ -740,30 +817,21 @@ class FreqaiDataKitchen:
trained_timerange = TimeRange()
data_load_timerange = TimeRange()
# find the max indicator length required
max_timeframe_chars = self.freqai_config.get("feature_parameters", {}).get(
"include_timeframes"
)[-1]
max_period = self.freqai_config.get("feature_parameters", {}).get(
"indicator_max_period_candles", 50
)
additional_seconds = 0
if max_timeframe_chars[-1] == "d":
additional_seconds = max_period * SECONDS_IN_DAY * int(max_timeframe_chars[-2])
elif max_timeframe_chars[-1] == "h":
additional_seconds = max_period * 3600 * int(max_timeframe_chars[-2])
elif max_timeframe_chars[-1] == "m":
if len(max_timeframe_chars) == 2:
additional_seconds = max_period * 60 * int(max_timeframe_chars[-2])
elif len(max_timeframe_chars) == 3:
additional_seconds = max_period * 60 * int(float(max_timeframe_chars[0:2]))
else:
logger.warning(
"FreqAI could not detect max timeframe and therefore may not "
"download the proper amount of data for training"
)
timeframes = self.freqai_config["feature_parameters"].get("include_timeframes")
# logger.info(f'Extending data download by {additional_seconds/SECONDS_IN_DAY:.2f} days')
max_tf_seconds = 0
for tf in timeframes:
secs = timeframe_to_seconds(tf)
if secs > max_tf_seconds:
max_tf_seconds = secs
# We notice that users like to use exotic indicators where
# they do not know the required timeperiod. Here we include a factor
# of safety by multiplying the user considered "max" by 2.
max_period = self.freqai_config["feature_parameters"].get(
"indicator_max_period_candles", 20
) * 2
additional_seconds = max_period * max_tf_seconds
if trained_timestamp != 0:
elapsed_time = (time - trained_timestamp) / SECONDS_IN_HOUR
@@ -784,7 +852,7 @@ class FreqaiDataKitchen:
data_load_timerange.stopts = int(time)
else: # user passed no live_trained_timerange in config
trained_timerange.startts = int(
time - self.freqai_config.get("train_period_days") * SECONDS_IN_DAY
time - self.freqai_config.get("train_period_days", 0) * SECONDS_IN_DAY
)
trained_timerange.stopts = int(time)
@@ -815,24 +883,22 @@ class FreqaiDataKitchen:
self.model_filename = f"cb_{coin.lower()}_{int(trained_timerange.stopts)}"
def download_all_data_for_training(self, timerange: TimeRange) -> None:
def download_all_data_for_training(self, timerange: TimeRange, dp: DataProvider) -> None:
"""
Called only once upon start of bot to download the necessary data for
populating indicators and training the model.
:params:
timerange: TimeRange = The full data timerange for populating the indicators
and training the model.
:param timerange: TimeRange = The full data timerange for populating the indicators
and training the model.
:param dp: DataProvider instance attached to the strategy
"""
exchange = ExchangeResolver.load_exchange(
self.config["exchange"]["name"], self.config, validate=False, load_leverage_tiers=False
)
new_pairs_days = int((timerange.stopts - timerange.startts) / SECONDS_IN_DAY)
if not dp._exchange:
# Not realistic - this is only called in live mode.
raise OperationalException("Dataprovider did not have an exchange attached.")
refresh_backtest_ohlcv_data(
exchange,
dp._exchange,
pairs=self.all_pairs,
timeframes=self.freqai_config.get("feature_parameters", {}).get("include_timeframes"),
timeframes=self.freqai_config["feature_parameters"].get("include_timeframes"),
datadir=self.config["datadir"],
timerange=timerange,
new_pairs_days=new_pairs_days,
@@ -845,7 +911,7 @@ class FreqaiDataKitchen:
def set_all_pairs(self) -> None:
self.all_pairs = copy.deepcopy(
self.freqai_config.get("feature_parameters", {}).get("include_corr_pairlist", [])
self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
)
for pair in self.config.get("exchange", "").get("pair_whitelist"):
if pair not in self.all_pairs:
@@ -876,8 +942,8 @@ class FreqaiDataKitchen:
# for prediction dataframe creation, we let dataprovider handle everything in the strategy
# so we create empty dictionaries, which allows us to pass None to
# `populate_any_indicators()`. Signaling we want the dp to give us the live dataframe.
tfs = self.freqai_config.get("feature_parameters", {}).get("include_timeframes")
pairs = self.freqai_config.get("feature_parameters", {}).get("include_corr_pairlist", [])
tfs = self.freqai_config["feature_parameters"].get("include_timeframes")
pairs = self.freqai_config["feature_parameters"].get("include_corr_pairlist", [])
if not prediction_dataframe.empty:
dataframe = prediction_dataframe.copy()
for tf in tfs:
@@ -889,29 +955,26 @@ class FreqaiDataKitchen:
else:
dataframe = base_dataframes[self.config["timeframe"]].copy()
sgi = True
sgi = False
for tf in tfs:
if tf == tfs[-1]:
sgi = True # doing this last allows user to use all tf raw prices in labels
dataframe = strategy.populate_any_indicators(
pair,
pair,
dataframe.copy(),
tf,
informative=base_dataframes[tf],
coin=pair.split("/")[0] + "-",
set_generalized_indicators=sgi,
set_generalized_indicators=sgi
)
sgi = False
if pairs:
for i in pairs:
if pair in i:
continue # dont repeat anything from whitelist
dataframe = strategy.populate_any_indicators(
pair,
i,
dataframe.copy(),
tf,
informative=corr_dataframes[i][tf],
coin=i.split("/")[0] + "-",
informative=corr_dataframes[i][tf]
)
return dataframe
@@ -923,17 +986,12 @@ class FreqaiDataKitchen:
import scipy as spy
self.data["labels_mean"], self.data["labels_std"] = {}, {}
for label in self.label_list:
for label in self.data_dictionary["train_labels"].columns:
if self.data_dictionary["train_labels"][label].dtype == object:
continue
f = spy.stats.norm.fit(self.data_dictionary["train_labels"][label])
self.data["labels_mean"][label], self.data["labels_std"][label] = f[0], f[1]
# KEEPME incase we want to let user start to grab quantiles.
# upper_q = spy.stats.norm.ppf(self.freqai_config['feature_parameters'][
# 'target_quantile'], *f)
# lower_q = spy.stats.norm.ppf(1 - self.freqai_config['feature_parameters'][
# 'target_quantile'], *f)
# self.data["upper_quantile"] = upper_q
# self.data["lower_quantile"] = lower_q
return
def remove_features_from_df(self, dataframe: DataFrame) -> DataFrame:
@@ -945,168 +1003,3 @@ class FreqaiDataKitchen:
col for col in dataframe.columns if not col.startswith("%") or col.startswith("%%")
]
return dataframe[to_keep]
def np_encoder(self, object):
if isinstance(object, np.generic):
return object.item()
# Functions containing useful data manpulation examples. but not actively in use.
# Possibly phasing these outlier removal methods below out in favor of
# use_SVM_to_remove_outliers (computationally more efficient and apparently higher performance).
# But these have good data manipulation examples, so keep them commented here for now.
# def determine_statistical_distributions(self) -> None:
# from fitter import Fitter
# logger.info('Determining best model for all features, may take some time')
# def compute_quantiles(ft):
# f = Fitter(self.data_dictionary["train_features"][ft],
# distributions=['gamma', 'cauchy', 'laplace',
# 'beta', 'uniform', 'lognorm'])
# f.fit()
# # f.summary()
# dist = list(f.get_best().items())[0][0]
# params = f.get_best()[dist]
# upper_q = getattr(spy.stats, list(f.get_best().items())[0][0]).ppf(0.999, **params)
# lower_q = getattr(spy.stats, list(f.get_best().items())[0][0]).ppf(0.001, **params)
# return ft, upper_q, lower_q, dist
# quantiles_tuple = Parallel(n_jobs=-1)(
# delayed(compute_quantiles)(ft) for ft in self.data_dictionary[
# 'train_features'].columns)
# df = pd.DataFrame(quantiles_tuple, columns=['features', 'upper_quantiles',
# 'lower_quantiles', 'dist'])
# self.data_dictionary['upper_quantiles'] = df['upper_quantiles']
# self.data_dictionary['lower_quantiles'] = df['lower_quantiles']
# return
# def remove_outliers(self, predict: bool) -> None:
# """
# Remove data that looks like an outlier based on the distribution of each
# variable.
# :params:
# :predict: boolean which tells the function if this is prediction data or
# training data coming in.
# """
# lower_quantile = self.data_dictionary["lower_quantiles"].to_numpy()
# upper_quantile = self.data_dictionary["upper_quantiles"].to_numpy()
# if predict:
# df = self.data_dictionary["prediction_features"][
# (self.data_dictionary["prediction_features"] < upper_quantile)
# & (self.data_dictionary["prediction_features"] > lower_quantile)
# ]
# drop_index = pd.isnull(df).any(1)
# self.data_dictionary["prediction_features"].fillna(0, inplace=True)
# drop_index = ~drop_index
# do_predict = np.array(drop_index.replace(True, 1).replace(False, 0))
# logger.info(
# "remove_outliers() tossed %s predictions",
# len(do_predict) - do_predict.sum(),
# )
# self.do_predict += do_predict
# self.do_predict -= 1
# else:
# filter_train_df = self.data_dictionary["train_features"][
# (self.data_dictionary["train_features"] < upper_quantile)
# & (self.data_dictionary["train_features"] > lower_quantile)
# ]
# drop_index = pd.isnull(filter_train_df).any(1)
# drop_index = drop_index.replace(True, 1).replace(False, 0)
# self.data_dictionary["train_features"] = self.data_dictionary["train_features"][
# (drop_index == 0)
# ]
# self.data_dictionary["train_labels"] = self.data_dictionary["train_labels"][
# (drop_index == 0)
# ]
# self.data_dictionary["train_weights"] = self.data_dictionary["train_weights"][
# (drop_index == 0)
# ]
# logger.info(
# f'remove_outliers() tossed {drop_index.sum()}'
# f' training points from {len(filter_train_df)}'
# )
# # do the same for the test data
# filter_test_df = self.data_dictionary["test_features"][
# (self.data_dictionary["test_features"] < upper_quantile)
# & (self.data_dictionary["test_features"] > lower_quantile)
# ]
# drop_index = pd.isnull(filter_test_df).any(1)
# drop_index = drop_index.replace(True, 1).replace(False, 0)
# self.data_dictionary["test_labels"] = self.data_dictionary["test_labels"][
# (drop_index == 0)
# ]
# self.data_dictionary["test_features"] = self.data_dictionary["test_features"][
# (drop_index == 0)
# ]
# self.data_dictionary["test_weights"] = self.data_dictionary["test_weights"][
# (drop_index == 0)
# ]
# logger.info(
# f'remove_outliers() tossed {drop_index.sum()}'
# f' test points from {len(filter_test_df)}'
# )
# return
# def standardize_data(self, data_dictionary: Dict) -> Dict[Any, Any]:
# """
# standardize all data in the data_dictionary according to the training dataset
# :params:
# :data_dictionary: dictionary containing the cleaned and split training/test data/labels
# :returns:
# :data_dictionary: updated dictionary with standardized values.
# """
# # standardize the data by training stats
# train_mean = data_dictionary["train_features"].mean()
# train_std = data_dictionary["train_features"].std()
# data_dictionary["train_features"] = (
# data_dictionary["train_features"] - train_mean
# ) / train_std
# data_dictionary["test_features"] = (
# data_dictionary["test_features"] - train_mean
# ) / train_std
# train_labels_std = data_dictionary["train_labels"].std()
# train_labels_mean = data_dictionary["train_labels"].mean()
# data_dictionary["train_labels"] = (
# data_dictionary["train_labels"] - train_labels_mean
# ) / train_labels_std
# data_dictionary["test_labels"] = (
# data_dictionary["test_labels"] - train_labels_mean
# ) / train_labels_std
# for item in train_std.keys():
# self.data[item + "_std"] = train_std[item]
# self.data[item + "_mean"] = train_mean[item]
# self.data["labels_std"] = train_labels_std
# self.data["labels_mean"] = train_labels_mean
# return data_dictionary
# def standardize_data_from_metadata(self, df: DataFrame) -> DataFrame:
# """
# Normalizes a set of data using the mean and standard deviation from
# the associated training data.
# :params:
# :df: Dataframe to be standardized
# """
# for item in df.keys():
# df[item] = (df[item] - self.data[item + "_mean"]) / self.data[item + "_std"]
# return df

121
freqtrade/freqai/freqai_interface.py
View File

@@ -1,7 +1,5 @@
# import contextlib
import copy
import datetime
import gc
import logging
import shutil
import threading
@@ -12,7 +10,7 @@ from typing import Any, Dict, Tuple
import numpy as np
import pandas as pd
from numpy.typing import ArrayLike
from numpy.typing import NDArray
from pandas import DataFrame
from freqtrade.configuration import TimeRange
@@ -47,7 +45,7 @@ class IFreqaiModel(ABC):
Robert Caulk @robcaulk
Theoretical brainstorming:
Elin Törnquist @thorntwig
Elin Törnquist @th0rntwig
Code review, software architecture brainstorming:
@xmatthias
@@ -82,6 +80,8 @@ class IFreqaiModel(ABC):
self.CONV_WIDTH = self.freqai_info.get("conv_width", 2)
self.pair_it = 0
self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist"))
self.last_trade_database_summary: DataFrame = {}
self.current_trade_database_summary: DataFrame = {}
def assert_config(self, config: Dict[str, Any]) -> None:
@@ -123,7 +123,7 @@ class IFreqaiModel(ABC):
dataframe = dk.remove_features_from_df(dk.return_dataframe)
del dk
return self.return_values(dataframe)
return dataframe
@threaded
def start_scanning(self, strategy: IStrategy) -> None:
@@ -183,8 +183,6 @@ class IFreqaiModel(ABC):
(_, _, _) = self.dd.get_pair_dict_info(metadata["pair"])
train_it += 1
total_trains = len(dk.backtesting_timeranges)
gc.collect()
dk.data = {} # clean the pair specific data between training window sliding
self.training_timerange = tr_train
dataframe_train = dk.slice_dataframe(tr_train, dataframe)
dataframe_backtest = dk.slice_dataframe(tr_backtest, dataframe)
@@ -204,14 +202,9 @@ class IFreqaiModel(ABC):
dk.data_path = Path(
dk.full_path
/ str(
"sub-train"
+ "-"
+ metadata["pair"].split("/")[0]
+ "_"
+ str(int(trained_timestamp.stopts))
/
f"sub-train-{metadata['pair'].split('/')[0]}_{int(trained_timestamp.stopts)}"
)
)
if not self.model_exists(
metadata["pair"], dk, trained_timestamp=int(trained_timestamp.stopts)
):
@@ -228,7 +221,7 @@ class IFreqaiModel(ABC):
pred_df, do_preds = self.predict(dataframe_backtest, dk)
dk.append_predictions(pred_df, do_preds, len(dataframe_backtest))
dk.append_predictions(pred_df, do_preds)
dk.fill_predictions(dataframe)
@@ -280,7 +273,7 @@ class IFreqaiModel(ABC):
"corr_pairlist, this may take a while if you do not have the "
"data saved"
)
dk.download_all_data_for_training(data_load_timerange)
dk.download_all_data_for_training(data_load_timerange, strategy.dp)
self.dd.load_all_pair_histories(data_load_timerange, dk)
if not self.scanning:
@@ -331,7 +324,8 @@ class IFreqaiModel(ABC):
return
elif self.dk.check_if_model_expired(trained_timestamp):
pred_df = DataFrame(np.zeros((2, len(dk.label_list))), columns=dk.label_list)
do_preds, dk.DI_values = np.ones(2) * 2, np.zeros(2)
do_preds = np.ones(2, dtype=np.int_) * 2
dk.DI_values = np.zeros(2)
logger.warning(
f"Model expired for {pair}, returning null values to strategy. Strategy "
"construction should take care to consider this event with "
@@ -379,17 +373,25 @@ class IFreqaiModel(ABC):
example of how outlier data points are dropped from the dataframe used for training.
"""
if self.freqai_info.get("feature_parameters", {}).get(
if self.freqai_info["feature_parameters"].get(
"principal_component_analysis", False
):
dk.principal_component_analysis()
if self.freqai_info.get("feature_parameters", {}).get("use_SVM_to_remove_outliers", False):
if self.freqai_info["feature_parameters"].get("use_SVM_to_remove_outliers", False):
dk.use_SVM_to_remove_outliers(predict=False)
if self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0):
if self.freqai_info["feature_parameters"].get("DI_threshold", 0):
dk.data["avg_mean_dist"] = dk.compute_distances()
if self.freqai_info["feature_parameters"].get("use_DBSCAN_to_remove_outliers", False):
if dk.pair in self.dd.old_DBSCAN_eps:
eps = self.dd.old_DBSCAN_eps[dk.pair]
else:
eps = None
dk.use_DBSCAN_to_remove_outliers(predict=False, eps=eps)
self.dd.old_DBSCAN_eps[dk.pair] = dk.data['DBSCAN_eps']
def data_cleaning_predict(self, dk: FreqaiDataKitchen, dataframe: DataFrame) -> None:
"""
Base data cleaning method for predict.
@@ -401,17 +403,20 @@ class IFreqaiModel(ABC):
of how the do_predict vector is modified. do_predict is ultimately passed back to strategy
for buy signals.
"""
if self.freqai_info.get("feature_parameters", {}).get(
if self.freqai_info["feature_parameters"].get(
"principal_component_analysis", False
):
dk.pca_transform(dataframe)
if self.freqai_info.get("feature_parameters", {}).get("use_SVM_to_remove_outliers", False):
if self.freqai_info["feature_parameters"].get("use_SVM_to_remove_outliers", False):
dk.use_SVM_to_remove_outliers(predict=True)
if self.freqai_info.get("feature_parameters", {}).get("DI_threshold", 0):
if self.freqai_info["feature_parameters"].get("DI_threshold", 0):
dk.check_if_pred_in_training_spaces()
if self.freqai_info["feature_parameters"].get("use_DBSCAN_to_remove_outliers", False):
dk.use_DBSCAN_to_remove_outliers(predict=True)
def model_exists(
self,
pair: str,
@@ -430,9 +435,9 @@ class IFreqaiModel(ABC):
coin, _ = pair.split("/")
if not self.live:
dk.model_filename = model_filename = "cb_" + coin.lower() + "_" + str(trained_timestamp)
dk.model_filename = model_filename = f"cb_{coin.lower()}_{trained_timestamp}"
path_to_modelfile = Path(dk.data_path / str(model_filename + "_model.joblib"))
path_to_modelfile = Path(dk.data_path / f"{model_filename}_model.joblib")
file_exists = path_to_modelfile.is_file()
if file_exists and not scanning:
logger.info("Found model at %s", dk.data_path / dk.model_filename)
@@ -442,7 +447,7 @@ class IFreqaiModel(ABC):
def set_full_path(self) -> None:
self.full_path = Path(
self.config["user_data_dir"] / "models" / str(self.freqai_info.get("identifier"))
self.config["user_data_dir"] / "models" / f"{self.freqai_info['identifier']}"
)
self.full_path.mkdir(parents=True, exist_ok=True)
shutil.copy(
@@ -500,13 +505,54 @@ class IFreqaiModel(ABC):
def set_initial_historic_predictions(
self, df: DataFrame, model: Any, dk: FreqaiDataKitchen, pair: str
) -> None:
trained_predictions = model.predict(df)
"""
This function is called only if the datadrawer failed to load an
existing set of historic predictions. In this case, it builds
the structure and sets fake predictions off the first training
data. After that, FreqAI will append new real predictions to the
set of historic predictions.
These values are used to generate live statistics which can be used
in the strategy for adaptive values. E.g. &*_mean/std are quantities
that can computed based on live predictions from the set of historical
predictions. Those values can be used in the user strategy to better
assess prediction rarity, and thus wait for probabilistically favorable
entries relative to the live historical predictions.
If the user reuses an identifier on a subsequent instance,
this function will not be called. In that case, "real" predictions
will be appended to the loaded set of historic predictions.
:param: df: DataFrame = the dataframe containing the training feature data
:param: model: Any = A model which was `fit` using a common library such as
catboost or lightgbm
:param: dk: FreqaiDataKitchen = object containing methods for data analysis
:param: pair: str = current pair
"""
num_candles = self.freqai_info.get('fit_live_predictions_candles', 600)
if not num_candles:
num_candles = 600
df_tail = df.tail(num_candles)
trained_predictions = model.predict(df_tail)
pred_df = DataFrame(trained_predictions, columns=dk.label_list)
pred_df = dk.denormalize_labels_from_metadata(pred_df)
self.dd.historic_predictions[pair] = pd.DataFrame()
self.dd.historic_predictions[pair] = copy.deepcopy(pred_df)
self.dd.historic_predictions[pair] = pred_df
hist_preds_df = self.dd.historic_predictions[pair]
for label in hist_preds_df.columns:
if hist_preds_df[label].dtype == object:
continue
hist_preds_df[f'{label}_mean'] = 0
hist_preds_df[f'{label}_std'] = 0
hist_preds_df['do_predict'] = 0
if self.freqai_info['feature_parameters'].get('DI_threshold', 0) > 0:
hist_preds_df['DI_values'] = 0
for return_str in dk.data['extra_returns_per_train']:
hist_preds_df[return_str] = 0
def fit_live_predictions(self, dk: FreqaiDataKitchen) -> None:
"""
@@ -517,13 +563,15 @@ class IFreqaiModel(ABC):
num_candles = self.freqai_info.get("fit_live_predictions_candles", 100)
dk.data["labels_mean"], dk.data["labels_std"] = {}, {}
for label in dk.label_list:
if self.dd.historic_predictions[dk.pair][label].dtype == object:
continue
f = spy.stats.norm.fit(self.dd.historic_predictions[dk.pair][label].tail(num_candles))
dk.data["labels_mean"][label], dk.data["labels_std"][label] = f[0], f[1]
return
# Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModlel.py for an example.
# See freqai/prediction_models/CatboostPredictionModel.py for an example.
@abstractmethod
def train(self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen) -> Any:
@@ -550,7 +598,7 @@ class IFreqaiModel(ABC):
@abstractmethod
def predict(
self, dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = True
) -> Tuple[DataFrame, ArrayLike]:
) -> Tuple[DataFrame, NDArray[np.int_]]:
"""
Filter the prediction features data and predict with it.
:param unfiltered_dataframe: Full dataframe for the current backtest period.
@@ -561,14 +609,3 @@ class IFreqaiModel(ABC):
:do_predict: np.array of 1s and 0s to indicate places where freqai needed to remove
data (NaNs) or felt uncertain about data (i.e. SVM and/or DI index)
"""
@abstractmethod
def return_values(self, dataframe: DataFrame) -> DataFrame:
"""
User defines the dataframe to be returned to strategy here.
:param dataframe: DataFrame = the full dataframe for the current prediction (live)
or --timerange (backtesting)
:return: dataframe: DataFrame = dataframe filled with user defined data
"""
return

16
freqtrade/freqai/prediction_models/BaseRegressionModel.py
View File

@@ -1,6 +1,7 @@
import logging
from typing import Any, Tuple
import numpy as np
import numpy.typing as npt
from pandas import DataFrame
@@ -18,15 +19,6 @@ class BaseRegressionModel(IFreqaiModel):
such as prediction_models/CatboostPredictionModel.py for guidance.
"""
def return_values(self, dataframe: DataFrame) -> DataFrame:
"""
User uses this function to add any additional return values to the dataframe.
e.g.
dataframe['volatility'] = dk.volatility_values
"""
return dataframe
def train(
self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen
) -> Any:
@@ -55,6 +47,8 @@ class BaseRegressionModel(IFreqaiModel):
f"{end_date}--------------------")
# split data into train/test data.
data_dictionary = dk.make_train_test_datasets(features_filtered, labels_filtered)
if not self.freqai_info.get('fit_live_predictions', 0) or not self.live:
dk.fit_labels()
# normalize all data based on train_dataset only
data_dictionary = dk.normalize_data(data_dictionary)
@@ -74,8 +68,6 @@ class BaseRegressionModel(IFreqaiModel):
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
self.fit_live_predictions(dk)
else:
dk.fit_labels()
self.dd.save_historic_predictions_to_disk()
@@ -85,7 +77,7 @@ class BaseRegressionModel(IFreqaiModel):
def predict(
self, unfiltered_dataframe: DataFrame, dk: FreqaiDataKitchen, first: bool = False
) -> Tuple[DataFrame, npt.ArrayLike]:
) -> Tuple[DataFrame, npt.NDArray[np.int_]]:
"""
Filter the prediction features data and predict with it.
:param: unfiltered_dataframe: Full dataframe for the current backtest period.

9
freqtrade/freqai/prediction_models/BaseTensorFlowModel.py
View File

@@ -16,15 +16,6 @@ class BaseTensorFlowModel(IFreqaiModel):
User *must* inherit from this class and set fit() and predict().
"""
def return_values(self, dataframe: DataFrame) -> DataFrame:
"""
User uses this function to add any additional return values to the dataframe.
e.g.
dataframe['volatility'] = dk.volatility_values
"""
return dataframe
def train(
self, unfiltered_dataframe: DataFrame, pair: str, dk: FreqaiDataKitchen
) -> Any:

41
freqtrade/freqai/prediction_models/CatboostClassifier.py Normal file
View File

@@ -0,0 +1,41 @@
import logging
from typing import Any, Dict
from catboost import CatBoostClassifier, Pool
from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel
logger = logging.getLogger(__name__)
class CatboostClassifier(BaseRegressionModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which
has its own DataHandler where data is held, saved, loaded, and managed.
"""
def fit(self, data_dictionary: Dict) -> Any:
"""
User sets up the training and test data to fit their desired model here
:params:
:data_dictionary: the dictionary constructed by DataHandler to hold
all the training and test data/labels.
"""
train_data = Pool(
data=data_dictionary["train_features"],
label=data_dictionary["train_labels"],
weight=data_dictionary["train_weights"],
)
cbr = CatBoostClassifier(
allow_writing_files=False,
loss_function='MultiClass',
**self.model_training_parameters,
)
cbr.fit(train_data)
return cbr

5
freqtrade/freqai/prediction_models/CatboostPredictionModel.py → freqtrade/freqai/prediction_models/CatboostRegressor.py
View File

@@ -1,6 +1,7 @@
import gc
import logging
from typing import Any, Dict
import gc
from catboost import CatBoostRegressor, Pool
from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel
@@ -9,7 +10,7 @@ from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressio
logger = logging.getLogger(__name__)
class CatboostPredictionModel(BaseRegressionModel):
class CatboostRegressor(BaseRegressionModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which

2
freqtrade/freqai/prediction_models/CatboostPredictionMultiModel.py → freqtrade/freqai/prediction_models/CatboostRegressorMultiTarget.py
View File

@@ -10,7 +10,7 @@ from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressio
logger = logging.getLogger(__name__)
class CatboostPredictionMultiModel(BaseRegressionModel):
class CatboostRegressorMultiTarget(BaseRegressionModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which

38
freqtrade/freqai/prediction_models/LightGBMClassifier.py Normal file
View File

@@ -0,0 +1,38 @@
import logging
from typing import Any, Dict
from lightgbm import LGBMClassifier
from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressionModel
logger = logging.getLogger(__name__)
class LightGBMClassifier(BaseRegressionModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which
has its own DataHandler where data is held, saved, loaded, and managed.
"""
def fit(self, data_dictionary: Dict) -> Any:
"""
User sets up the training and test data to fit their desired model here
:params:
:data_dictionary: the dictionary constructed by DataHandler to hold
all the training and test data/labels.
"""
if self.freqai_info.get('data_split_parameters', {}).get('test_size', 0.1) == 0:
eval_set = None
else:
eval_set = (data_dictionary["test_features"], data_dictionary["test_labels"])
X = data_dictionary["train_features"]
y = data_dictionary["train_labels"]
model = LGBMClassifier(**self.model_training_parameters)
model.fit(X=X, y=y, eval_set=eval_set)
return model

2
freqtrade/freqai/prediction_models/LightGBMPredictionModel.py → freqtrade/freqai/prediction_models/LightGBMRegressor.py
View File

@@ -9,7 +9,7 @@ from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressio
logger = logging.getLogger(__name__)
class LightGBMPredictionModel(BaseRegressionModel):
class LightGBMRegressor(BaseRegressionModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which

2
freqtrade/freqai/prediction_models/LightGBMPredictionMultiModel.py → freqtrade/freqai/prediction_models/LightGBMRegressorMultiTarget.py
View File

@@ -10,7 +10,7 @@ from freqtrade.freqai.prediction_models.BaseRegressionModel import BaseRegressio
logger = logging.getLogger(__name__)
class LightGBMPredictionMultiModel(BaseRegressionModel):
class LightGBMRegressorMultiTarget(BaseRegressionModel):
"""
User created prediction model. The class needs to override three necessary
functions, predict(), train(), fit(). The class inherits ModelHandler which

242
freqtrade/freqtradebot.py
View File

@@ -5,6 +5,7 @@ import copy
import logging
import traceback
from datetime import datetime, time, timedelta, timezone
from decimal import Decimal
from math import isclose
from threading import Lock
from typing import Any, Dict, List, Optional, Tuple
@@ -25,7 +26,7 @@ from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.misc import safe_value_fallback, safe_value_fallback2
from freqtrade.mixins import LoggingMixin
from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db
from freqtrade.persistence import Order, PairLocks, Trade, init_db
from freqtrade.plugins.pairlistmanager import PairListManager
from freqtrade.plugins.protectionmanager import ProtectionManager
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -149,7 +150,7 @@ class FreqtradeBot(LoggingMixin):
self.check_for_open_trades()
self.rpc.cleanup()
cleanup_db()
Trade.commit()
self.exchange.close()
def startup(self) -> None:
@@ -214,6 +215,7 @@ class FreqtradeBot(LoggingMixin):
if self.trading_mode == TradingMode.FUTURES:
self._schedule.run_pending()
Trade.commit()
self.rpc.process_msg_queue(self.dataprovider._msg_queue)
self.last_process = datetime.now(timezone.utc)
def process_stopped(self) -> None:
@@ -524,39 +526,61 @@ class FreqtradeBot(LoggingMixin):
If the strategy triggers the adjustment, a new order gets issued.
Once that completes, the existing trade is modified to match new data.
"""
if self.strategy.max_entry_position_adjustment > -1:
count_of_buys = trade.nr_of_successful_entries
if count_of_buys > self.strategy.max_entry_position_adjustment:
logger.debug(f"Max adjustment entries for {trade.pair} has been reached.")
return
else:
logger.debug("Max adjustment entries is set to unlimited.")
current_rate = self.exchange.get_rate(
trade.pair, side='entry', is_short=trade.is_short, refresh=True)
current_profit = trade.calc_profit_ratio(current_rate)
current_entry_rate, current_exit_rate = self.exchange.get_rates(
trade.pair, True, trade.is_short)
min_stake_amount = self.exchange.get_min_pair_stake_amount(trade.pair,
current_rate,
self.strategy.stoploss)
max_stake_amount = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate)
current_entry_profit = trade.calc_profit_ratio(current_entry_rate)
current_exit_profit = trade.calc_profit_ratio(current_exit_rate)
min_entry_stake = self.exchange.get_min_pair_stake_amount(trade.pair,
current_entry_rate,
self.strategy.stoploss)
min_exit_stake = self.exchange.get_min_pair_stake_amount(trade.pair,
current_exit_rate,
self.strategy.stoploss)
max_entry_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_entry_rate)
stake_available = self.wallets.get_available_stake_amount()
logger.debug(f"Calling adjust_trade_position for pair {trade.pair}")
stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
default_retval=None)(
trade=trade, current_time=datetime.now(timezone.utc), current_rate=current_rate,
current_profit=current_profit, min_stake=min_stake_amount,
max_stake=min(max_stake_amount, stake_available))
trade=trade,
current_time=datetime.now(timezone.utc), current_rate=current_entry_rate,
current_profit=current_entry_profit, min_stake=min_entry_stake,
max_stake=min(max_entry_stake, stake_available),
current_entry_rate=current_entry_rate, current_exit_rate=current_exit_rate,
current_entry_profit=current_entry_profit, current_exit_profit=current_exit_profit
)
if stake_amount is not None and stake_amount > 0.0:
# We should increase our position
self.execute_entry(trade.pair, stake_amount, price=current_rate,
if self.strategy.max_entry_position_adjustment > -1:
count_of_entries = trade.nr_of_successful_entries
if count_of_entries > self.strategy.max_entry_position_adjustment:
logger.debug(f"Max adjustment entries for {trade.pair} has been reached.")
return
else:
logger.debug("Max adjustment entries is set to unlimited.")
self.execute_entry(trade.pair, stake_amount, price=current_entry_rate,
trade=trade, is_short=trade.is_short)
if stake_amount is not None and stake_amount < 0.0:
# We should decrease our position
# TODO: Selling part of the trade not implemented yet.
logger.error(f"Unable to decrease trade position / sell partially"
f" for pair {trade.pair}, feature not implemented.")
amount = abs(float(Decimal(stake_amount) / Decimal(current_exit_rate)))
if amount > trade.amount:
# This is currently ineffective as remaining would become < min tradable
# Fixing this would require checking for 0.0 there -
# if we decide that this callback is allowed to "fully exit"
logger.info(
f"Adjusting amount to trade.amount as it is higher. {amount} > {trade.amount}")
amount = trade.amount
remaining = (trade.amount - amount) * current_exit_rate
if remaining < min_exit_stake:
logger.info(f'Remaining amount of {remaining} would be too small.')
return
self.execute_trade_exit(trade, current_exit_rate, exit_check=ExitCheckTuple(
exit_type=ExitType.PARTIAL_EXIT), sub_trade_amt=amount)
def _check_depth_of_market(self, pair: str, conf: Dict, side: SignalDirection) -> bool:
"""
@@ -600,7 +624,8 @@ class FreqtradeBot(LoggingMixin):
ordertype: Optional[str] = None,
enter_tag: Optional[str] = None,
trade: Optional[Trade] = None,
order_adjust: bool = False
order_adjust: bool = False,
leverage_: Optional[float] = None,
) -> bool:
"""
Executes a limit buy for the given pair
@@ -616,7 +641,7 @@ class FreqtradeBot(LoggingMixin):
pos_adjust = trade is not None
enter_limit_requested, stake_amount, leverage = self.get_valid_enter_price_and_stake(
pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust)
pair, price, stake_amount, trade_side, enter_tag, trade, order_adjust, leverage_)
if not stake_amount:
return False
@@ -730,7 +755,7 @@ class FreqtradeBot(LoggingMixin):
# Updating wallets
self.wallets.update()
self._notify_enter(trade, order, order_type)
self._notify_enter(trade, order_obj, order_type, sub_trade=pos_adjust)
if pos_adjust:
if order_status == 'closed':
@@ -739,8 +764,8 @@ class FreqtradeBot(LoggingMixin):
else:
logger.info(f"DCA order {order_status}, will wait for resolution: {trade}")
# Update fees if order is closed
if order_status == 'closed':
# Update fees if order is non-opened
if order_status in constants.NON_OPEN_EXCHANGE_STATES:
self.update_trade_state(trade, order_id, order)
return True
@@ -763,6 +788,7 @@ class FreqtradeBot(LoggingMixin):
entry_tag: Optional[str],
trade: Optional[Trade],
order_adjust: bool,
leverage_: Optional[float],
) -> Tuple[float, float, float]:
if price:
@@ -785,16 +811,19 @@ class FreqtradeBot(LoggingMixin):
if not enter_limit_requested:
raise PricingError('Could not determine entry price.')
if trade is None:
if self.trading_mode != TradingMode.SPOT and trade is None:
max_leverage = self.exchange.get_max_leverage(pair, stake_amount)
leverage = strategy_safe_wrapper(self.strategy.leverage, default_retval=1.0)(
pair=pair,
current_time=datetime.now(timezone.utc),
current_rate=enter_limit_requested,
proposed_leverage=1.0,
max_leverage=max_leverage,
side=trade_side, entry_tag=entry_tag,
) if self.trading_mode != TradingMode.SPOT else 1.0
if leverage_:
leverage = leverage_
else:
leverage = strategy_safe_wrapper(self.strategy.leverage, default_retval=1.0)(
pair=pair,
current_time=datetime.now(timezone.utc),
current_rate=enter_limit_requested,
proposed_leverage=1.0,
max_leverage=max_leverage,
side=trade_side, entry_tag=entry_tag,
)
# Cap leverage between 1.0 and max_leverage.
leverage = min(max(leverage, 1.0), max_leverage)
else:
@@ -829,13 +858,14 @@ class FreqtradeBot(LoggingMixin):
return enter_limit_requested, stake_amount, leverage
def _notify_enter(self, trade: Trade, order: Dict, order_type: Optional[str] = None,
fill: bool = False) -> None:
def _notify_enter(self, trade: Trade, order: Order, order_type: Optional[str] = None,
fill: bool = False, sub_trade: bool = False) -> None:
"""
Sends rpc notification when a entry order occurred.
"""
msg_type = RPCMessageType.ENTRY_FILL if fill else RPCMessageType.ENTRY
open_rate = safe_value_fallback(order, 'average', 'price')
open_rate = order.safe_price
if open_rate is None:
open_rate = trade.open_rate
@@ -859,15 +889,17 @@ class FreqtradeBot(LoggingMixin):
'stake_amount': trade.stake_amount,
'stake_currency': self.config['stake_currency'],
'fiat_currency': self.config.get('fiat_display_currency', None),
'amount': safe_value_fallback(order, 'filled', 'amount') or trade.amount,
'amount': order.safe_amount_after_fee,
'open_date': trade.open_date or datetime.utcnow(),
'current_rate': current_rate,
'sub_trade': sub_trade,
}
# Send the message
self.rpc.send_msg(msg)
def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str,
sub_trade: bool = False) -> None:
"""
Sends rpc notification when a entry order cancel occurred.
"""
@@ -892,6 +924,7 @@ class FreqtradeBot(LoggingMixin):
'open_date': trade.open_date,
'current_rate': current_rate,
'reason': reason,
'sub_trade': sub_trade,
}
# Send the message
@@ -1015,7 +1048,7 @@ class FreqtradeBot(LoggingMixin):
trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}')
logger.warning('Exiting the trade forcefully')
self.execute_trade_exit(trade, trade.stop_loss, exit_check=ExitCheckTuple(
self.execute_trade_exit(trade, stop_price, exit_check=ExitCheckTuple(
exit_type=ExitType.EMERGENCY_EXIT))
except ExchangeError:
@@ -1085,7 +1118,7 @@ class FreqtradeBot(LoggingMixin):
if (trade.is_open
and stoploss_order
and stoploss_order['status'] in ('canceled', 'cancelled')):
if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
if self.create_stoploss_order(trade=trade, stop_price=trade.stoploss_or_liquidation):
return False
else:
trade.stoploss_order_id = None
@@ -1114,7 +1147,7 @@ class FreqtradeBot(LoggingMixin):
:param order: Current on exchange stoploss order
:return: None
"""
stoploss_norm = self.exchange.price_to_precision(trade.pair, trade.stop_loss)
stoploss_norm = self.exchange.price_to_precision(trade.pair, trade.stoploss_or_liquidation)
if self.exchange.stoploss_adjust(stoploss_norm, order, side=trade.exit_side):
# we check if the update is necessary
@@ -1132,7 +1165,7 @@ class FreqtradeBot(LoggingMixin):
f"for pair {trade.pair}")
# Create new stoploss order
if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss):
if not self.create_stoploss_order(trade=trade, stop_price=stoploss_norm):
logger.warning(f"Could not create trailing stoploss order "
f"for pair {trade.pair}.")
@@ -1365,16 +1398,22 @@ class FreqtradeBot(LoggingMixin):
trade.open_order_id = None
trade.exit_reason = None
cancelled = True
self.wallets.update()
else:
# TODO: figure out how to handle partially complete sell orders
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
cancelled = False
self.wallets.update()
order_obj = trade.select_order_by_order_id(order['id'])
if not order_obj:
raise DependencyException(
f"Order_obj not found for {order['id']}. This should not have happened.")
sub_trade = order_obj.amount != trade.amount
self._notify_exit_cancel(
trade,
order_type=self.strategy.order_types['exit'],
reason=reason
reason=reason, order=order_obj, sub_trade=sub_trade
)
return cancelled
@@ -1415,6 +1454,7 @@ class FreqtradeBot(LoggingMixin):
*,
exit_tag: Optional[str] = None,
ordertype: Optional[str] = None,
sub_trade_amt: float = None,
) -> bool:
"""
Executes a trade exit for the given trade and limit
@@ -1431,14 +1471,15 @@ class FreqtradeBot(LoggingMixin):
)
exit_type = 'exit'
exit_reason = exit_tag or exit_check.exit_reason
if exit_check.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS):
if exit_check.exit_type in (
ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS, ExitType.LIQUIDATION):
exit_type = 'stoploss'
# if stoploss is on exchange and we are on dry_run mode,
# we consider the sell price stop price
if (self.config['dry_run'] and exit_type == 'stoploss'
and self.strategy.order_types['stoploss_on_exchange']):
limit = trade.stop_loss
and self.strategy.order_types['stoploss_on_exchange']):
limit = trade.stoploss_or_liquidation
# set custom_exit_price if available
proposed_limit_rate = limit
@@ -1460,14 +1501,17 @@ class FreqtradeBot(LoggingMixin):
# Emergency sells (default to market!)
order_type = self.strategy.order_types.get("emergency_exit", "market")
amount = self._safe_exit_amount(trade.pair, trade.amount)
amount = self._safe_exit_amount(trade.pair, sub_trade_amt or trade.amount)
time_in_force = self.strategy.order_time_in_force['exit']
if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
time_in_force=time_in_force, exit_reason=exit_reason,
sell_reason=exit_reason, # sellreason -> compatibility
current_time=datetime.now(timezone.utc)):
if (exit_check.exit_type != ExitType.LIQUIDATION
and not sub_trade_amt
and not strategy_safe_wrapper(
self.strategy.confirm_trade_exit, default_retval=True)(
pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
time_in_force=time_in_force, exit_reason=exit_reason,
sell_reason=exit_reason, # sellreason -> compatibility
current_time=datetime.now(timezone.utc))):
logger.info(f"User denied exit for {trade.pair}.")
return False
@@ -1501,7 +1545,7 @@ class FreqtradeBot(LoggingMixin):
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_exit(trade, order_type)
self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') in ('closed', 'expired'):
self.update_trade_state(trade, trade.open_order_id, order)
@@ -1509,16 +1553,27 @@ class FreqtradeBot(LoggingMixin):
return True
def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False) -> None:
def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False,
sub_trade: bool = False, order: Order = None) -> None:
"""
Sends rpc notification when a sell occurred.
"""
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
# Use cached rates here - it was updated seconds ago.
current_rate = self.exchange.get_rate(
trade.pair, side='exit', is_short=trade.is_short, refresh=False) if not fill else None
profit_ratio = trade.calc_profit_ratio(profit_rate)
# second condition is for mypy only; order will always be passed during sub trade
if sub_trade and order is not None:
amount = order.safe_filled if fill else order.amount
profit_rate = order.safe_price
profit = trade.calc_profit(rate=profit_rate, amount=amount, open_rate=trade.open_rate)
profit_ratio = trade.calc_profit_ratio(profit_rate, amount, trade.open_rate)
else:
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit = trade.calc_profit(rate=profit_rate) + (0.0 if fill else trade.realized_profit)
profit_ratio = trade.calc_profit_ratio(profit_rate)
amount = trade.amount
gain = "profit" if profit_ratio > 0 else "loss"
msg = {
@@ -1532,11 +1587,11 @@ class FreqtradeBot(LoggingMixin):
'gain': gain,
'limit': profit_rate,
'order_type': order_type,
'amount': trade.amount,
'amount': amount,
'open_rate': trade.open_rate,
'close_rate': trade.close_rate,
'close_rate': profit_rate,
'current_rate': current_rate,
'profit_amount': profit_trade,
'profit_amount': profit,
'profit_ratio': profit_ratio,
'buy_tag': trade.enter_tag,
'enter_tag': trade.enter_tag,
@@ -1544,19 +1599,18 @@ class FreqtradeBot(LoggingMixin):
'exit_reason': trade.exit_reason,
'open_date': trade.open_date,
'close_date': trade.close_date or datetime.utcnow(),
'stake_amount': trade.stake_amount,
'stake_currency': self.config['stake_currency'],
'fiat_currency': self.config.get('fiat_display_currency'),
'sub_trade': sub_trade,
'cumulative_profit': trade.realized_profit,
}
if 'fiat_display_currency' in self.config:
msg.update({
'fiat_currency': self.config['fiat_display_currency'],
})
# Send the message
self.rpc.send_msg(msg)
def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str,
order: Order, sub_trade: bool = False) -> None:
"""
Sends rpc notification when a sell cancel occurred.
"""
@@ -1582,7 +1636,7 @@ class FreqtradeBot(LoggingMixin):
'gain': gain,
'limit': profit_rate or 0,
'order_type': order_type,
'amount': trade.amount,
'amount': order.safe_amount_after_fee,
'open_rate': trade.open_rate,
'current_rate': current_rate,
'profit_amount': profit_trade,
@@ -1596,6 +1650,8 @@ class FreqtradeBot(LoggingMixin):
'stake_currency': self.config['stake_currency'],
'fiat_currency': self.config.get('fiat_display_currency', None),
'reason': reason,
'sub_trade': sub_trade,
'stake_amount': trade.stake_amount,
}
if 'fiat_display_currency' in self.config:
@@ -1650,41 +1706,51 @@ class FreqtradeBot(LoggingMixin):
self.handle_order_fee(trade, order_obj, order)
trade.update_trade(order_obj)
# TODO: is the below necessary? it's already done in update_trade for filled buys
trade.recalc_trade_from_orders()
Trade.commit()
if order['status'] in constants.NON_OPEN_EXCHANGE_STATES:
if order.get('status') in constants.NON_OPEN_EXCHANGE_STATES:
# If a entry order was closed, force update on stoploss on exchange
if order.get('side') == trade.entry_side:
trade = self.cancel_stoploss_on_exchange(trade)
if not self.edge:
# TODO: should shorting/leverage be supported by Edge,
# then this will need to be fixed.
trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True)
if order.get('side') == trade.entry_side or trade.amount > 0:
# Must also run for partial exits
# TODO: Margin will need to use interest_rate as well.
# interest_rate = self.exchange.get_interest_rate()
trade.set_isolated_liq(self.exchange.get_liquidation_price(
trade.set_liquidation_price(self.exchange.get_liquidation_price(
leverage=trade.leverage,
pair=trade.pair,
amount=trade.amount,
open_rate=trade.open_rate,
is_short=trade.is_short
))
if not self.edge:
# TODO: should shorting/leverage be supported by Edge,
# then this will need to be fixed.
trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True)
# Updating wallets when order is closed
self.wallets.update()
Trade.commit()
if not trade.is_open:
if send_msg and not stoploss_order and not trade.open_order_id:
self._notify_exit(trade, '', True)
self.handle_protections(trade.pair, trade.trade_direction)
elif send_msg and not trade.open_order_id and not stoploss_order:
# Enter fill
self._notify_enter(trade, order, fill=True)
self.order_close_notify(trade, order_obj, stoploss_order, send_msg)
return False
def order_close_notify(
self, trade: Trade, order: Order, stoploss_order: bool, send_msg: bool):
"""send "fill" notifications"""
sub_trade = not isclose(order.safe_amount_after_fee,
trade.amount, abs_tol=constants.MATH_CLOSE_PREC)
if order.ft_order_side == trade.exit_side:
# Exit notification
if send_msg and not stoploss_order and not trade.open_order_id:
self._notify_exit(trade, '', fill=True, sub_trade=sub_trade, order=order)
if not trade.is_open:
self.handle_protections(trade.pair, trade.trade_direction)
elif send_msg and not trade.open_order_id and not stoploss_order:
# Enter fill
self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
def handle_protections(self, pair: str, side: LongShort) -> None:
prot_trig = self.protections.stop_per_pair(pair, side=side)
if prot_trig:

149
freqtrade/optimize/backtesting.py Executable file → Normal file
View File

@@ -89,6 +89,9 @@ class Backtesting:
self.dataprovider = DataProvider(self.config, self.exchange)
if self.config.get('strategy_list'):
if self.config.get('freqai', {}).get('enabled', False):
raise OperationalException(
"You can't use strategy_list and freqai at the same time.")
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
@@ -207,8 +210,12 @@ class Backtesting:
"""
self.progress.init_step(BacktestState.DATALOAD, 1)
if self.config.get('freqai') is not None:
self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 1000))
if self.config.get('freqai', {}).get('enabled', False):
startup_candles = int(self.config.get('freqai', {}).get('startup_candles', 0))
if not startup_candles:
raise OperationalException('FreqAI backtesting module requires user set '
'startup_candles in config.')
self.required_startup += int(self.config.get('freqai', {}).get('startup_candles', 0))
logger.info(f'Increasing startup_candle_count for freqai to {self.required_startup}')
self.config['startup_candle_count'] = self.required_startup
@@ -292,8 +299,8 @@ class Backtesting:
if unavailable_pairs:
raise OperationalException(
f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. "
"It is therefore impossible to backtest with this pair at the moment.")
f"Pairs {', '.join(unavailable_pairs)} got no leverage tiers available. "
"It is therefore impossible to backtest with this pair at the moment.")
else:
self.futures_data = {}
@@ -386,7 +393,8 @@ class Backtesting:
Get close rate for backtesting result
"""
# Special handling if high or low hit STOP_LOSS or ROI
if exit.exit_type in (ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS):
if exit.exit_type in (
ExitType.STOP_LOSS, ExitType.TRAILING_STOP_LOSS, ExitType.LIQUIDATION):
return self._get_close_rate_for_stoploss(row, trade, exit, trade_dur)
elif exit.exit_type == (ExitType.ROI):
return self._get_close_rate_for_roi(row, trade, exit, trade_dur)
@@ -401,11 +409,16 @@ class Backtesting:
is_short = trade.is_short or False
leverage = trade.leverage or 1.0
side_1 = -1 if is_short else 1
if exit.exit_type == ExitType.LIQUIDATION and trade.liquidation_price:
stoploss_value = trade.liquidation_price
else:
stoploss_value = trade.stop_loss
if is_short:
if trade.stop_loss < row[LOW_IDX]:
if stoploss_value < row[LOW_IDX]:
return row[OPEN_IDX]
else:
if trade.stop_loss > row[HIGH_IDX]:
if stoploss_value > row[HIGH_IDX]:
return row[OPEN_IDX]
# Special case: trailing triggers within same candle as trade opened. Assume most
@@ -438,7 +451,7 @@ class Backtesting:
return max(row[LOW_IDX], stop_rate)
# Set close_rate to stoploss
return trade.stop_loss
return stoploss_value
def _get_close_rate_for_roi(self, row: Tuple, trade: LocalTrade, exit: ExitCheckTuple,
trade_dur: int) -> float:
@@ -502,16 +515,20 @@ class Backtesting:
def _get_adjust_trade_entry_for_candle(self, trade: LocalTrade, row: Tuple
) -> LocalTrade:
current_profit = trade.calc_profit_ratio(row[OPEN_IDX])
min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, row[OPEN_IDX], -0.1)
max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, row[OPEN_IDX])
current_rate = row[OPEN_IDX]
current_date = row[DATE_IDX].to_pydatetime()
current_profit = trade.calc_profit_ratio(current_rate)
min_stake = self.exchange.get_min_pair_stake_amount(trade.pair, current_rate, -0.1)
max_stake = self.exchange.get_max_pair_stake_amount(trade.pair, current_rate)
stake_available = self.wallets.get_available_stake_amount()
stake_amount = strategy_safe_wrapper(self.strategy.adjust_trade_position,
default_retval=None)(
trade=trade, # type: ignore[arg-type]
current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX],
current_time=current_date, current_rate=current_rate,
current_profit=current_profit, min_stake=min_stake,
max_stake=min(max_stake, stake_available))
max_stake=min(max_stake, stake_available),
current_entry_rate=current_rate, current_exit_rate=current_rate,
current_entry_profit=current_profit, current_exit_profit=current_profit)
# Check if we should increase our position
if stake_amount is not None and stake_amount > 0.0:
@@ -522,6 +539,24 @@ class Backtesting:
self.wallets.update()
return pos_trade
if stake_amount is not None and stake_amount < 0.0:
amount = abs(stake_amount) / current_rate
if amount > trade.amount:
# This is currently ineffective as remaining would become < min tradable
amount = trade.amount
remaining = (trade.amount - amount) * current_rate
if remaining < min_stake:
# Remaining stake is too low to be sold.
return trade
pos_trade = self._exit_trade(trade, row, current_rate, amount)
if pos_trade is not None:
order = pos_trade.orders[-1]
if self._get_order_filled(order.price, row):
order.close_bt_order(current_date, trade)
trade.recalc_trade_from_orders()
self.wallets.update()
return pos_trade
return trade
def _get_order_filled(self, rate: float, row: Tuple) -> bool:
@@ -572,7 +607,7 @@ class Backtesting:
if exit_.exit_type in (ExitType.EXIT_SIGNAL, ExitType.CUSTOM_EXIT):
# Checks and adds an exit tag, after checking that the length of the
# row has the length for an exit tag column
if(
if (
len(row) > EXIT_TAG_IDX
and row[EXIT_TAG_IDX] is not None
and len(row[EXIT_TAG_IDX]) > 0
@@ -597,46 +632,53 @@ class Backtesting:
# Confirm trade exit:
time_in_force = self.strategy.order_time_in_force['exit']
if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
if (exit_.exit_type != ExitType.LIQUIDATION and not strategy_safe_wrapper(
self.strategy.confirm_trade_exit, default_retval=True)(
pair=trade.pair,
trade=trade, # type: ignore[arg-type]
order_type='limit',
order_type=order_type,
amount=trade.amount,
rate=close_rate,
time_in_force=time_in_force,
sell_reason=exit_reason, # deprecated
exit_reason=exit_reason,
current_time=exit_candle_time):
current_time=exit_candle_time)):
return None
trade.exit_reason = exit_reason
self.order_id_counter += 1
order = Order(
id=self.order_id_counter,
ft_trade_id=trade.id,
order_date=exit_candle_time,
order_update_date=exit_candle_time,
ft_is_open=True,
ft_pair=trade.pair,
order_id=str(self.order_id_counter),
symbol=trade.pair,
ft_order_side=trade.exit_side,
side=trade.exit_side,
order_type=order_type,
status="open",
price=close_rate,
average=close_rate,
amount=trade.amount,
filled=0,
remaining=trade.amount,
cost=trade.amount * close_rate,
)
trade.orders.append(order)
return trade
return self._exit_trade(trade, row, close_rate, trade.amount)
return None
def _exit_trade(self, trade: LocalTrade, sell_row: Tuple,
close_rate: float, amount: float = None) -> Optional[LocalTrade]:
self.order_id_counter += 1
exit_candle_time = sell_row[DATE_IDX].to_pydatetime()
order_type = self.strategy.order_types['exit']
amount = amount or trade.amount
order = Order(
id=self.order_id_counter,
ft_trade_id=trade.id,
order_date=exit_candle_time,
order_update_date=exit_candle_time,
ft_is_open=True,
ft_pair=trade.pair,
order_id=str(self.order_id_counter),
symbol=trade.pair,
ft_order_side=trade.exit_side,
side=trade.exit_side,
order_type=order_type,
status="open",
price=close_rate,
average=close_rate,
amount=amount,
filled=0,
remaining=amount,
cost=amount * close_rate,
)
trade.orders.append(order)
return trade
def _get_exit_trade_entry(self, trade: LocalTrade, row: Tuple) -> Optional[LocalTrade]:
exit_candle_time: datetime = row[DATE_IDX].to_pydatetime()
@@ -812,7 +854,7 @@ class Backtesting:
trade.adjust_stop_loss(trade.open_rate, self.strategy.stoploss, initial=True)
trade.set_isolated_liq(self.exchange.get_liquidation_price(
trade.set_liquidation_price(self.exchange.get_liquidation_price(
pair=pair,
open_rate=propose_rate,
amount=amount,
@@ -863,6 +905,8 @@ class Backtesting:
# Ignore trade if entry-order did not fill yet
continue
exit_row = data[pair][-1]
self._exit_trade(trade, exit_row, exit_row[OPEN_IDX], trade.amount)
trade.orders[-1].close_bt_order(exit_row[DATE_IDX].to_pydatetime(), trade)
trade.close_date = exit_row[DATE_IDX].to_pydatetime()
trade.exit_reason = ExitType.FORCE_EXIT.value
@@ -1004,7 +1048,7 @@ class Backtesting:
return None
return row
def backtest(self, processed: Dict,
def backtest(self, processed: Dict, # noqa: max-complexity: 13
start_date: datetime, end_date: datetime,
max_open_trades: int = 0, position_stacking: bool = False,
enable_protections: bool = False) -> Dict[str, Any]:
@@ -1106,14 +1150,19 @@ class Backtesting:
if order and self._get_order_filled(order.price, row):
order.close_bt_order(current_time, trade)
trade.open_order_id = None
trade.close_date = current_time
trade.close(order.price, show_msg=False)
sub_trade = order.safe_amount_after_fee != trade.amount
if sub_trade:
order.close_bt_order(current_time, trade)
trade.recalc_trade_from_orders()
else:
trade.close_date = current_time
trade.close(order.price, show_msg=False)
# logger.debug(f"{pair} - Backtesting exit {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
LocalTrade.close_bt_trade(trade)
trades.append(trade)
# logger.debug(f"{pair} - Backtesting exit {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
LocalTrade.close_bt_trade(trade)
trades.append(trade)
self.wallets.update()
self.run_protections(
enable_protections, pair, current_time, trade.trade_direction)

2
freqtrade/optimize/optimize_reports.py
View File

@@ -639,7 +639,7 @@ def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_curr
:param stake_currency: stake-currency - used to correctly name headers
:return: pretty printed table with tabulate as string
"""
if(tag_type == "enter_tag"):
if (tag_type == "enter_tag"):
headers = _get_line_header("TAG", stake_currency)
else:
headers = _get_line_header("TAG", stake_currency, 'Sells')

2
freqtrade/persistence/__init__.py
View File

@@ -1,5 +1,5 @@
# flake8: noqa: F401
from freqtrade.persistence.models import cleanup_db, init_db
from freqtrade.persistence.models import init_db
from freqtrade.persistence.pairlock_middleware import PairLocks
from freqtrade.persistence.trade_model import LocalTrade, Order, Trade

10
freqtrade/persistence/migrations.py
View File

@@ -95,6 +95,7 @@ def migrate_trades_and_orders_table(
exit_reason = get_column_def(cols, 'sell_reason', get_column_def(cols, 'exit_reason', 'null'))
strategy = get_column_def(cols, 'strategy', 'null')
enter_tag = get_column_def(cols, 'buy_tag', get_column_def(cols, 'enter_tag', 'null'))
realized_profit = get_column_def(cols, 'realized_profit', '0.0')
trading_mode = get_column_def(cols, 'trading_mode', 'null')
@@ -155,7 +156,7 @@ def migrate_trades_and_orders_table(
max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag,
timeframe, open_trade_value, close_profit_abs,
trading_mode, leverage, liquidation_price, is_short,
interest_rate, funding_fees
interest_rate, funding_fees, realized_profit
)
select id, lower(exchange), pair, {base_currency} base_currency,
{stake_currency} stake_currency,
@@ -181,7 +182,7 @@ def migrate_trades_and_orders_table(
{open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs,
{trading_mode} trading_mode, {leverage} leverage, {liquidation_price} liquidation_price,
{is_short} is_short, {interest_rate} interest_rate,
{funding_fees} funding_fees
{funding_fees} funding_fees, {realized_profit} realized_profit
from {trade_back_name}
"""))
@@ -297,8 +298,9 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
# Check if migration necessary
# Migrates both trades and orders table!
if not has_column(cols_orders, 'stop_price'):
# if not has_column(cols_trades, 'base_currency'):
# if ('orders' not in previous_tables
# or not has_column(cols_orders, 'stop_price')):
if not has_column(cols_trades, 'realized_profit'):
logger.info(f"Running database migration for trades - "
f"backup: {table_back_name}, {order_table_bak_name}")
migrate_trades_and_orders_table(

8
freqtrade/persistence/models.py
View File

@@ -61,11 +61,3 @@ def init_db(db_url: str) -> None:
previous_tables = inspect(engine).get_table_names()
_DECL_BASE.metadata.create_all(engine)
check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables)
def cleanup_db() -> None:
"""
Flushes all pending operations to disk.
:return: None
"""
Trade.commit()

213
freqtrade/persistence/trade_model.py
View File

@@ -4,13 +4,15 @@ This module contains the class to persist trades into SQLite
import logging
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from math import isclose
from typing import Any, Dict, List, Optional
from sqlalchemy import (Boolean, Column, DateTime, Enum, Float, ForeignKey, Integer, String,
UniqueConstraint, desc, func)
from sqlalchemy.orm import Query, lazyload, relationship
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort
from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES,
BuySell, LongShort)
from freqtrade.enums import ExitType, TradingMode
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.leverage import interest
@@ -176,10 +178,9 @@ class Order(_DECL_BASE):
self.remaining = 0
self.status = 'closed'
self.ft_is_open = False
if (self.ft_order_side == trade.entry_side
and len(trade.select_filled_orders(trade.entry_side)) == 1):
if (self.ft_order_side == trade.entry_side):
trade.open_rate = self.price
trade.recalc_open_trade_value()
trade.recalc_trade_from_orders()
trade.adjust_stop_loss(trade.open_rate, trade.stop_loss_pct, refresh=True)
@staticmethod
@@ -195,7 +196,7 @@ class Order(_DECL_BASE):
if filtered_orders:
oobj = filtered_orders[0]
oobj.update_from_ccxt_object(order)
Order.query.session.commit()
Trade.commit()
else:
logger.warning(f"Did not find order for {order}.")
@@ -237,6 +238,7 @@ class LocalTrade():
trades: List['LocalTrade'] = []
trades_open: List['LocalTrade'] = []
total_profit: float = 0
realized_profit: float = 0
id: int = 0
@@ -302,6 +304,16 @@ class LocalTrade():
# Futures properties
funding_fees: Optional[float] = None
@property
def stoploss_or_liquidation(self) -> float:
if self.liquidation_price:
if self.is_short:
return min(self.stop_loss, self.liquidation_price)
else:
return max(self.stop_loss, self.liquidation_price)
return self.stop_loss
@property
def buy_tag(self) -> Optional[str]:
"""
@@ -437,6 +449,7 @@ class LocalTrade():
if self.close_date else None),
'close_timestamp': int(self.close_date.replace(
tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None,
'realized_profit': self.realized_profit or 0.0,
'close_rate': self.close_rate,
'close_rate_requested': self.close_rate_requested,
'close_profit': self.close_profit, # Deprecated
@@ -497,7 +510,7 @@ class LocalTrade():
self.max_rate = max(current_price, self.max_rate or self.open_rate)
self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
def set_isolated_liq(self, liquidation_price: Optional[float]):
def set_liquidation_price(self, liquidation_price: Optional[float]):
"""
Method you should use to set self.liquidation price.
Assures stop_loss is not passed the liquidation price
@@ -506,22 +519,13 @@ class LocalTrade():
return
self.liquidation_price = liquidation_price
def _set_stop_loss(self, stop_loss: float, percent: float):
def __set_stop_loss(self, stop_loss: float, percent: float):
"""
Method you should use to set self.stop_loss.
Assures stop_loss is not passed the liquidation price
Method used internally to set self.stop_loss.
"""
if self.liquidation_price is not None:
if self.is_short:
sl = min(stop_loss, self.liquidation_price)
else:
sl = max(stop_loss, self.liquidation_price)
else:
sl = stop_loss
if not self.stop_loss:
self.initial_stop_loss = sl
self.stop_loss = sl
self.initial_stop_loss = stop_loss
self.stop_loss = stop_loss
self.stop_loss_pct = -1 * abs(percent)
self.stoploss_last_update = datetime.utcnow()
@@ -543,18 +547,12 @@ class LocalTrade():
leverage = self.leverage or 1.0
if self.is_short:
new_loss = float(current_price * (1 + abs(stoploss / leverage)))
# If trading with leverage, don't set the stoploss below the liquidation price
if self.liquidation_price:
new_loss = min(self.liquidation_price, new_loss)
else:
new_loss = float(current_price * (1 - abs(stoploss / leverage)))
# If trading with leverage, don't set the stoploss below the liquidation price
if self.liquidation_price:
new_loss = max(self.liquidation_price, new_loss)
# no stop loss assigned yet
if self.initial_stop_loss_pct is None or refresh:
self._set_stop_loss(new_loss, stoploss)
self.__set_stop_loss(new_loss, stoploss)
self.initial_stop_loss = new_loss
self.initial_stop_loss_pct = -1 * abs(stoploss)
@@ -569,7 +567,7 @@ class LocalTrade():
# ? decreasing the minimum stoploss
if (higher_stop and not self.is_short) or (lower_stop and self.is_short):
logger.debug(f"{self.pair} - Adjusting stoploss...")
self._set_stop_loss(new_loss, stoploss)
self.__set_stop_loss(new_loss, stoploss)
else:
logger.debug(f"{self.pair} - Keeping current stoploss...")
@@ -601,14 +599,28 @@ class LocalTrade():
if self.is_open:
payment = "SELL" if self.is_short else "BUY"
logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
self.open_order_id = None
# condition to avoid reset value when updating fees
if self.open_order_id == order.order_id:
self.open_order_id = None
else:
logger.warning(
f'Got different open_order_id {self.open_order_id} != {order.order_id}')
self.recalc_trade_from_orders()
elif order.ft_order_side == self.exit_side:
if self.is_open:
payment = "BUY" if self.is_short else "SELL"
# * On margin shorts, you buy a little bit more than the amount (amount + interest)
logger.info(f'{order.order_type.upper()}_{payment} has been fulfilled for {self}.')
self.close(order.safe_price)
# condition to avoid reset value when updating fees
if self.open_order_id == order.order_id:
self.open_order_id = None
else:
logger.warning(
f'Got different open_order_id {self.open_order_id} != {order.order_id}')
if isclose(order.safe_amount_after_fee, self.amount, abs_tol=MATH_CLOSE_PREC):
self.close(order.safe_price)
else:
self.recalc_trade_from_orders()
elif order.ft_order_side == 'stoploss':
self.stoploss_order_id = None
self.close_rate_requested = self.stop_loss
@@ -627,11 +639,11 @@ class LocalTrade():
"""
self.close_rate = rate
self.close_date = self.close_date or datetime.utcnow()
self.close_profit = self.calc_profit_ratio(rate)
self.close_profit_abs = self.calc_profit(rate)
self.close_profit_abs = self.calc_profit(rate) + self.realized_profit
self.is_open = False
self.exit_order_status = 'closed'
self.open_order_id = None
self.recalc_trade_from_orders(is_closing=True)
if show_msg:
logger.info(
'Marking %s as closed as the trade is fulfilled and found no open orders for it.',
@@ -677,12 +689,12 @@ class LocalTrade():
"""
return len([o for o in self.orders if o.ft_order_side == self.exit_side])
def _calc_open_trade_value(self) -> float:
def _calc_open_trade_value(self, amount: float, open_rate: float) -> float:
"""
Calculate the open_rate including open_fee.
:return: Price in of the open trade incl. Fees
"""
open_trade = Decimal(self.amount) * Decimal(self.open_rate)
open_trade = Decimal(amount) * Decimal(open_rate)
fees = open_trade * Decimal(self.fee_open)
if self.is_short:
return float(open_trade - fees)
@@ -694,7 +706,7 @@ class LocalTrade():
Recalculate open_trade_value.
Must be called whenever open_rate, fee_open is changed.
"""
self.open_trade_value = self._calc_open_trade_value()
self.open_trade_value = self._calc_open_trade_value(self.amount, self.open_rate)
def calculate_interest(self) -> Decimal:
"""
@@ -726,7 +738,7 @@ class LocalTrade():
else:
return close_trade - fees
def calc_close_trade_value(self, rate: float) -> float:
def calc_close_trade_value(self, rate: float, amount: float = None) -> float:
"""
Calculate the Trade's close value including fees
:param rate: rate to compare with.
@@ -735,96 +747,143 @@ class LocalTrade():
if rate is None and not self.close_rate:
return 0.0
amount = Decimal(self.amount)
amount1 = Decimal(amount or self.amount)
trading_mode = self.trading_mode or TradingMode.SPOT
if trading_mode == TradingMode.SPOT:
return float(self._calc_base_close(amount, rate, self.fee_close))
return float(self._calc_base_close(amount1, rate, self.fee_close))
elif (trading_mode == TradingMode.MARGIN):
total_interest = self.calculate_interest()
if self.is_short:
amount = amount + total_interest
return float(self._calc_base_close(amount, rate, self.fee_close))
amount1 = amount1 + total_interest
return float(self._calc_base_close(amount1, rate, self.fee_close))
else:
# Currency already owned for longs, no need to purchase
return float(self._calc_base_close(amount, rate, self.fee_close) - total_interest)
return float(self._calc_base_close(amount1, rate, self.fee_close) - total_interest)
elif (trading_mode == TradingMode.FUTURES):
funding_fees = self.funding_fees or 0.0
# Positive funding_fees -> Trade has gained from fees.
# Negative funding_fees -> Trade had to pay the fees.
if self.is_short:
return float(self._calc_base_close(amount, rate, self.fee_close)) - funding_fees
return float(self._calc_base_close(amount1, rate, self.fee_close)) - funding_fees
else:
return float(self._calc_base_close(amount, rate, self.fee_close)) + funding_fees
return float(self._calc_base_close(amount1, rate, self.fee_close)) + funding_fees
else:
raise OperationalException(
f"{self.trading_mode.value} trading is not yet available using freqtrade")
def calc_profit(self, rate: float) -> float:
def calc_profit(self, rate: float, amount: float = None, open_rate: float = None) -> float:
"""
Calculate the absolute profit in stake currency between Close and Open trade
:param rate: close rate to compare with.
:param amount: Amount to use for the calculation. Falls back to trade.amount if not set.
:param open_rate: open_rate to use. Defaults to self.open_rate if not provided.
:return: profit in stake currency as float
"""
close_trade_value = self.calc_close_trade_value(rate)
close_trade_value = self.calc_close_trade_value(rate, amount)
if amount is None or open_rate is None:
open_trade_value = self.open_trade_value
else:
open_trade_value = self._calc_open_trade_value(amount, open_rate)
if self.is_short:
profit = self.open_trade_value - close_trade_value
profit = open_trade_value - close_trade_value
else:
profit = close_trade_value - self.open_trade_value
profit = close_trade_value - open_trade_value
return float(f"{profit:.8f}")
def calc_profit_ratio(self, rate: float) -> float:
def calc_profit_ratio(
self, rate: float, amount: float = None, open_rate: float = None) -> float:
"""
Calculates the profit as ratio (including fee).
:param rate: rate to compare with.
:param amount: Amount to use for the calculation. Falls back to trade.amount if not set.
:param open_rate: open_rate to use. Defaults to self.open_rate if not provided.
:return: profit ratio as float
"""
close_trade_value = self.calc_close_trade_value(rate)
close_trade_value = self.calc_close_trade_value(rate, amount)
if amount is None or open_rate is None:
open_trade_value = self.open_trade_value
else:
open_trade_value = self._calc_open_trade_value(amount, open_rate)
short_close_zero = (self.is_short and close_trade_value == 0.0)
long_close_zero = (not self.is_short and self.open_trade_value == 0.0)
long_close_zero = (not self.is_short and open_trade_value == 0.0)
leverage = self.leverage or 1.0
if (short_close_zero or long_close_zero):
return 0.0
else:
if self.is_short:
profit_ratio = (1 - (close_trade_value / self.open_trade_value)) * leverage
profit_ratio = (1 - (close_trade_value / open_trade_value)) * leverage
else:
profit_ratio = ((close_trade_value / self.open_trade_value) - 1) * leverage
profit_ratio = ((close_trade_value / open_trade_value) - 1) * leverage
return float(f"{profit_ratio:.8f}")
def recalc_trade_from_orders(self):
def recalc_trade_from_orders(self, is_closing: bool = False):
current_amount = 0.0
current_stake = 0.0
total_stake = 0.0 # Total stake after all buy orders (does not subtract!)
avg_price = 0.0
close_profit = 0.0
close_profit_abs = 0.0
total_amount = 0.0
total_stake = 0.0
for o in self.orders:
if (o.ft_is_open or
(o.ft_order_side != self.entry_side) or
(o.status not in NON_OPEN_EXCHANGE_STATES)):
if o.ft_is_open or not o.filled:
continue
tmp_amount = o.safe_amount_after_fee
tmp_price = o.average or o.price
if tmp_amount > 0.0 and tmp_price is not None:
total_amount += tmp_amount
total_stake += tmp_price * tmp_amount
tmp_price = o.safe_price
if total_amount > 0:
is_exit = o.ft_order_side != self.entry_side
side = -1 if is_exit else 1
if tmp_amount > 0.0 and tmp_price is not None:
current_amount += tmp_amount * side
price = avg_price if is_exit else tmp_price
current_stake += price * tmp_amount * side
if current_amount > 0:
avg_price = current_stake / current_amount
if is_exit:
# Process partial exits
exit_rate = o.safe_price
exit_amount = o.safe_amount_after_fee
profit = self.calc_profit(rate=exit_rate, amount=exit_amount, open_rate=avg_price)
close_profit_abs += profit
close_profit = self.calc_profit_ratio(
exit_rate, amount=exit_amount, open_rate=avg_price)
if current_amount <= 0:
profit = close_profit_abs
else:
total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price)
if close_profit:
self.close_profit = close_profit
self.realized_profit = close_profit_abs
self.close_profit_abs = profit
if current_amount > 0:
# Trade is still open
# Leverage not updated, as we don't allow changing leverage through DCA at the moment.
self.open_rate = total_stake / total_amount
self.stake_amount = total_stake / (self.leverage or 1.0)
self.amount = total_amount
self.fee_open_cost = self.fee_open * total_stake
self.open_rate = current_stake / current_amount
self.stake_amount = current_stake / (self.leverage or 1.0)
self.amount = current_amount
self.fee_open_cost = self.fee_open * current_stake
self.recalc_open_trade_value()
if self.stop_loss_pct is not None and self.open_rate is not None:
self.adjust_stop_loss(self.open_rate, self.stop_loss_pct)
elif is_closing and total_stake > 0:
# Close profit abs / maximum owned
# Fees are considered as they are part of close_profit_abs
self.close_profit = (close_profit_abs / total_stake) * self.leverage
def select_order_by_order_id(self, order_id: str) -> Optional[Order]:
"""
@@ -846,7 +905,7 @@ class LocalTrade():
"""
orders = self.orders
if order_side:
orders = [o for o in self.orders if o.ft_order_side == order_side]
orders = [o for o in orders if o.ft_order_side == order_side]
if is_open is not None:
orders = [o for o in orders if o.ft_is_open == is_open]
if len(orders) > 0:
@@ -861,9 +920,9 @@ class LocalTrade():
:return: array of Order objects
"""
return [o for o in self.orders if ((o.ft_order_side == order_side) or (order_side is None))
and o.ft_is_open is False and
(o.filled or 0) > 0 and
o.status in NON_OPEN_EXCHANGE_STATES]
and o.ft_is_open is False
and o.filled
and o.status in NON_OPEN_EXCHANGE_STATES]
def select_filled_or_open_orders(self) -> List['Order']:
"""
@@ -1028,6 +1087,7 @@ class Trade(_DECL_BASE, LocalTrade):
open_trade_value = Column(Float)
close_rate: Optional[float] = Column(Float)
close_rate_requested = Column(Float)
realized_profit = Column(Float, default=0.0)
close_profit = Column(Float)
close_profit_abs = Column(Float)
stake_amount = Column(Float, nullable=False)
@@ -1073,6 +1133,7 @@ class Trade(_DECL_BASE, LocalTrade):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.realized_profit = 0
self.recalc_open_trade_value()
def delete(self) -> None:
@@ -1087,6 +1148,10 @@ class Trade(_DECL_BASE, LocalTrade):
def commit():
Trade.query.session.commit()
@staticmethod
def rollback():
Trade.query.session.rollback()
@staticmethod
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
open_date: datetime = None, close_date: datetime = None,
@@ -1239,7 +1304,7 @@ class Trade(_DECL_BASE, LocalTrade):
"""
filters = [Trade.is_open.is_(False)]
if(pair is not None):
if (pair is not None):
filters.append(Trade.pair == pair)
enter_tag_perf = Trade.query.with_entities(
@@ -1272,7 +1337,7 @@ class Trade(_DECL_BASE, LocalTrade):
"""
filters = [Trade.is_open.is_(False)]
if(pair is not None):
if (pair is not None):
filters.append(Trade.pair == pair)
sell_tag_perf = Trade.query.with_entities(
@@ -1305,7 +1370,7 @@ class Trade(_DECL_BASE, LocalTrade):
"""
filters = [Trade.is_open.is_(False)]
if(pair is not None):
if (pair is not None):
filters.append(Trade.pair == pair)
mix_tag_perf = Trade.query.with_entities(
@@ -1325,7 +1390,7 @@ class Trade(_DECL_BASE, LocalTrade):
enter_tag = enter_tag if enter_tag is not None else "Other"
exit_reason = exit_reason if exit_reason is not None else "Other"
if(exit_reason is not None and enter_tag is not None):
if (exit_reason is not None and enter_tag is not None):
mix_tag = enter_tag + " " + exit_reason
i = 0
if not any(item["mix_tag"] == mix_tag for item in return_list):

2
freqtrade/plugins/pairlist/AgeFilter.py
View File

@@ -8,11 +8,11 @@ from typing import Any, Dict, List, Optional
import arrow
from pandas import DataFrame
from freqtrade.configuration import PeriodicCache
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
from freqtrade.util import PeriodicCache
logger = logging.getLogger(__name__)

10
freqtrade/plugins/pairlist/pairlist_helpers.py
View File

@@ -43,12 +43,10 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
def dynamic_expand_pairlist(config: dict, markets: list) -> List[str]:
if config.get('freqai', {}):
expanded_pairs = expand_pairlist(config['pairs'], markets)
if config.get('freqai', {}).get('enabled', False):
corr_pairlist = config['freqai']['feature_parameters']['include_corr_pairlist']
full_pairs = config['pairs'] + [pair for pair in corr_pairlist
if pair not in config['pairs']]
expanded_pairs = expand_pairlist(full_pairs, markets)
else:
expanded_pairs = expand_pairlist(config['pairs'], markets)
expanded_pairs += [pair for pair in corr_pairlist
if pair not in config['pairs']]
return expanded_pairs

2
freqtrade/plugins/protections/stoploss_guard.py
View File

@@ -49,7 +49,7 @@ class StoplossGuard(IProtection):
trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until)
trades = [trade for trade in trades1 if (str(trade.exit_reason) in (
ExitType.TRAILING_STOP_LOSS.value, ExitType.STOP_LOSS.value,
ExitType.STOPLOSS_ON_EXCHANGE.value)
ExitType.STOPLOSS_ON_EXCHANGE.value, ExitType.LIQUIDATION.value)
and trade.close_profit and trade.close_profit < self._profit_limit)]
if self._only_per_side:

6
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -194,11 +194,11 @@ class OrderSchema(BaseModel):
pair: str
order_id: str
status: str
remaining: float
remaining: Optional[float]
amount: float
safe_price: float
cost: float
filled: float
filled: Optional[float]
ft_order_side: str
order_type: str
is_open: bool
@@ -325,11 +325,13 @@ class ForceEnterPayload(BaseModel):
ordertype: Optional[OrderTypeValues]
stakeamount: Optional[float]
entry_tag: Optional[str]
leverage: Optional[float]
class ForceExitPayload(BaseModel):
tradeid: str
ordertype: Optional[OrderTypeValues]
amount: Optional[float]
class BlacklistPayload(BaseModel):

12
freqtrade/rpc/api_server/api_v1.py
View File

@@ -37,7 +37,8 @@ logger = logging.getLogger(__name__)
# 2.14: Add entry/exit orders to trade response
# 2.15: Add backtest history endpoints
# 2.16: Additional daily metrics
API_VERSION = 2.16
# 2.17: Forceentry - leverage, partial force_exit
API_VERSION = 2.17
# Public API, requires no auth.
router_public = APIRouter()
@@ -142,12 +143,11 @@ def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(g
@router.post('/forcebuy', response_model=ForceEnterResponse, tags=['trading'])
def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)):
ordertype = payload.ordertype.value if payload.ordertype else None
stake_amount = payload.stakeamount if payload.stakeamount else None
entry_tag = payload.entry_tag if payload.entry_tag else 'force_entry'
trade = rpc._rpc_force_entry(payload.pair, payload.price, order_side=payload.side,
order_type=ordertype, stake_amount=stake_amount,
enter_tag=entry_tag)
order_type=ordertype, stake_amount=payload.stakeamount,
enter_tag=payload.entry_tag or 'force_entry',
leverage=payload.leverage)
if trade:
return ForceEnterResponse.parse_obj(trade.to_json())
@@ -161,7 +161,7 @@ def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)):
@router.post('/forcesell', response_model=ResultMsg, tags=['trading'])
def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)):
ordertype = payload.ordertype.value if payload.ordertype else None
return rpc._rpc_force_exit(payload.tradeid, ordertype)
return rpc._rpc_force_exit(payload.tradeid, ordertype, amount=payload.amount)
@router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist'])

4
freqtrade/rpc/api_server/deps.py
View File

@@ -18,9 +18,9 @@ def get_rpc_optional() -> Optional[RPC]:
def get_rpc() -> Optional[Iterator[RPC]]:
_rpc = get_rpc_optional()
if _rpc:
Trade.query.session.rollback()
Trade.rollback()
yield _rpc
Trade.query.session.rollback()
Trade.rollback()
else:
raise RPCException('Bot is not in the correct state')

7
freqtrade/rpc/api_server/web_ui.py
View File

@@ -1,4 +1,5 @@
from pathlib import Path
from typing import Optional
from fastapi import APIRouter
from fastapi.exceptions import HTTPException
@@ -50,8 +51,12 @@ async def index_html(rest_of_path: str):
filename = uibase / rest_of_path
# It's security relevant to check "relative_to".
# Without this, Directory-traversal is possible.
media_type: Optional[str] = None
if filename.suffix == '.js':
# Force text/javascript for .js files - Circumvent faulty system configuration
media_type = 'application/javascript'
if filename.is_file() and is_relative_to(filename, uibase):
return FileResponse(str(filename))
return FileResponse(str(filename), media_type=media_type)
index_file = uibase / 'index.html'
if not index_file.is_file():

8
freqtrade/rpc/fiat_convert.py
View File

@@ -12,6 +12,7 @@ from pycoingecko import CoinGeckoAPI
from requests.exceptions import RequestException
from freqtrade.constants import SUPPORTED_FIAT
from freqtrade.mixins.logging_mixin import LoggingMixin
logger = logging.getLogger(__name__)
@@ -27,7 +28,7 @@ coingecko_mapping = {
}
class CryptoToFiatConverter:
class CryptoToFiatConverter(LoggingMixin):
"""
Main class to initiate Crypto to FIAT.
This object contains a list of pair Crypto, FIAT
@@ -54,6 +55,7 @@ class CryptoToFiatConverter:
# Timeout: 6h
self._pair_price: TTLCache = TTLCache(maxsize=500, ttl=6 * 60 * 60)
LoggingMixin.__init__(self, logger, 3600)
self._load_cryptomap()
def _load_cryptomap(self) -> None:
@@ -177,7 +179,9 @@ class CryptoToFiatConverter:
if not _gekko_id:
# return 0 for unsupported stake currencies (fiat-convert should not break the bot)
logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)
self.log_once(
f"unsupported crypto-symbol {crypto_symbol.upper()} - returning 0.0",
logger.warning)
return 0.0
try:

116
freqtrade/rpc/rpc.py
View File

@@ -179,8 +179,10 @@ class RPC:
else:
current_rate = trade.close_rate
if len(trade.select_filled_orders(trade.entry_side)) > 0:
current_profit = trade.calc_profit_ratio(current_rate)
current_profit_abs = trade.calc_profit(current_rate)
current_profit = trade.calc_profit_ratio(
current_rate) if not isnan(current_rate) else NAN
current_profit_abs = trade.calc_profit(
current_rate) if not isnan(current_rate) else NAN
current_profit_fiat: Optional[float] = None
# Calculate fiat profit
if self._fiat_converter:
@@ -201,7 +203,7 @@ class RPC:
trade_dict = trade.to_json()
trade_dict.update(dict(
close_profit=trade.close_profit if trade.close_profit is not None else None,
close_profit=trade.close_profit if not trade.is_open else None,
current_rate=current_rate,
current_profit=current_profit, # Deprecated
current_profit_pct=round(current_profit * 100, 2), # Deprecated
@@ -239,12 +241,15 @@ class RPC:
trade.pair, side='exit', is_short=trade.is_short, refresh=False)
except (PricingError, ExchangeError):
current_rate = NAN
if len(trade.select_filled_orders(trade.entry_side)) > 0:
trade_profit = trade.calc_profit(current_rate)
profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}'
trade_profit = NAN
profit_str = f'{NAN:.2%}'
else:
trade_profit = 0.0
profit_str = f'{0.0:.2f}'
if trade.nr_of_successful_entries > 0:
trade_profit = trade.calc_profit(current_rate)
profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}'
else:
trade_profit = 0.0
profit_str = f'{0.0:.2f}'
direction_str = ('S' if trade.is_short else 'L') if nonspot else ''
if self._fiat_converter:
fiat_profit = self._fiat_converter.convert_amount(
@@ -424,21 +429,20 @@ class RPC:
for trade in trades:
current_rate: float = 0.0
if not trade.open_rate:
continue
if trade.close_date:
durations.append((trade.close_date - trade.open_date).total_seconds())
if not trade.is_open:
profit_ratio = trade.close_profit
profit_closed_coin.append(trade.close_profit_abs)
profit_abs = trade.close_profit_abs
profit_closed_coin.append(profit_abs)
profit_closed_ratio.append(profit_ratio)
if trade.close_profit >= 0:
winning_trades += 1
winning_profit += trade.close_profit_abs
winning_profit += profit_abs
else:
losing_trades += 1
losing_profit += trade.close_profit_abs
losing_profit += profit_abs
else:
# Get current rate
try:
@@ -446,11 +450,15 @@ class RPC:
trade.pair, side='exit', is_short=trade.is_short, refresh=False)
except (PricingError, ExchangeError):
current_rate = NAN
profit_ratio = trade.calc_profit_ratio(rate=current_rate)
if isnan(current_rate):
profit_ratio = NAN
profit_abs = NAN
else:
profit_ratio = trade.calc_profit_ratio(rate=current_rate)
profit_abs = trade.calc_profit(
rate=trade.close_rate or current_rate) + trade.realized_profit
profit_all_coin.append(
trade.calc_profit(rate=trade.close_rate or current_rate)
)
profit_all_coin.append(profit_abs)
profit_all_ratio.append(profit_ratio)
best_pair = Trade.get_best_pair(start_date)
@@ -659,36 +667,48 @@ class RPC:
return {'status': 'No more buy will occur from now. Run /reload_config to reset.'}
def _rpc_force_exit(self, trade_id: str, ordertype: Optional[str] = None) -> Dict[str, str]:
def __exec_force_exit(self, trade: Trade, ordertype: Optional[str],
amount: Optional[float] = None) -> None:
# Check if there is there is an open order
fully_canceled = False
if trade.open_order_id:
order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
if order['side'] == trade.entry_side:
fully_canceled = self._freqtrade.handle_cancel_enter(
trade, order, CANCEL_REASON['FORCE_EXIT'])
if order['side'] == trade.exit_side:
# Cancel order - so it is placed anew with a fresh price.
self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT'])
if not fully_canceled:
# Get current rate and execute sell
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, side='exit', is_short=trade.is_short, refresh=True)
exit_check = ExitCheckTuple(exit_type=ExitType.FORCE_EXIT)
order_type = ordertype or self._freqtrade.strategy.order_types.get(
"force_exit", self._freqtrade.strategy.order_types["exit"])
sub_amount: Optional[float] = None
if amount and amount < trade.amount:
# Partial exit ...
min_exit_stake = self._freqtrade.exchange.get_min_pair_stake_amount(
trade.pair, current_rate, trade.stop_loss_pct)
remaining = (trade.amount - amount) * current_rate
if remaining < min_exit_stake:
raise RPCException(f'Remaining amount of {remaining} would be too small.')
sub_amount = amount
self._freqtrade.execute_trade_exit(
trade, current_rate, exit_check, ordertype=order_type,
sub_trade_amt=sub_amount)
def _rpc_force_exit(self, trade_id: str, ordertype: Optional[str] = None, *,
amount: Optional[float] = None) -> Dict[str, str]:
"""
Handler for forceexit <id>.
Sells the given trade at current price
"""
def _exec_force_exit(trade: Trade) -> None:
# Check if there is there is an open order
fully_canceled = False
if trade.open_order_id:
order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
if order['side'] == trade.entry_side:
fully_canceled = self._freqtrade.handle_cancel_enter(
trade, order, CANCEL_REASON['FORCE_EXIT'])
if order['side'] == trade.exit_side:
# Cancel order - so it is placed anew with a fresh price.
self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT'])
if not fully_canceled:
# Get current rate and execute sell
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, side='exit', is_short=trade.is_short, refresh=True)
exit_check = ExitCheckTuple(exit_type=ExitType.FORCE_EXIT)
order_type = ordertype or self._freqtrade.strategy.order_types.get(
"force_exit", self._freqtrade.strategy.order_types["exit"])
self._freqtrade.execute_trade_exit(
trade, current_rate, exit_check, ordertype=order_type)
# ---- EOF def _exec_forcesell ----
if self._freqtrade.state != State.RUNNING:
raise RPCException('trader is not running')
@@ -697,7 +717,7 @@ class RPC:
if trade_id == 'all':
# Execute sell for all open orders
for trade in Trade.get_open_trades():
_exec_force_exit(trade)
self.__exec_force_exit(trade, ordertype)
Trade.commit()
self._freqtrade.wallets.update()
return {'result': 'Created sell orders for all open trades.'}
@@ -710,7 +730,7 @@ class RPC:
logger.warning('force_exit: Invalid argument received')
raise RPCException('invalid argument')
_exec_force_exit(trade)
self.__exec_force_exit(trade, ordertype, amount)
Trade.commit()
self._freqtrade.wallets.update()
return {'result': f'Created sell order for trade {trade_id}.'}
@@ -719,7 +739,8 @@ class RPC:
order_type: Optional[str] = None,
order_side: SignalDirection = SignalDirection.LONG,
stake_amount: Optional[float] = None,
enter_tag: Optional[str] = 'force_entry') -> Optional[Trade]:
enter_tag: Optional[str] = 'force_entry',
leverage: Optional[float] = None) -> Optional[Trade]:
"""
Handler for forcebuy <asset> <price>
Buys a pair trade at the given or current price
@@ -761,6 +782,7 @@ class RPC:
ordertype=order_type, trade=trade,
is_short=is_short,
enter_tag=enter_tag,
leverage_=leverage,
):
Trade.commit()
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
@@ -875,7 +897,7 @@ class RPC:
lock.active = False
lock.lock_end_time = datetime.now(timezone.utc)
PairLock.query.session.commit()
Trade.commit()
return self._rpc_locks()

12
freqtrade/rpc/rpc_manager.py
View File

@@ -2,6 +2,7 @@
This module contains class to manage RPC communications (Telegram, API, ...)
"""
import logging
from collections import deque
from typing import Any, Dict, List
from freqtrade.enums import RPCMessageType
@@ -77,6 +78,17 @@ class RPCManager:
except NotImplementedError:
logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.")
def process_msg_queue(self, queue: deque) -> None:
"""
Process all messages in the queue.
"""
while queue:
msg = queue.popleft()
self.send_msg({
'type': RPCMessageType.STRATEGY_MSG,
'msg': msg,
})
def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None:
if config['dry_run']:
self.send_msg({

163
freqtrade/rpc/telegram.py
View File

@@ -16,8 +16,8 @@ from typing import Any, Callable, Dict, List, Optional, Union
import arrow
from tabulate import tabulate
from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton,
ParseMode, ReplyKeyboardMarkup, Update)
from telegram import (MAX_MESSAGE_LENGTH, CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup,
KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update)
from telegram.error import BadRequest, NetworkError, TelegramError
from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater
from telegram.utils.helpers import escape_markdown
@@ -35,8 +35,6 @@ logger = logging.getLogger(__name__)
logger.debug('Included module rpc.telegram ...')
MAX_TELEGRAM_MESSAGE_LENGTH = 4096
@dataclass
class TimeunitMappings:
@@ -72,7 +70,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]:
)
return wrapper
# Rollback session to avoid getting data stored in a transaction.
Trade.query.session.rollback()
Trade.rollback()
logger.debug(
'Executing handler: %s for chat_id: %s',
command_handler.__name__,
@@ -274,7 +272,7 @@ class Telegram(RPCHandler):
f"{emoji} *{self._exchange_from_msg(msg)}:*"
f" {entry_side['entered'] if is_fill else entry_side['enter']} {msg['pair']}"
f" (#{msg['trade_id']})\n"
)
)
message += self._add_analyzed_candle(msg['pair'])
message += f"*Enter Tag:* `{msg['enter_tag']}`\n" if msg.get('enter_tag') else ""
message += f"*Amount:* `{msg['amount']:.8f}`\n"
@@ -315,20 +313,36 @@ class Telegram(RPCHandler):
msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount(
msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
msg['profit_extra'] = (
f" ({msg['gain']}: {msg['profit_amount']:.8f} {msg['stake_currency']}"
f" / {msg['profit_fiat']:.3f} {msg['fiat_currency']})")
f" / {msg['profit_fiat']:.3f} {msg['fiat_currency']}")
else:
msg['profit_extra'] = ''
msg['profit_extra'] = (
f" ({msg['gain']}: {msg['profit_amount']:.8f} {msg['stake_currency']}"
f"{msg['profit_extra']})")
is_fill = msg['type'] == RPCMessageType.EXIT_FILL
is_sub_trade = msg.get('sub_trade')
is_sub_profit = msg['profit_amount'] != msg.get('cumulative_profit')
profit_prefix = ('Sub ' if is_sub_profit
else 'Cumulative ') if is_sub_trade else ''
cp_extra = ''
if is_sub_profit and is_sub_trade:
if self._rpc._fiat_converter:
cp_fiat = self._rpc._fiat_converter.convert_amount(
msg['cumulative_profit'], msg['stake_currency'], msg['fiat_currency'])
cp_extra = f" / {cp_fiat:.3f} {msg['fiat_currency']}"
else:
cp_extra = ''
cp_extra = f"*Cumulative Profit:* (`{msg['cumulative_profit']:.8f} " \
f"{msg['stake_currency']}{cp_extra}`)\n"
message = (
f"{msg['emoji']} *{self._exchange_from_msg(msg)}:* "
f"{'Exited' if is_fill else 'Exiting'} {msg['pair']} (#{msg['trade_id']})\n"
f"{self._add_analyzed_candle(msg['pair'])}"
f"*{'Profit' if is_fill else 'Unrealized Profit'}:* "
f"*{f'{profit_prefix}Profit' if is_fill else f'Unrealized {profit_prefix}Profit'}:* "
f"`{msg['profit_ratio']:.2%}{msg['profit_extra']}`\n"
f"{cp_extra}"
f"*Enter Tag:* `{msg['enter_tag']}`\n"
f"*Exit Reason:* `{msg['exit_reason']}`\n"
f"*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`\n"
f"*Direction:* `{msg['direction']}`\n"
f"{msg['leverage_text']}"
f"*Amount:* `{msg['amount']:.8f}`\n"
@@ -336,11 +350,25 @@ class Telegram(RPCHandler):
)
if msg['type'] == RPCMessageType.EXIT:
message += (f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
f"*Close Rate:* `{msg['limit']:.8f}`")
f"*Exit Rate:* `{msg['limit']:.8f}`")
elif msg['type'] == RPCMessageType.EXIT_FILL:
message += f"*Close Rate:* `{msg['close_rate']:.8f}`"
message += f"*Exit Rate:* `{msg['close_rate']:.8f}`"
if msg.get('sub_trade'):
if self._rpc._fiat_converter:
msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
else:
msg['stake_amount_fiat'] = 0
rem = round_coin_value(msg['stake_amount'], msg['stake_currency'])
message += f"\n*Remaining:* `({rem}"
if msg.get('fiat_currency', None):
message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
message += ")`"
else:
message += f"\n*Duration:* `{msg['duration']} ({msg['duration_min']:.1f} min)`"
return message
def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str:
@@ -353,7 +381,8 @@ class Telegram(RPCHandler):
elif msg_type in (RPCMessageType.ENTRY_CANCEL, RPCMessageType.EXIT_CANCEL):
msg['message_side'] = 'enter' if msg_type in [RPCMessageType.ENTRY_CANCEL] else 'exit'
message = (f"\N{WARNING SIGN} *{self._exchange_from_msg(msg)}:* "
f"Cancelling {msg['message_side']} Order for {msg['pair']} "
f"Cancelling {'partial ' if msg.get('sub_trade') else ''}"
f"{msg['message_side']} Order for {msg['pair']} "
f"(#{msg['trade_id']}). Reason: {msg['reason']}.")
elif msg_type == RPCMessageType.PROTECTION_TRIGGER:
@@ -376,7 +405,8 @@ class Telegram(RPCHandler):
elif msg_type == RPCMessageType.STARTUP:
message = f"{msg['status']}"
elif msg_type == RPCMessageType.STRATEGY_MSG:
message = f"{msg['msg']}"
else:
raise NotImplementedError(f"Unknown message type: {msg_type}")
return message
@@ -423,54 +453,63 @@ class Telegram(RPCHandler):
else:
return "\N{CROSS MARK}"
def _prepare_entry_details(self, filled_orders: List, quote_currency: str, is_open: bool):
def _prepare_order_details(self, filled_orders: List, quote_currency: str, is_open: bool):
"""
Prepare details of trade with entry adjustment enabled
"""
lines: List[str] = []
lines_detail: List[str] = []
if len(filled_orders) > 0:
first_avg = filled_orders[0]["safe_price"]
for x, order in enumerate(filled_orders):
if not order['ft_is_entry'] or order['is_open'] is True:
lines: List[str] = []
if order['is_open'] is True:
continue
wording = 'Entry' if order['ft_is_entry'] else 'Exit'
cur_entry_datetime = arrow.get(order["order_filled_date"])
cur_entry_amount = order["amount"]
cur_entry_amount = order["filled"] or order["amount"]
cur_entry_average = order["safe_price"]
lines.append(" ")
if x == 0:
lines.append(f"*Entry #{x+1}:*")
lines.append(f"*{wording} #{x+1}:*")
lines.append(
f"*Entry Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})")
lines.append(f"*Average Entry Price:* {cur_entry_average}")
f"*Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})")
lines.append(f"*Average Price:* {cur_entry_average}")
else:
sumA = 0
sumB = 0
for y in range(x):
sumA += (filled_orders[y]["amount"] * filled_orders[y]["safe_price"])
sumB += filled_orders[y]["amount"]
amount = filled_orders[y]["filled"] or filled_orders[y]["amount"]
sumA += amount * filled_orders[y]["safe_price"]
sumB += amount
prev_avg_price = sumA / sumB
# TODO: This calculation ignores fees.
price_to_1st_entry = ((cur_entry_average - first_avg) / first_avg)
minus_on_entry = 0
if prev_avg_price:
minus_on_entry = (cur_entry_average - prev_avg_price) / prev_avg_price
dur_entry = cur_entry_datetime - arrow.get(
filled_orders[x - 1]["order_filled_date"])
days = dur_entry.days
hours, remainder = divmod(dur_entry.seconds, 3600)
minutes, seconds = divmod(remainder, 60)
lines.append(f"*Entry #{x+1}:* at {minus_on_entry:.2%} avg profit")
lines.append(f"*{wording} #{x+1}:* at {minus_on_entry:.2%} avg profit")
if is_open:
lines.append("({})".format(cur_entry_datetime
.humanize(granularity=["day", "hour", "minute"])))
lines.append(
f"*Entry Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})")
lines.append(f"*Average Entry Price:* {cur_entry_average} "
f"*Amount:* {cur_entry_amount} ({order['cost']:.8f} {quote_currency})")
lines.append(f"*Average {wording} Price:* {cur_entry_average} "
f"({price_to_1st_entry:.2%} from 1st entry rate)")
lines.append(f"*Order filled at:* {order['order_filled_date']}")
lines.append(f"({days}d {hours}h {minutes}m {seconds}s from previous entry)")
return lines
lines.append(f"*Order filled:* {order['order_filled_date']}")
# TODO: is this really useful?
# dur_entry = cur_entry_datetime - arrow.get(
# filled_orders[x - 1]["order_filled_date"])
# days = dur_entry.days
# hours, remainder = divmod(dur_entry.seconds, 3600)
# minutes, seconds = divmod(remainder, 60)
# lines.append(
# f"({days}d {hours}h {minutes}m {seconds}s from previous {wording.lower()})")
lines_detail.append("\n".join(lines))
return lines_detail
@authorized_only
def _status(self, update: Update, context: CallbackContext) -> None:
@@ -485,7 +524,14 @@ class Telegram(RPCHandler):
if context.args and 'table' in context.args:
self._status_table(update, context)
return
else:
self._status_msg(update, context)
def _status_msg(self, update: Update, context: CallbackContext) -> None:
"""
handler for `/status` and `/status <id>`.
"""
try:
# Check if there's at least one numerical ID provided.
@@ -497,14 +543,13 @@ class Telegram(RPCHandler):
results = self._rpc._rpc_trade_status(trade_ids=trade_ids)
position_adjust = self._config.get('position_adjustment_enable', False)
max_entries = self._config.get('max_entry_position_adjustment', -1)
messages = []
for r in results:
r['open_date_hum'] = arrow.get(r['open_date']).humanize()
r['num_entries'] = len([o for o in r['orders'] if o['ft_is_entry']])
r['exit_reason'] = r.get('exit_reason', "")
lines = [
"*Trade ID:* `{trade_id}`" +
("` (since {open_date_hum})`" if r['is_open'] else ""),
(" `(since {open_date_hum})`" if r['is_open'] else ""),
"*Current Pair:* {pair}",
"*Direction:* " + ("`Short`" if r.get('is_short') else "`Long`"),
"*Leverage:* `{leverage}`" if r.get('leverage') else "",
@@ -528,6 +573,8 @@ class Telegram(RPCHandler):
])
if r['is_open']:
if r.get('realized_profit'):
lines.append("*Realized Profit:* `{realized_profit:.8f}`")
if (r['stop_loss_abs'] != r['initial_stop_loss_abs']
and r['initial_stop_loss_ratio'] is not None):
# Adding initial stoploss only if it is different from stoploss
@@ -540,24 +587,34 @@ class Telegram(RPCHandler):
lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` "
"`({stoploss_current_dist_ratio:.2%})`")
if r['open_order']:
if r['exit_order_status']:
lines.append("*Open Order:* `{open_order}` - `{exit_order_status}`")
else:
lines.append("*Open Order:* `{open_order}`")
lines.append(
"*Open Order:* `{open_order}`"
+ "- `{exit_order_status}`" if r['exit_order_status'] else "")
lines_detail = self._prepare_entry_details(
lines_detail = self._prepare_order_details(
r['orders'], r['quote_currency'], r['is_open'])
lines.extend(lines_detail if lines_detail else "")
# Filter empty lines using list-comprehension
messages.append("\n".join([line for line in lines if line]).format(**r))
for msg in messages:
self._send_msg(msg)
self.__send_status_msg(lines, r)
except RPCException as e:
self._send_msg(str(e))
def __send_status_msg(self, lines: List[str], r: Dict[str, Any]) -> None:
"""
Send status message.
"""
msg = ''
for line in lines:
if line:
if (len(msg) + len(line) + 1) < MAX_MESSAGE_LENGTH:
msg += line + '\n'
else:
self._send_msg(msg.format(**r))
msg = "*Trade ID:* `{trade_id}` - continued\n" + line + '\n'
self._send_msg(msg.format(**r))
@authorized_only
def _status_table(self, update: Update, context: CallbackContext) -> None:
"""
@@ -860,7 +917,7 @@ class Telegram(RPCHandler):
total_dust_currencies += 1
# Handle overflowing message length
if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH:
if len(output + curr_output) >= MAX_MESSAGE_LENGTH:
self._send_msg(output)
output = curr_output
else:
@@ -1123,7 +1180,7 @@ class Telegram(RPCHandler):
f"({trade['profit_ratio']:.2%}) "
f"({trade['count']})</code>\n")
if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
self._send_msg(output, parse_mode=ParseMode.HTML)
output = stat_line
else:
@@ -1158,7 +1215,7 @@ class Telegram(RPCHandler):
f"({trade['profit_ratio']:.2%}) "
f"({trade['count']})</code>\n")
if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
self._send_msg(output, parse_mode=ParseMode.HTML)
output = stat_line
else:
@@ -1193,7 +1250,7 @@ class Telegram(RPCHandler):
f"({trade['profit_ratio']:.2%}) "
f"({trade['count']})</code>\n")
if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
self._send_msg(output, parse_mode=ParseMode.HTML)
output = stat_line
else:
@@ -1228,7 +1285,7 @@ class Telegram(RPCHandler):
f"({trade['profit']:.2%}) "
f"({trade['count']})</code>\n")
if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
if len(output + stat_line) >= MAX_MESSAGE_LENGTH:
self._send_msg(output, parse_mode=ParseMode.HTML)
output = stat_line
else:
@@ -1367,7 +1424,7 @@ class Telegram(RPCHandler):
escape_markdown(logrec[2], version=2),
escape_markdown(logrec[3], version=2),
escape_markdown(logrec[4], version=2))
if len(msgs + msg) + 10 >= MAX_TELEGRAM_MESSAGE_LENGTH:
if len(msgs + msg) + 10 >= MAX_MESSAGE_LENGTH:
# Send message immediately if it would become too long
self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2)
msgs = msg + '\n'

54
freqtrade/strategy/interface.py
View File

@@ -146,11 +146,19 @@ class IStrategy(ABC, HyperStrategyMixin):
self._ft_informative.append((informative_data, cls_method))
def load_freqAI_model(self) -> None:
if self.config.get('freqai', None):
if self.config.get('freqai', {}).get('enabled', False):
# Import here to avoid importing this if freqAI is disabled
from freqtrade.resolvers.freqaimodel_resolver import FreqaiModelResolver
self.freqai = FreqaiModelResolver.load_freqaimodel(self.config)
self.freqai_info = self.config["freqai"]
else:
# Gracious failures if freqAI is disabled but "start" is called.
class DummyClass():
def start(self, *args, **kwargs):
raise OperationalException(
'freqAI is not enabled. Please enable it in your config to use this strategy.')
self.freqai = DummyClass() # type: ignore
def ft_bot_start(self, **kwargs) -> None:
"""
@@ -472,10 +480,13 @@ class IStrategy(ABC, HyperStrategyMixin):
def adjust_trade_position(self, trade: Trade, current_time: datetime,
current_rate: float, current_profit: float,
min_stake: Optional[float], max_stake: float,
current_entry_rate: float, current_exit_rate: float,
current_entry_profit: float, current_exit_profit: float,
**kwargs) -> Optional[float]:
"""
Custom trade adjustment logic, returning the stake amount that a trade should be increased.
This means extra buy orders with additional fees.
Custom trade adjustment logic, returning the stake amount that a trade should be
increased or decreased.
This means extra buy or sell orders with additional fees.
Only called when `position_adjustment_enable` is set to True.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
@@ -486,10 +497,16 @@ class IStrategy(ABC, HyperStrategyMixin):
:param current_time: datetime object, containing the current datetime
:param current_rate: Current buy rate.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param min_stake: Minimal stake size allowed by exchange.
:param max_stake: Balance available for trading.
:param min_stake: Minimal stake size allowed by exchange (for both entries and exits)
:param max_stake: Maximum stake allowed (either through balance, or by exchange limits).
:param current_entry_rate: Current rate using entry pricing.
:param current_exit_rate: Current rate using exit pricing.
:param current_entry_profit: Current profit using entry pricing.
:param current_exit_profit: Current profit using exit pricing.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: Stake amount to adjust your trade
:return float: Stake amount to adjust your trade,
Positive values to increase position, Negative values to decrease position.
Return None for no action.
"""
return None
@@ -557,8 +574,8 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
return None
def populate_any_indicators(self, basepair: str, pair: str, df: DataFrame, tf: str,
informative: DataFrame = None, coin: str = "",
def populate_any_indicators(self, pair: str, df: DataFrame, tf: str,
informative: DataFrame = None,
set_generalized_indicators: bool = False) -> DataFrame:
"""
Function designed to automatically generate, name and merge features
@@ -570,7 +587,6 @@ class IStrategy(ABC, HyperStrategyMixin):
:param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names
:param informative: the dataframe associated with the informative pair
:param coin: the name of the coin which will modify the feature names.
"""
return df
@@ -989,7 +1005,7 @@ class IStrategy(ABC, HyperStrategyMixin):
# ROI
# Trailing stoploss
if stoplossflag.exit_type == ExitType.STOP_LOSS:
if stoplossflag.exit_type in (ExitType.STOP_LOSS, ExitType.LIQUIDATION):
logger.debug(f"{trade.pair} - Stoploss hit. exit_type={stoplossflag.exit_type}")
exits.append(stoplossflag)
@@ -1061,6 +1077,17 @@ class IStrategy(ABC, HyperStrategyMixin):
sl_higher_long = (trade.stop_loss >= (low or current_rate) and not trade.is_short)
sl_lower_short = (trade.stop_loss <= (high or current_rate) and trade.is_short)
liq_higher_long = (trade.liquidation_price
and trade.liquidation_price >= (low or current_rate)
and not trade.is_short)
liq_lower_short = (trade.liquidation_price
and trade.liquidation_price <= (high or current_rate)
and trade.is_short)
if (liq_higher_long or liq_lower_short):
logger.debug(f"{trade.pair} - Liquidation price hit. exit_type=ExitType.LIQUIDATION")
return ExitCheckTuple(exit_type=ExitType.LIQUIDATION)
# evaluate if the stoploss was hit if stoploss is not on exchange
# in Dry-Run, this handles stoploss logic as well, as the logic will not be different to
# regular stoploss handling.
@@ -1078,13 +1105,6 @@ class IStrategy(ABC, HyperStrategyMixin):
f"stoploss is {trade.stop_loss:.6f}, "
f"initial stoploss was at {trade.initial_stop_loss:.6f}, "
f"trade opened at {trade.open_rate:.6f}")
new_stoploss = (
trade.stop_loss + trade.initial_stop_loss
if trade.is_short else
trade.stop_loss - trade.initial_stop_loss
)
logger.debug(f"{trade.pair} - Trailing stop saved "
f"{new_stoploss:.6f}")
return ExitCheckTuple(exit_type=exit_type)

45
freqtrade/templates/FreqaiExampleStrategy.py
View File

@@ -65,7 +65,7 @@ class FreqaiExampleStrategy(IStrategy):
return informative_pairs
def populate_any_indicators(
self, metadata, pair, df, tf, informative=None, coin="", set_generalized_indicators=False
self, pair, df, tf, informative=None, set_generalized_indicators=False
):
"""
Function designed to automatically generate, name and merge features
@@ -78,9 +78,10 @@ class FreqaiExampleStrategy(IStrategy):
:param df: strategy dataframe which will receive merges from informatives
:param tf: timeframe of the dataframe which will modify the feature names
:param informative: the dataframe associated with the informative pair
:param coin: the name of the coin which will modify the feature names.
"""
coin = pair.split('/')[0]
with self.freqai.lock:
if informative is None:
informative = self.dp.get_pair_dataframe(pair, tf)
@@ -92,11 +93,8 @@ class FreqaiExampleStrategy(IStrategy):
informative[f"%-{coin}rsi-period_{t}"] = ta.RSI(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
informative[f"%-{coin}adx-period_{t}"] = ta.ADX(informative, window=t)
informative[f"{coin}20sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"{coin}21ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
informative[f"%-{coin}close_over_20sma-period_{t}"] = (
informative["close"] / informative[f"{coin}20sma-period_{t}"]
)
informative[f"%-{coin}sma-period_{t}"] = ta.SMA(informative, timeperiod=t)
informative[f"%-{coin}ema-period_{t}"] = ta.EMA(informative, timeperiod=t)
informative[f"%-{coin}mfi-period_{t}"] = ta.MFI(informative, timeperiod=t)
@@ -148,8 +146,6 @@ class FreqaiExampleStrategy(IStrategy):
df["%-hour_of_day"] = (df["date"].dt.hour + 1) / 25
# user adds targets here by prepending them with &- (see convention below)
# If user wishes to use multiple targets, a multioutput prediction model
# needs to be used such as templates/CatboostPredictionMultiModel.py
df["&-s_close"] = (
df["close"]
.shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
@@ -159,12 +155,31 @@ class FreqaiExampleStrategy(IStrategy):
- 1
)
# Classifiers are typically set up with strings as targets:
# df['&s-up_or_down'] = np.where( df["close"].shift(-100) >
# df["close"], 'up', 'down')
# If user wishes to use multiple targets, they can add more by
# appending more columns with '&'. User should keep in mind that multi targets
# requires a multioutput prediction model such as
# templates/CatboostPredictionMultiModel.py,
# df["&-s_range"] = (
# df["close"]
# .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
# .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
# .max()
# -
# df["close"]
# .shift(-self.freqai_info["feature_parameters"]["label_period_candles"])
# .rolling(self.freqai_info["feature_parameters"]["label_period_candles"])
# .min()
# )
return df
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
self.freqai_info = self.config["freqai"]
# All indicators must be populated by populate_any_indicators() for live functionality
# to work correctly.
@@ -235,16 +250,16 @@ class FreqaiExampleStrategy(IStrategy):
if (
"prediction" + entry_tag not in pair_dict[pair]
or pair_dict[pair]["prediction" + entry_tag] > 0
or pair_dict[pair]['extras']["prediction" + entry_tag] == 0
):
with self.freqai.lock:
pair_dict[pair]["prediction" + entry_tag] = abs(trade_candle["&-s_close"])
pair_dict[pair]['extras']["prediction" + entry_tag] = abs(trade_candle["&-s_close"])
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:
self.freqai.dd.save_follower_dict_to_disk()
roi_price = pair_dict[pair]["prediction" + entry_tag]
roi_price = pair_dict[pair]['extras']["prediction" + entry_tag]
roi_time = self.max_roi_time_long.value
roi_decay = roi_price * (
@@ -282,7 +297,7 @@ class FreqaiExampleStrategy(IStrategy):
pair_dict = self.freqai.dd.follower_dict
with self.freqai.lock:
pair_dict[pair]["prediction" + entry_tag] = 0
pair_dict[pair]['extras']["prediction" + entry_tag] = 0
if not follow_mode:
self.freqai.dd.save_drawer_to_disk()
else:

1
freqtrade/templates/base_config.json.j2
View File

@@ -12,6 +12,7 @@
"tradable_balance_ratio": 0.99,
"fiat_display_currency": "{{ fiat_display_currency }}",{{ ('\n "timeframe": "' + timeframe + '",') if timeframe else '' }}
"dry_run": {{ dry_run | lower }},
"dry_run_wallet": 1000,
"cancel_open_orders_on_exit": false,
"trading_mode": "{{ trading_mode }}",
"margin_mode": "{{ margin_mode }}",

26
freqtrade/templates/subtemplates/strategy_methods_advanced.j2
View File

@@ -247,12 +247,16 @@ def check_exit_timeout(self, pair: str, trade: 'Trade', order: 'Order',
"""
return False
def adjust_trade_position(self, trade: 'Trade', current_time: 'datetime',
current_rate: float, current_profit: float, min_stake: Optional[float],
max_stake: float, **kwargs) -> 'Optional[float]':
def adjust_trade_position(self, trade: 'Trade', current_time: datetime,
current_rate: float, current_profit: float,
min_stake: Optional[float], max_stake: float,
current_entry_rate: float, current_exit_rate: float,
current_entry_profit: float, current_exit_profit: float,
**kwargs) -> Optional[float]:
"""
Custom trade adjustment logic, returning the stake amount that a trade should be increased.
This means extra buy orders with additional fees.
Custom trade adjustment logic, returning the stake amount that a trade should be
increased or decreased.
This means extra buy or sell orders with additional fees.
Only called when `position_adjustment_enable` is set to True.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
@@ -263,10 +267,16 @@ def adjust_trade_position(self, trade: 'Trade', current_time: 'datetime',
:param current_time: datetime object, containing the current datetime
:param current_rate: Current buy rate.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param min_stake: Minimal stake size allowed by exchange.
:param max_stake: Balance available for trading.
:param min_stake: Minimal stake size allowed by exchange (for both entries and exits)
:param max_stake: Maximum stake allowed (either through balance, or by exchange limits).
:param current_entry_rate: Current rate using entry pricing.
:param current_exit_rate: Current rate using exit pricing.
:param current_entry_profit: Current profit using entry pricing.
:param current_exit_profit: Current profit using exit pricing.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: Stake amount to adjust your trade
:return float: Stake amount to adjust your trade,
Positive values to increase position, Negative values to decrease position.
Return None for no action.
"""
return None

3
freqtrade/util/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
# flake8: noqa: F401
from freqtrade.util.ft_precise import FtPrecise
from freqtrade.util.periodic_cache import PeriodicCache

12
freqtrade/util/ft_precise.py Normal file
View File

@@ -0,0 +1,12 @@
"""
Slim wrapper around ccxt's Precise (string math)
To have imports from freqtrade - and support float initializers
"""
from ccxt import Precise
class FtPrecise(Precise):
def __init__(self, number, decimals=None):
if not isinstance(number, str):
number = str(number)
super().__init__(number, decimals)

0
freqtrade/configuration/PeriodicCache.py → freqtrade/util/periodic_cache.py
View File