Merge branch 'develop' of github.com:lolongcovas/freqtrade into feat/freqai

This commit is contained in:
longyu
2022-08-24 10:39:32 +02:00
27 changed files with 203 additions and 91 deletions

View File

@@ -81,7 +81,7 @@ def start_download_data(args: Dict[str, Any]) -> None:
data_format_trades=config['dataformat_trades'],
)
else:
if not exchange._ft_has.get('ohlcv_has_history', True):
if not exchange.get_option('ohlcv_has_history', True):
raise OperationalException(
f"Historic klines not available for {exchange.name}. "
"Please use `--dl-trades` instead for this exchange "

View File

@@ -302,8 +302,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
if trading_mode == 'futures':
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
tf_mark = exchange._ft_has['mark_ohlcv_timeframe']
fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price'])
tf_mark = exchange.get_option('mark_ohlcv_timeframe')
fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price'))
# All exchanges need FundingRate for futures trading.
# The timeframe is aligned to the mark-price timeframe.
for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
@@ -330,13 +330,12 @@ def _download_trades_history(exchange: Exchange,
try:
until = None
since = 0
if timerange:
if timerange.starttype == 'date':
since = timerange.startts * 1000
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
trades = data_handler.trades_load(pair)
@@ -349,6 +348,9 @@ def _download_trades_history(exchange: Exchange,
logger.info(f"Start earlier than available data. Redownloading trades for {pair}...")
trades = []
if not since:
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
from_id = trades[-1][1] if trades else None
if trades and since < trades[-1][0]:
# Reset since to the last available point

View File

@@ -9,7 +9,8 @@ from freqtrade.exchange.bitpanda import Bitpanda
from freqtrade.exchange.bittrex import Bittrex
from freqtrade.exchange.bybit import Bybit
from freqtrade.exchange.coinbasepro import Coinbasepro
from freqtrade.exchange.exchange import (amount_to_precision, available_exchanges, ccxt_exchanges,
from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision,
available_exchanges, ccxt_exchanges, contracts_to_amount,
date_minus_candles, is_exchange_known_ccxt,
is_exchange_officially_supported, market_is_active,
price_to_precision, timeframe_to_minutes,

View File

@@ -54,8 +54,8 @@ class Exchange:
# Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
_params: Dict = {}
# Additional headers - added to the ccxt object
_headers: Dict = {}
# Additional parameters - added to the ccxt object
_ccxt_params: Dict = {}
# Dict to specify which options each exchange implements
# This defines defaults, which can be selectively overridden by subclasses using _ft_has
@@ -242,9 +242,9 @@ class Exchange:
}
if ccxt_kwargs:
logger.info('Applying additional ccxt config: %s', ccxt_kwargs)
if self._headers:
# Inject static headers after the above output to not confuse users.
ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs)
if self._ccxt_params:
# Inject static options after the above output to not confuse users.
ccxt_kwargs = deep_merge_dicts(self._ccxt_params, ccxt_kwargs)
if ccxt_kwargs:
ex_config.update(ccxt_kwargs)
try:
@@ -408,7 +408,7 @@ class Exchange:
else:
return DataFrame()
def _get_contract_size(self, pair: str) -> float:
def get_contract_size(self, pair: str) -> float:
if self.trading_mode == TradingMode.FUTURES:
market = self.markets[pair]
contract_size: float = 1.0
@@ -421,7 +421,7 @@ class Exchange:
def _trades_contracts_to_amount(self, trades: List) -> List:
if len(trades) > 0 and 'symbol' in trades[0]:
contract_size = self._get_contract_size(trades[0]['symbol'])
contract_size = self.get_contract_size(trades[0]['symbol'])
if contract_size != 1:
for trade in trades:
trade['amount'] = trade['amount'] * contract_size
@@ -429,7 +429,7 @@ class Exchange:
def _order_contracts_to_amount(self, order: Dict) -> Dict:
if 'symbol' in order and order['symbol'] is not None:
contract_size = self._get_contract_size(order['symbol'])
contract_size = self.get_contract_size(order['symbol'])
if contract_size != 1:
for prop in self._ft_has.get('order_props_in_contracts', []):
if prop in order and order[prop] is not None:
@@ -438,19 +438,13 @@ class Exchange:
def _amount_to_contracts(self, pair: str, amount: float) -> float:
contract_size = self._get_contract_size(pair)
if contract_size and contract_size != 1:
return amount / contract_size
else:
return amount
contract_size = self.get_contract_size(pair)
return amount_to_contracts(amount, contract_size)
def _contracts_to_amount(self, pair: str, num_contracts: float) -> float:
contract_size = self._get_contract_size(pair)
if contract_size and contract_size != 1:
return num_contracts * contract_size
else:
return num_contracts
contract_size = self.get_contract_size(pair)
return contracts_to_amount(num_contracts, contract_size)
def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None:
if exchange_config.get('sandbox'):
@@ -674,6 +668,12 @@ class Exchange:
f"Freqtrade does not support {mm_value} {trading_mode.value} on {self.name}"
)
def get_option(self, param: str, default: Any = None) -> Any:
"""
Get parameter value from _ft_has
"""
return self._ft_has.get(param, default)
def exchange_has(self, endpoint: str) -> bool:
"""
Checks if exchange implements a specific API endpoint.
@@ -2892,6 +2892,33 @@ def market_is_active(market: Dict) -> bool:
return market.get('active', True) is not False
def amount_to_contracts(amount: float, contract_size: Optional[float]) -> float:
"""
Convert amount to contracts.
:param amount: amount to convert
:param contract_size: contract size - taken from exchange.get_contract_size(pair)
:return: num-contracts
"""
if contract_size and contract_size != 1:
return amount / contract_size
else:
return amount
def contracts_to_amount(num_contracts: float, contract_size: Optional[float]) -> float:
"""
Takes num-contracts and converts it to contract size
:param num_contracts: number of contracts
:param contract_size: contract size - taken from exchange.get_contract_size(pair)
:return: Amount
"""
if contract_size and contract_size != 1:
return num_contracts * contract_size
else:
return num_contracts
def amount_to_precision(amount: float, amount_precision: Optional[float],
precisionMode: Optional[int]) -> float:
"""

View File

@@ -25,7 +25,6 @@ class Gateio(Exchange):
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
"ohlcv_volume_currency": "quote",
"time_in_force_parameter": "timeInForce",
"order_time_in_force": ['gtc', 'ioc'],
"stoploss_order_types": {"limit": "limit"},
@@ -34,7 +33,6 @@ class Gateio(Exchange):
_ft_has_futures: Dict = {
"needs_trading_fees": True,
"ohlcv_volume_currency": "base",
"fee_cost_in_contracts": False, # Set explicitly to false for clarity
"order_props_in_contracts": ['amount', 'filled', 'remaining'],
}

View File

@@ -39,6 +39,8 @@ class Okx(Exchange):
net_only = True
_ccxt_params: Dict = {'options': {'brokerId': 'ffb5405ad327SUDE'}}
def ohlcv_candle_limit(
self, timeframe: str, candle_type: CandleType, since_ms: Optional[int] = None) -> int:
"""

View File

@@ -421,7 +421,7 @@ class FreqaiDataDrawer:
)
# if self.live:
self.model_dictionary[dk.model_filename] = model
self.model_dictionary[coin] = model
self.pair_dict[coin]["model_filename"] = dk.model_filename
self.pair_dict[coin]["data_path"] = str(dk.data_path)
self.save_drawer_to_disk()
@@ -460,8 +460,8 @@ class FreqaiDataDrawer:
)
# try to access model in memory instead of loading object from disk to save time
if dk.live and dk.model_filename in self.model_dictionary:
model = self.model_dictionary[dk.model_filename]
if dk.live and coin in self.model_dictionary:
model = self.model_dictionary[coin]
elif not dk.keras:
model = load(dk.data_path / f"{dk.model_filename}_model.joblib")
else:

View File

@@ -601,6 +601,8 @@ class FreqaiDataKitchen:
is an outlier.
"""
from math import cos, sin
if predict:
train_ft_df = self.data_dictionary['train_features']
pred_ft_df = self.data_dictionary['prediction_features']
@@ -619,23 +621,47 @@ class FreqaiDataKitchen:
else:
def normalise_distances(distances):
normalised_distances = (distances - distances.min()) / \
(distances.max() - distances.min())
return normalised_distances
def rotate_point(origin, point, angle):
# rotate a point counterclockwise by a given angle (in radians)
# around a given origin
x = origin[0] + cos(angle) * (point[0] - origin[0]) - \
sin(angle) * (point[1] - origin[1])
y = origin[1] + sin(angle) * (point[0] - origin[0]) + \
cos(angle) * (point[1] - origin[1])
return (x, y)
MinPts = len(self.data_dictionary['train_features'].columns) * 2
# measure pairwise distances to train_features.shape[1]*2 nearest neighbours
neighbors = NearestNeighbors(
n_neighbors=MinPts, n_jobs=self.thread_count)
neighbors_fit = neighbors.fit(self.data_dictionary['train_features'])
distances, _ = neighbors_fit.kneighbors(self.data_dictionary['train_features'])
distances = np.sort(distances, axis=0)
index_ten_pct = int(len(distances[:, 1]) * 0.1)
distances = distances[index_ten_pct:, 1]
epsilon = distances[-1]
distances = np.sort(distances, axis=0).mean(axis=1)
normalised_distances = normalise_distances(distances)
x_range = np.linspace(0, 1, len(distances))
line = np.linspace(normalised_distances[0],
normalised_distances[-1], len(normalised_distances))
deflection = np.abs(normalised_distances - line)
max_deflection_loc = np.where(deflection == deflection.max())[0][0]
origin = x_range[max_deflection_loc], line[max_deflection_loc]
point = x_range[max_deflection_loc], normalised_distances[max_deflection_loc]
rot_angle = np.pi / 4
elbow_loc = rotate_point(origin, point, rot_angle)
epsilon = elbow_loc[1] * (distances[-1] - distances[0]) + distances[0]
clustering = DBSCAN(eps=epsilon, min_samples=MinPts,
n_jobs=int(self.thread_count)).fit(
self.data_dictionary['train_features']
)
logger.info(f'DBSCAN found eps of {epsilon}.')
logger.info(f'DBSCAN found eps of {epsilon:.2f}.')
self.data['DBSCAN_eps'] = epsilon
self.data['DBSCAN_min_samples'] = MinPts
@@ -806,7 +832,7 @@ class FreqaiDataKitchen:
if (len(do_predict) - do_predict.sum()) > 0:
logger.info(
f"DI tossed {len(do_predict) - do_predict.sum():.2f} predictions for "
f"DI tossed {len(do_predict) - do_predict.sum()} predictions for "
"being too far from training data"
)
@@ -981,13 +1007,6 @@ class FreqaiDataKitchen:
data_load_timerange.stopts = int(time)
retrain = True
# logger.info(
# f"downloading data for "
# f"{(data_load_timerange.stopts-data_load_timerange.startts)/SECONDS_IN_DAY:.2f} "
# " days. "
# f"Extension of {additional_seconds/SECONDS_IN_DAY:.2f} days"
# )
return retrain, trained_timerange, data_load_timerange
def set_new_model_names(self, pair: str, trained_timerange: TimeRange):

View File

@@ -82,12 +82,15 @@ class IFreqaiModel(ABC):
if self.ft_params.get("inlier_metric_window", 0):
self.CONV_WIDTH = self.ft_params.get("inlier_metric_window", 0) * 2
self.pair_it = 0
self.pair_it_train = 0
self.total_pairs = len(self.config.get("exchange", {}).get("pair_whitelist"))
self.last_trade_database_summary: DataFrame = {}
self.current_trade_database_summary: DataFrame = {}
self.analysis_lock = Lock()
self.inference_time: float = 0
self.train_time: float = 0
self.begin_time: float = 0
self.begin_time_train: float = 0
self.base_tf_seconds = timeframe_to_seconds(self.config['timeframe'])
def assert_config(self, config: Dict[str, Any]) -> None:
@@ -130,11 +133,20 @@ class IFreqaiModel(ABC):
dk = self.start_backtesting(dataframe, metadata, self.dk)
dataframe = dk.remove_features_from_df(dk.return_dataframe)
del dk
self.clean_up()
if self.live:
self.inference_timer('stop')
return dataframe
def clean_up(self):
"""
Objects that should be handled by GC already between coins, but
are explicitly shown here to help demonstrate the non-persistence of these
objects.
"""
self.model = None
self.dk = None
@threaded
def start_scanning(self, strategy: IStrategy) -> None:
"""
@@ -161,9 +173,11 @@ class IFreqaiModel(ABC):
dk.set_paths(pair, new_trained_timerange.stopts)
if retrain:
self.train_timer('start')
self.train_model_in_series(
new_trained_timerange, pair, strategy, dk, data_load_timerange
)
self.train_timer('stop')
self.dd.save_historic_predictions_to_disk()
@@ -490,8 +504,7 @@ class IFreqaiModel(ABC):
data_load_timerange: TimeRange,
):
"""
Retrieve data and train model in single threaded mode (only used if model directory is empty
upon startup for dry/live )
Retrieve data and train model.
:param new_trained_timerange: TimeRange = the timerange to train the model on
:param metadata: dict = strategy provided metadata
:param strategy: IStrategy = user defined strategy object
@@ -622,6 +635,24 @@ class IFreqaiModel(ABC):
self.inference_time = 0
return
def train_timer(self, do='start'):
"""
Timer designed to track the cumulative time spent training the full pairlist in
FreqAI.
"""
if do == 'start':
self.pair_it_train += 1
self.begin_time_train = time.time()
elif do == 'stop':
end = time.time()
self.train_time += (end - self.begin_time_train)
if self.pair_it_train == self.total_pairs:
logger.info(
f'Total time spent training pairlist {self.train_time:.2f} seconds')
self.pair_it_train = 0
self.train_time = 0
return
# Following methods which are overridden by user made prediction models.
# See freqai/prediction_models/CatboostPredictionModel.py for an example.

View File

@@ -271,7 +271,7 @@ class FreqtradeBot(LoggingMixin):
Return the number of free open trades slots or 0 if
max number of open trades reached
"""
open_trades = len(Trade.get_open_trades())
open_trades = Trade.get_open_trade_count()
return max(0, self.config['max_open_trades'] - open_trades)
def update_funding_fees(self):
@@ -290,13 +290,14 @@ class FreqtradeBot(LoggingMixin):
def startup_backpopulate_precision(self):
trades = Trade.get_trades([Trade.precision_mode.is_(None)])
trades = Trade.get_trades([Trade.contract_size.is_(None)])
for trade in trades:
if trade.exchange != self.exchange.id:
continue
trade.precision_mode = self.exchange.precisionMode
trade.amount_precision = self.exchange.get_precision_amount(trade.pair)
trade.price_precision = self.exchange.get_precision_price(trade.pair)
trade.contract_size = self.exchange.get_contract_size(trade.pair)
Trade.commit()
def startup_update_open_orders(self):
@@ -755,6 +756,7 @@ class FreqtradeBot(LoggingMixin):
amount_precision=self.exchange.get_precision_amount(pair),
price_precision=self.exchange.get_precision_price(pair),
precision_mode=self.exchange.precisionMode,
contract_size=self.exchange.get_contract_size(pair),
)
else:
# This is additional buy, we reset fee_open_currency so timeout checking can work

View File

@@ -24,7 +24,8 @@ from freqtrade.enums import (BacktestState, CandleType, ExitCheckTuple, ExitType
TradingMode)
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import amount_to_precision
from freqtrade.exchange.exchange import (amount_to_contracts, amount_to_precision,
contracts_to_amount)
from freqtrade.mixins import LoggingMixin
from freqtrade.optimize.backtest_caching import get_strategy_run_id
from freqtrade.optimize.bt_progress import BTProgress
@@ -267,7 +268,7 @@ class Backtesting:
funding_rates_dict = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'],
timeframe=self.exchange.get_option('mark_ohlcv_timeframe'),
timerange=self.timerange,
startup_candles=0,
fail_without_data=True,
@@ -279,12 +280,12 @@ class Backtesting:
mark_rates_dict = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.exchange._ft_has['mark_ohlcv_timeframe'],
timeframe=self.exchange.get_option('mark_ohlcv_timeframe'),
timerange=self.timerange,
startup_candles=0,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
candle_type=CandleType.from_string(self.exchange._ft_has["mark_ohlcv_price"])
candle_type=CandleType.from_string(self.exchange.get_option("mark_ohlcv_price"))
)
# Combine data to avoid combining the data per trade.
unavailable_pairs = []
@@ -823,11 +824,13 @@ class Backtesting:
self.order_id_counter += 1
base_currency = self.exchange.get_pair_base_currency(pair)
amount_p = (stake_amount / propose_rate) * leverage
amount = self.exchange._contracts_to_amount(
pair, amount_to_precision(
self.exchange._amount_to_contracts(pair, amount_p),
self.exchange.get_precision_amount(pair), self.precision_mode)
)
contract_size = self.exchange.get_contract_size(pair)
precision_amount = self.exchange.get_precision_amount(pair)
amount = contracts_to_amount(
amount_to_precision(
amount_to_contracts(amount_p, contract_size),
precision_amount, self.precision_mode),
contract_size)
# Backcalculate actual stake amount.
stake_amount = amount * propose_rate / leverage
@@ -859,9 +862,10 @@ class Backtesting:
trading_mode=self.trading_mode,
leverage=leverage,
# interest_rate=interest_rate,
amount_precision=self.exchange.get_precision_amount(pair),
amount_precision=precision_amount,
price_precision=self.exchange.get_precision_price(pair),
precision_mode=self.precision_mode,
contract_size=contract_size,
orders=[],
)

View File

@@ -133,6 +133,7 @@ def migrate_trades_and_orders_table(
amount_precision = get_column_def(cols, 'amount_precision', 'null')
price_precision = get_column_def(cols, 'price_precision', 'null')
precision_mode = get_column_def(cols, 'precision_mode', 'null')
contract_size = get_column_def(cols, 'contract_size', 'null')
# Schema migration necessary
with engine.begin() as connection:
@@ -161,7 +162,7 @@ def migrate_trades_and_orders_table(
timeframe, open_trade_value, close_profit_abs,
trading_mode, leverage, liquidation_price, is_short,
interest_rate, funding_fees, realized_profit,
amount_precision, price_precision, precision_mode
amount_precision, price_precision, precision_mode, contract_size
)
select id, lower(exchange), pair, {base_currency} base_currency,
{stake_currency} stake_currency,
@@ -189,7 +190,7 @@ def migrate_trades_and_orders_table(
{is_short} is_short, {interest_rate} interest_rate,
{funding_fees} funding_fees, {realized_profit} realized_profit,
{amount_precision} amount_precision, {price_precision} price_precision,
{precision_mode} precision_mode
{precision_mode} precision_mode, {contract_size} contract_size
from {trade_back_name}
"""))
@@ -308,7 +309,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
# if ('orders' not in previous_tables
# or not has_column(cols_orders, 'stop_price')):
migrating = False
if not has_column(cols_trades, 'precision_mode'):
if not has_column(cols_trades, 'contract_size'):
migrating = True
logger.info(f"Running database migration for trades - "
f"backup: {table_back_name}, {order_table_bak_name}")

View File

@@ -15,6 +15,7 @@ from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPE
from freqtrade.enums import ExitType, TradingMode
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import amount_to_precision, price_to_precision
from freqtrade.exchange.exchange import amount_to_contracts, contracts_to_amount
from freqtrade.leverage import interest
from freqtrade.persistence.base import _DECL_BASE
from freqtrade.util import FtPrecise
@@ -296,6 +297,7 @@ class LocalTrade():
amount_precision: Optional[float] = None
price_precision: Optional[float] = None
precision_mode: Optional[int] = None
contract_size: Optional[float] = None
# Leverage trading properties
liquidation_price: Optional[float] = None
@@ -623,7 +625,11 @@ class LocalTrade():
else:
logger.warning(
f'Got different open_order_id {self.open_order_id} != {order.order_id}')
amount_tr = amount_to_precision(self.amount, self.amount_precision, self.precision_mode)
amount_tr = contracts_to_amount(
amount_to_precision(
amount_to_contracts(self.amount, self.contract_size),
self.amount_precision, self.precision_mode),
self.contract_size)
if isclose(order.safe_amount_after_fee, amount_tr, abs_tol=MATH_CLOSE_PREC):
self.close(order.safe_price)
else:
@@ -1044,6 +1050,16 @@ class LocalTrade():
"""
return Trade.get_trades_proxy(is_open=True)
@staticmethod
def get_open_trade_count() -> int:
"""
get open trade count
"""
if Trade.use_db:
return Trade.query.filter(Trade.is_open.is_(True)).count()
else:
return len(LocalTrade.trades_open)
@staticmethod
def stoploss_reinitialization(desired_stoploss):
"""
@@ -1132,6 +1148,7 @@ class Trade(_DECL_BASE, LocalTrade):
amount_precision = Column(Float, nullable=True)
price_precision = Column(Float, nullable=True)
precision_mode = Column(Integer, nullable=True)
contract_size = Column(Float, nullable=True)
# Leverage trading properties
leverage = Column(Float, nullable=True, default=1.0)

View File

@@ -73,7 +73,7 @@ class VolumePairList(IPairList):
if (not self._use_range and not (
self._exchange.exchange_has('fetchTickers')
and self._exchange._ft_has["tickers_have_quoteVolume"])):
and self._exchange.get_option("tickers_have_quoteVolume"))):
raise OperationalException(
"Exchange does not support dynamic whitelist in this configuration. "
"Please edit your config and either remove Volumepairlist, "
@@ -193,7 +193,7 @@ class VolumePairList(IPairList):
) in candles else None
# in case of candle data calculate typical price and quoteVolume for candle
if pair_candles is not None and not pair_candles.empty:
if self._exchange._ft_has["ohlcv_volume_currency"] == "base":
if self._exchange.get_option("ohlcv_volume_currency") == "base":
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3

View File

@@ -193,7 +193,10 @@ class IResolver:
:return: List of dicts containing 'name', 'class' and 'location' entries
"""
logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'")
objects = []
objects: List[Dict[str, Any]] = []
if not directory.is_dir():
logger.info(f"'{directory}' is not a directory, skipping.")
return objects
for entry in directory.iterdir():
if (
recursive and entry.is_dir()