adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into dev-merge-rl

Browse Source
This commit is contained in:
robcaulk
2022-10-05 15:21:45 +02:00
parent 292d72d593 b0eff4160f
commit 9c73411ac2
32 changed files with 400 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/configuration/__init__.py
View File

@@ -1,6 +1,5 @@
# flake8: noqa: F401
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration

4
freqtrade/configuration/configuration.py
View File

@@ -8,7 +8,6 @@ from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
@@ -100,6 +99,9 @@ class Configuration:
self._process_freqai_options(config)
# Import check_exchange here to avoid import cycle problems
from freqtrade.exchange.check_exchange import check_exchange
# Check if the exchange set by the user is supported
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))

4
freqtrade/exchange/__init__.py
View File

@@ -12,8 +12,8 @@ from freqtrade.exchange.coinbasepro import Coinbasepro
from freqtrade.exchange.exchange import (amount_to_contract_precision, amount_to_contracts,
amount_to_precision, available_exchanges, ccxt_exchanges,
contracts_to_amount, date_minus_candles,
is_exchange_known_ccxt, is_exchange_officially_supported,
market_is_active, price_to_precision, timeframe_to_minutes,
is_exchange_known_ccxt, market_is_active,
price_to_precision, timeframe_to_minutes,
timeframe_to_msecs, timeframe_to_next_date,
timeframe_to_prev_date, timeframe_to_seconds,
validate_exchange, validate_exchanges)

6
freqtrade/configuration/check_exchange.py → freqtrade/exchange/check_exchange.py
View File

@@ -3,8 +3,8 @@ import logging
from freqtrade.constants import Config
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
is_exchange_officially_supported, validate_exchange)
from freqtrade.exchange import available_exchanges, is_exchange_known_ccxt, validate_exchange
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS, SUPPORTED_EXCHANGES
logger = logging.getLogger(__name__)
@@ -52,7 +52,7 @@ def check_exchange(config: Config, check_for_bad: bool = True) -> bool:
else:
logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}')
if is_exchange_officially_supported(exchange):
if MAP_EXCHANGE_CHILDCLASS.get(exchange, exchange) in SUPPORTED_EXCHANGES:
logger.info(f'Exchange "{exchange}" is officially supported '
f'by the Freqtrade development team.')
else:

76
freqtrade/exchange/exchange.py
View File

@@ -30,8 +30,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun
RetryableOrderError, TemporaryError)
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
SUPPORTED_EXCHANGES, remove_credentials, retrier,
retrier_async)
remove_credentials, retrier, retrier_async)
from freqtrade.misc import (chunks, deep_merge_dicts, file_dump_json, file_load_json,
safe_value_fallback2)
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -1292,7 +1291,14 @@ class Exchange:
order = self.fetch_order(order_id, pair)
except InvalidOrderException:
logger.warning(f"Could not fetch cancelled order {order_id}.")
order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
order = {
'id': order_id,
'status': 'canceled',
'amount': amount,
'filled': 0.0,
'fee': {},
'info': {}
}
return order
@@ -1863,6 +1869,38 @@ class Exchange:
return self._async_get_candle_history(
pair, timeframe, since_ms=since_ms, candle_type=candle_type)
def _build_ohlcv_dl_jobs(
self, pair_list: ListPairsWithTimeframes, since_ms: Optional[int],
cache: bool) -> Tuple[List[Coroutine], List[Tuple[str, str, CandleType]]]:
"""
Build Coroutines to execute as part of refresh_latest_ohlcv
"""
input_coroutines = []
cached_pairs = []
for pair, timeframe, candle_type in set(pair_list):
if (
timeframe not in self.timeframes
and candle_type in (CandleType.SPOT, CandleType.FUTURES)
):
logger.warning(
f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
f"not available on {self.name}. Available timeframes are "
f"{', '.join(self.timeframes)}.")
continue
if ((pair, timeframe, candle_type) not in self._klines or not cache
or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
input_coroutines.append(self._build_coroutine(
pair, timeframe, candle_type=candle_type, since_ms=since_ms))
else:
logger.debug(
f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
)
cached_pairs.append((pair, timeframe, candle_type))
return input_coroutines, cached_pairs
def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
since_ms: Optional[int] = None, cache: bool = True,
drop_incomplete: Optional[bool] = None
@@ -1880,27 +1918,9 @@ class Exchange:
"""
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
drop_incomplete = self._ohlcv_partial_candle if drop_incomplete is None else drop_incomplete
input_coroutines = []
cached_pairs = []
# Gather coroutines to run
for pair, timeframe, candle_type in set(pair_list):
if (timeframe not in self.timeframes
and candle_type in (CandleType.SPOT, CandleType.FUTURES)):
logger.warning(
f"Cannot download ({pair}, {timeframe}) combination as this timeframe is "
f"not available on {self.name}. Available timeframes are "
f"{', '.join(self.timeframes)}.")
continue
if ((pair, timeframe, candle_type) not in self._klines or not cache
or self._now_is_time_to_refresh(pair, timeframe, candle_type)):
input_coroutines.append(self._build_coroutine(
pair, timeframe, candle_type=candle_type, since_ms=since_ms))
else:
logger.debug(
f"Using cached candle (OHLCV) data for {pair}, {timeframe}, {candle_type} ..."
)
cached_pairs.append((pair, timeframe, candle_type))
# Gather coroutines to run
input_coroutines, cached_pairs = self._build_ohlcv_dl_jobs(pair_list, since_ms, cache)
results_df = {}
# Chunk requests into batches of 100 to avoid overwelming ccxt Throttling
@@ -1941,10 +1961,8 @@ class Exchange:
interval_in_sec = timeframe_to_seconds(timeframe)
return not (
(self._pairs_last_refresh_time.get(
(pair, timeframe, candle_type),
0
) + interval_in_sec) >= arrow.utcnow().int_timestamp
(self._pairs_last_refresh_time.get((pair, timeframe, candle_type), 0)
+ interval_in_sec) >= arrow.utcnow().int_timestamp
)
@retrier_async
@@ -2754,10 +2772,6 @@ def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = Non
return exchange_name in ccxt_exchanges(ccxt_module)
def is_exchange_officially_supported(exchange_name: str) -> bool:
return exchange_name in SUPPORTED_EXCHANGES
def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
"""
Return the list of all exchanges known to ccxt

12
freqtrade/freqai/data_drawer.py
View File

@@ -264,7 +264,7 @@ class FreqaiDataDrawer:
def append_model_predictions(self, pair: str, predictions: DataFrame,
do_preds: NDArray[np.int_],
dk: FreqaiDataKitchen, len_df: int) -> None:
dk: FreqaiDataKitchen, strat_df: DataFrame) -> None:
"""
Append model predictions to historic predictions dataframe, then set the
strategy return dataframe to the tail of the historic predictions. The length of
@@ -273,6 +273,7 @@ class FreqaiDataDrawer:
historic predictions.
"""
len_df = len(strat_df)
index = self.historic_predictions[pair].index[-1:]
columns = self.historic_predictions[pair].columns
@@ -300,6 +301,15 @@ class FreqaiDataDrawer:
for return_str in rets:
df[return_str].iloc[-1] = rets[return_str]
# this logic carries users between version without needing to
# change their identifier
if 'close_price' not in df.columns:
df['close_price'] = np.nan
df['date_pred'] = np.nan
df['close_price'].iloc[-1] = strat_df['close'].iloc[-1]
df['date_pred'].iloc[-1] = strat_df['date'].iloc[-1]
self.model_return_values[pair] = df.tail(len_df).reset_index(drop=True)
def attach_return_values_to_return_dataframe(

9
freqtrade/freqai/freqai_interface.py
View File

@@ -407,7 +407,7 @@ class IFreqaiModel(ABC):
# allows FreqUI to show full return values.
pred_df, do_preds = self.predict(dataframe, dk)
if pair not in self.dd.historic_predictions:
self.set_initial_historic_predictions(pred_df, dk, pair)
self.set_initial_historic_predictions(pred_df, dk, pair, dataframe)
self.dd.set_initial_return_values(pair, pred_df)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
@@ -428,7 +428,7 @@ class IFreqaiModel(ABC):
if self.freqai_info.get('fit_live_predictions_candles', 0) and self.live:
self.fit_live_predictions(dk, pair)
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, len(dataframe))
self.dd.append_model_predictions(pair, pred_df, do_preds, dk, dataframe)
dk.return_dataframe = self.dd.attach_return_values_to_return_dataframe(pair, dataframe)
return
@@ -597,7 +597,7 @@ class IFreqaiModel(ABC):
self.dd.purge_old_models()
def set_initial_historic_predictions(
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str
self, pred_df: DataFrame, dk: FreqaiDataKitchen, pair: str, strat_df: DataFrame
) -> None:
"""
This function is called only if the datadrawer failed to load an
@@ -640,6 +640,9 @@ class IFreqaiModel(ABC):
for return_str in dk.data['extra_returns_per_train']:
hist_preds_df[return_str] = 0
hist_preds_df['close_price'] = strat_df['close']
hist_preds_df['date_pred'] = strat_df['date']
# # for keras type models, the conv_window needs to be prepended so
# # viewing is correct in frequi
if self.freqai_info.get('keras', False) or self.ft_params.get('inlier_metric_window', 0):

87
freqtrade/freqtradebot.py
View File

@@ -1311,7 +1311,7 @@ class FreqtradeBot(LoggingMixin):
# place new order only if new price is supplied
self.execute_entry(
pair=trade.pair,
stake_amount=(order_obj.remaining * order_obj.price),
stake_amount=(order_obj.remaining * order_obj.price / trade.leverage),
price=adjusted_entry_price,
trade=trade,
is_short=trade.is_short,
@@ -1389,11 +1389,13 @@ class FreqtradeBot(LoggingMixin):
reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}"
else:
self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None
logger.info(f'{side} Order timeout for {trade}.')
else:
# update_trade_state (and subsequently recalc_trade_from_orders) will handle updates
# to the trade object
self.update_trade_state(trade, trade.open_order_id, corder)
trade.open_order_id = None
logger.info(f'Partial {trade.entry_side} order timeout for {trade}.')
reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
@@ -1409,47 +1411,63 @@ class FreqtradeBot(LoggingMixin):
:return: True if exit order was cancelled, false otherwise
"""
cancelled = False
# if trade is not partially completed, just cancel the order
if order['remaining'] == order['amount'] or order.get('filled') == 0.0:
if not self.exchange.check_order_canceled_empty(order):
try:
# if trade is not partially completed, just delete the order
co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
trade.update_order(co)
except InvalidOrderException:
logger.exception(
f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
return False
logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
else:
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
logger.info('%s order %s for %s.', trade.exit_side.capitalize(), reason, trade)
trade.update_order(order)
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
filled_val: float = order.get('filled', 0.0) or 0.0
filled_rem_stake = trade.stake_amount - filled_val * trade.open_rate
minstake = self.exchange.get_min_pair_stake_amount(
trade.pair, trade.open_rate, self.strategy.stoploss)
# Double-check remaining amount
if filled_val > 0:
reason = constants.CANCEL_REASON['PARTIALLY_FILLED']
if minstake and filled_rem_stake < minstake:
logger.warning(
f"Order {trade.open_order_id} for {trade.pair} not cancelled, as "
f"the filled amount of {filled_val} would result in an unexitable trade.")
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
self._notify_exit_cancel(
trade,
order_type=self.strategy.order_types['exit'],
reason=reason, order_id=order['id'],
sub_trade=trade.amount != order['amount']
)
return False
try:
co = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
except InvalidOrderException:
logger.exception(
f"Could not cancel {trade.exit_side} order {trade.open_order_id}")
return False
trade.close_rate = None
trade.close_rate_requested = None
trade.close_profit = None
trade.close_profit_abs = None
trade.open_order_id = None
trade.exit_reason = None
# Set exit_reason for fill message
exit_reason_prev = trade.exit_reason
trade.exit_reason = trade.exit_reason + f", {reason}" if trade.exit_reason else reason
self.update_trade_state(trade, trade.open_order_id, co)
# Order might be filled above in odd timing issues.
if co.get('status') in ('canceled', 'cancelled'):
trade.exit_reason = None
trade.open_order_id = None
else:
trade.exit_reason = exit_reason_prev
logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
cancelled = True
self.wallets.update()
else:
# TODO: figure out how to handle partially complete sell orders
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
cancelled = False
reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE']
logger.info(f'{trade.exit_side.capitalize()} order {reason} for {trade}.')
self.update_trade_state(trade, trade.open_order_id, order)
trade.open_order_id = None
order_obj = trade.select_order_by_order_id(order['id'])
if not order_obj:
raise DependencyException(
f"Order_obj not found for {order['id']}. This should not have happened.")
sub_trade = order_obj.amount != trade.amount
self._notify_exit_cancel(
trade,
order_type=self.strategy.order_types['exit'],
reason=reason, order=order_obj, sub_trade=sub_trade
reason=reason, order_id=order['id'], sub_trade=trade.amount != order['amount']
)
return cancelled
@@ -1646,7 +1664,7 @@ class FreqtradeBot(LoggingMixin):
self.rpc.send_msg(msg)
def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str,
order: Order, sub_trade: bool = False) -> None:
order_id: str, sub_trade: bool = False) -> None:
"""
Sends rpc notification when a sell cancel occurred.
"""
@@ -1655,6 +1673,11 @@ class FreqtradeBot(LoggingMixin):
else:
trade.exit_order_status = reason
order = trade.select_order_by_order_id(order_id)
if not order:
raise DependencyException(
f"Order_obj not found for {order_id}. This should not have happened.")
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
current_rate = self.exchange.get_rate(

2
freqtrade/optimize/backtesting.py
View File

@@ -1045,7 +1045,7 @@ class Backtesting:
if requested_rate:
self._enter_trade(pair=trade.pair, row=row, trade=trade,
requested_rate=requested_rate,
requested_stake=(order.remaining * order.price),
requested_stake=(order.remaining * order.price / trade.leverage),
direction='short' if trade.is_short else 'long')
self.replaced_entry_orders += 1
else:

7
freqtrade/optimize/hyperopt.py
View File

@@ -24,6 +24,7 @@ from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN, Config
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.data.metrics import calculate_market_change
from freqtrade.enums import HyperoptState
from freqtrade.exceptions import OperationalException
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
@@ -111,6 +112,7 @@ class Hyperopt:
self.clean_hyperopt()
self.market_change = 0.0
self.num_epochs_saved = 0
self.current_best_epoch: Optional[Dict[str, Any]] = None
@@ -357,7 +359,7 @@ class Hyperopt:
strat_stats = generate_strategy_stats(
self.pairlist, self.backtesting.strategy.get_strategy_name(),
backtesting_results, min_date, max_date, market_change=0
backtesting_results, min_date, max_date, market_change=self.market_change
)
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
@@ -425,6 +427,9 @@ class Hyperopt:
# Trim startup period from analyzed dataframe to get correct dates for output.
trimmed = trim_dataframes(preprocessed, self.timerange, self.backtesting.required_startup)
self.min_date, self.max_date = get_timerange(trimmed)
if not self.market_change:
self.market_change = calculate_market_change(trimmed, 'close')
# Real trimming will happen as part of backtesting.
return preprocessed

6
freqtrade/rpc/api_server/webserver.py
View File

@@ -198,8 +198,10 @@ class ApiServer(RPCHandler):
logger.debug(f"Found message of type: {message.get('type')}")
# Broadcast it
await self._ws_channel_manager.broadcast(message)
# Sleep, make this configurable?
await asyncio.sleep(0.1)
# Limit messages per sec.
# Could cause problems with queue size if too low, and
# problems with network traffik if too high.
await asyncio.sleep(0.001)
except asyncio.CancelledError:
pass