adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feat/short' into lev-exchange

Browse Source
This commit is contained in:
Sam Germain
2021-09-04 20:15:36 -06:00
parent d4389eb07d 5184cc7749
commit 7eab855476
95 changed files with 1757 additions and 1148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/commands/arguments.py
View File

@@ -22,7 +22,7 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
"max_open_trades", "stake_amount", "fee", "pairs"]
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
"enable_protections", "dry_run_wallet",
"enable_protections", "dry_run_wallet", "timeframe_detail",
"strategy_list", "export", "exportfilename"]
ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",

10
freqtrade/commands/build_config_commands.py
View File

@@ -66,16 +66,22 @@ def ask_user_config() -> Dict[str, Any]:
{
"type": "text",
"name": "stake_amount",
"message": "Please insert your stake amount:",
"message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
"default": "0.01",
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
if val == UNLIMITED_STAKE_AMOUNT
else val
},
{
"type": "text",
"name": "max_open_trades",
"message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):",
"default": "3",
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val)
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val),
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
if val == UNLIMITED_STAKE_AMOUNT
else val
},
{
"type": "text",

6
freqtrade/commands/cli_options.py
View File

@@ -135,6 +135,10 @@ AVAILABLE_CLI_OPTIONS = {
help='Override the value of the `stake_amount` configuration setting.',
),
# Backtesting
"timeframe_detail": Arg(
'--timeframe-detail',
help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).',
),
"position_stacking": Arg(
'--eps', '--enable-position-stacking',
help='Allow buying the same pair multiple times (position stacking).',
@@ -162,7 +166,7 @@ AVAILABLE_CLI_OPTIONS = {
'Please note that ticker-interval needs to be set either in config '
'or via command line. When using this together with `--export trades`, '
'the strategy-name is injected into the filename '
'(so `backtest-data.json` becomes `backtest-data-DefaultStrategy.json`',
'(so `backtest-data.json` becomes `backtest-data-SampleStrategy.json`',
nargs='+',
),
"export": Arg(

4
freqtrade/commands/deploy_commands.py
View File

@@ -74,8 +74,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if "strategy" in args and args["strategy"]:
if args["strategy"] == "DefaultStrategy":
raise OperationalException("DefaultStrategy is not allowed as name.")
new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
@@ -128,8 +126,6 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if 'hyperopt' in args and args['hyperopt']:
if args['hyperopt'] == 'DefaultHyperopt':
raise OperationalException("DefaultHyperopt is not allowed as name.")
new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')

180
freqtrade/commands/hyperopt_commands.py
View File

@@ -1,6 +1,6 @@
import logging
from operator import itemgetter
from typing import Any, Dict, List
from typing import Any, Dict
from colorama import init as colorama_init
@@ -28,30 +28,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
no_details = config.get('hyperopt_list_no_details', False)
no_header = False
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
}
results_file = get_latest_hyperopt_file(
config['user_data_dir'] / 'hyperopt_results',
config.get('hyperoptexportfilename'))
# Previous evaluations
epochs = HyperoptTools.load_previous_results(results_file)
total_epochs = len(epochs)
epochs = hyperopt_filter_epochs(epochs, filteroptions)
epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
if print_colorized:
colorama_init(autoreset=True)
@@ -59,7 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
if not export_csv:
try:
print(HyperoptTools.get_result_table(config, epochs, total_epochs,
not filteroptions['only_best'],
not config.get('hyperopt_list_best', False),
print_colorized, 0))
except KeyboardInterrupt:
print('User interrupted..')
@@ -71,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
if epochs and export_csv:
HyperoptTools.export_csv_file(
config, epochs, total_epochs, not filteroptions['only_best'], export_csv
config, epochs, total_epochs, not config.get('hyperopt_list_best', False), export_csv
)
@@ -91,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
n = config.get('hyperopt_show_index', -1)
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
'filter_max_objective': config.get('hyperopt_list_max_objective', None)
}
# Previous evaluations
epochs = HyperoptTools.load_previous_results(results_file)
total_epochs = len(epochs)
epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
epochs = hyperopt_filter_epochs(epochs, filteroptions)
filtered_epochs = len(epochs)
if n > filtered_epochs:
@@ -137,138 +102,3 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
header_str="Epoch details")
def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
"""
Filter our items from the list of hyperopt results
TODO: after 2021.5 remove all "legacy" mode queries.
"""
if filteroptions['only_best']:
epochs = [x for x in epochs if x['is_best']]
if filteroptions['only_profitable']:
epochs = [x for x in epochs if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total', 0)) > 0]
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
logger.info(f"{len(epochs)} " +
("best " if filteroptions['only_best'] else "") +
("profitable " if filteroptions['only_profitable'] else "") +
"epochs found.")
return epochs
def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
"""
Filter epochs with trade-counts > trades
"""
return [
x for x in epochs
if x['results_metrics'].get(
'trade_count', x['results_metrics'].get('total_trades', 0)
) > trade_count
]
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_trades'] > 0:
epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
if filteroptions['filter_max_trades'] > 0:
epochs = [
x for x in epochs
if x['results_metrics'].get(
'trade_count', x['results_metrics'].get('total_trades')
) < filteroptions['filter_max_trades']
]
return epochs
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
def get_duration_value(x):
# Duration in minutes ...
if 'duration' in x['results_metrics']:
return x['results_metrics']['duration']
else:
# New mode
if 'holding_avg_s' in x['results_metrics']:
avg = x['results_metrics']['holding_avg_s']
return avg // 60
raise OperationalException(
"Holding-average not available. Please omit the filter on average time, "
"or rerun hyperopt with this version")
if filteroptions['filter_min_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) > filteroptions['filter_min_avg_time']
]
if filteroptions['filter_max_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) < filteroptions['filter_max_avg_time']
]
return epochs
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
) > filteroptions['filter_min_avg_profit']
]
if filteroptions['filter_max_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
) < filteroptions['filter_max_avg_profit']
]
if filteroptions['filter_min_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total_abs', 0)
) > filteroptions['filter_min_total_profit']
]
if filteroptions['filter_max_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total_abs', 0)
) < filteroptions['filter_max_total_profit']
]
return epochs
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
if filteroptions['filter_max_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
return epochs

3
freqtrade/configuration/configuration.py
View File

@@ -242,6 +242,9 @@ class Configuration:
except ValueError:
pass
self._args_to_config(config, argname='timeframe_detail',
logstring='Parameter --timeframe-detail detected, '
'using {} for intra-candle backtesting ...')
self._args_to_config(config, argname='stake_amount',
logstring='Parameter --stake-amount detected, '
'overriding stake_amount to: {} ...')

5
freqtrade/constants.py
View File

@@ -49,6 +49,8 @@ USERPATH_NOTEBOOKS = 'notebooks'
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
ENV_VAR_PREFIX = 'FREQTRADE__'
NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
# Define decimals per coin for outputs
# Only used for outputs.
@@ -191,6 +193,9 @@ CONF_SCHEMA = {
},
'required': ['price_side']
},
'custom_price_max_distance_ratio': {
'type': 'number', 'minimum': 0.0
},
'order_types': {
'type': 'object',
'properties': {

2
freqtrade/data/btanalysis.py
View File

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
"trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
# Mid-term format, crated by BacktestResult Named Tuple
# Mid-term format, created by BacktestResult Named Tuple
BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
'fee_close', 'amount', 'profit_abs', 'profit_ratio']

4
freqtrade/data/converter.py
View File

@@ -242,7 +242,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase souce data (does not apply if source and target format are identical)
:param erase: Erase source data (does not apply if source and target format are identical)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)
@@ -267,7 +267,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase souce data (does not apply if source and target format are identical)
:param erase: Erase source data (does not apply if source and target format are identical)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)

33
freqtrade/data/history/history_utils.py
View File

@@ -117,10 +117,11 @@ def refresh_data(datadir: Path,
:param timerange: Limit data to be loaded to this timerange
"""
data_handler = get_datahandler(datadir, data_format)
for pair in pairs:
_download_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
exchange=exchange, data_handler=data_handler)
for idx, pair in enumerate(pairs):
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
timeframe=timeframe, datadir=datadir,
timerange=timerange, exchange=exchange, data_handler=data_handler)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -153,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
return data, start_ms
def _download_pair_history(datadir: Path,
def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler = None) -> bool:
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
@@ -177,7 +179,7 @@ def _download_pair_history(datadir: Path,
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
f'and store in {datadir}.'
)
@@ -234,7 +236,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
"""
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format)
for pair in pairs:
for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
@@ -247,10 +249,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
f'Deleting existing data for pair {pair}, interval {timeframe}.')
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
_download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe),
new_pairs_days=new_pairs_days,
timerange=timerange, data_handler=data_handler)
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
return pairs_not_available

2
freqtrade/edge/edge_positioning.py
View File

@@ -151,7 +151,7 @@ class Edge:
# Fake run-mode to Edge
prior_rm = self.config['runmode']
self.config['runmode'] = RunMode.EDGE
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
preprocessed = self.strategy.advise_all_indicators(data)
self.config['runmode'] = prior_rm
# Print timeframe

1
freqtrade/exchange/__init__.py
View File

@@ -15,6 +15,7 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
timeframe_to_seconds, validate_exchange,
validate_exchanges)
from freqtrade.exchange.ftx import Ftx
from freqtrade.exchange.gateio import Gateio
from freqtrade.exchange.hitbtc import Hitbtc
from freqtrade.exchange.kraken import Kraken
from freqtrade.exchange.kucoin import Kucoin

34
freqtrade/exchange/exchange.py
View File

@@ -19,7 +19,8 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRU
decimal_to_precision)
from pandas import DataFrame
from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes
from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES,
ListPairsWithTimeframes)
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.enums import Collateral
from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
@@ -366,9 +367,16 @@ class Exchange:
def validate_stakecurrency(self, stake_currency: str) -> None:
"""
Checks stake-currency against available currencies on the exchange.
Only runs on startup. If markets have not been loaded, there's been a problem with
the connection to the exchange.
:param stake_currency: Stake-currency to validate
:raise: OperationalException if stake-currency is not available.
"""
if not self._markets:
raise OperationalException(
'Could not load markets, therefore cannot start. '
'Please investigate the above error for more details.'
)
quote_currencies = self.get_quote_currencies()
if stake_currency not in quote_currencies:
raise OperationalException(
@@ -646,6 +654,8 @@ class Exchange:
if self.exchange_has('fetchL2OrderBook'):
ob = self.fetch_l2_order_book(pair, 20)
ob_type = 'asks' if side == 'buy' else 'bids'
slippage = 0.05
max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage))
remaining_amount = amount
filled_amount = 0
@@ -654,7 +664,9 @@ class Exchange:
book_entry_coin_volume = book_entry[1]
if remaining_amount > 0:
if remaining_amount < book_entry_coin_volume:
# Orderbook at this slot bigger than remaining amount
filled_amount += remaining_amount * book_entry_price
break
else:
filled_amount += book_entry_coin_volume * book_entry_price
remaining_amount -= book_entry_coin_volume
@@ -663,7 +675,14 @@ class Exchange:
else:
# If remaining_amount wasn't consumed completely (break was not called)
filled_amount += remaining_amount * book_entry_price
forecast_avg_filled_price = filled_amount / amount
forecast_avg_filled_price = max(filled_amount, 0) / amount
# Limit max. slippage to specified value
if side == 'buy':
forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val)
else:
forecast_avg_filled_price = max(forecast_avg_filled_price, max_slippage_val)
return self.price_to_precision(pair, forecast_avg_filled_price)
return rate
@@ -822,7 +841,7 @@ class Exchange:
:param order: Order dict as returned from fetch_order()
:return: True if order has been cancelled without being filled, False otherwise.
"""
return (order.get('status') in ('closed', 'canceled', 'cancelled')
return (order.get('status') in NON_OPEN_EXCHANGE_STATES
and order.get('filled') == 0.0)
@retrier
@@ -1056,7 +1075,7 @@ class Exchange:
logger.debug(f"Using Last {conf_strategy['price_side'].capitalize()} / Last Price")
ticker = self.fetch_ticker(pair)
ticker_rate = ticker[conf_strategy['price_side']]
if ticker['last']:
if ticker['last'] and ticker_rate:
if side == 'buy' and ticker_rate > ticker['last']:
balance = conf_strategy['ask_last_balance']
ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
@@ -1271,7 +1290,7 @@ class Exchange:
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
input_coroutines = []
cached_pairs = []
# Gather coroutines to run
for pair, timeframe in set(pair_list):
if (((pair, timeframe) not in self._klines)
@@ -1283,6 +1302,7 @@ class Exchange:
"Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
pair, timeframe
)
cached_pairs.append((pair, timeframe))
results = asyncio.get_event_loop().run_until_complete(
asyncio.gather(*input_coroutines, return_exceptions=True))
@@ -1305,6 +1325,10 @@ class Exchange:
results_df[(pair, timeframe)] = ohlcv_df
if cache:
self._klines[(pair, timeframe)] = ohlcv_df
# Return cached klines
for pair, timeframe in cached_pairs:
results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False)
return results_df
def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool:

23
freqtrade/exchange/gateio.py Normal file
View File

@@ -0,0 +1,23 @@
""" Gate.io exchange subclass """
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Gateio(Exchange):
"""
Gate.io exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
}

70
freqtrade/freqtradebot.py
View File

@@ -433,11 +433,11 @@ class FreqtradeBot(LoggingMixin):
if ((bid_check_dom.get('enabled', False)) and
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
if self._check_depth_of_market_buy(pair, bid_check_dom):
return self.execute_buy(pair, stake_amount, buy_tag=buy_tag)
return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
else:
return False
return self.execute_buy(pair, stake_amount, buy_tag=buy_tag)
return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
else:
return False
@@ -465,8 +465,8 @@ class FreqtradeBot(LoggingMixin):
logger.info(f"Bids to asks delta for {pair} does not satisfy condition.")
return False
def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None,
forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool:
def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None,
forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool:
"""
Executes a limit buy for the given pair
:param pair: pair for which we want to create a LIMIT_BUY
@@ -479,7 +479,13 @@ class FreqtradeBot(LoggingMixin):
buy_limit_requested = price
else:
# Calculate price
buy_limit_requested = self.exchange.get_rate(pair, refresh=True, side="buy")
proposed_buy_rate = self.exchange.get_rate(pair, refresh=True, side="buy")
custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price,
default_retval=proposed_buy_rate)(
pair=pair, current_time=datetime.now(timezone.utc),
proposed_rate=proposed_buy_rate)
buy_limit_requested = self.get_valid_price(custom_entry_price, proposed_buy_rate)
if not buy_limit_requested:
raise PricingError('Could not determine buy price.')
@@ -743,7 +749,7 @@ class FreqtradeBot(LoggingMixin):
trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}')
logger.warning('Selling the trade forcefully')
self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple(
sell_type=SellType.EMERGENCY_SELL))
except ExchangeError:
@@ -861,7 +867,7 @@ class FreqtradeBot(LoggingMixin):
if should_sell.sell_flag:
logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
self.execute_sell(trade, sell_rate, should_sell)
self.execute_trade_exit(trade, sell_rate, should_sell)
return True
return False
@@ -943,7 +949,7 @@ class FreqtradeBot(LoggingMixin):
was_trade_fully_canceled = False
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in ('cancelled', 'canceled', 'closed'):
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
filled_val = order.get('filled', 0.0) or 0.0
filled_stake = filled_val * trade.open_rate
minstake = self.exchange.get_min_pair_stake_amount(
@@ -959,7 +965,7 @@ class FreqtradeBot(LoggingMixin):
# Avoid race condition where the order could not be cancelled coz its already filled.
# Simply bailing here is the only safe way - as this order will then be
# handled in the next iteration.
if corder.get('status') not in ('cancelled', 'canceled', 'closed'):
if corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES:
logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.")
return False
else:
@@ -981,7 +987,7 @@ class FreqtradeBot(LoggingMixin):
# if trade is partially complete, edit the stake details for the trade
# and close the order
# cancel_order may not contain the full order dict, so we need to fallback
# to the order dict aquired before cancelling.
# to the order dict acquired before cancelling.
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = filled_amount
trade.stake_amount = trade.amount * trade.open_rate
@@ -1062,9 +1068,9 @@ class FreqtradeBot(LoggingMixin):
raise DependencyException(
f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
def execute_trade_exit(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
"""
Executes a limit sell for the given trade and limit
Executes a trade exit for the given trade and limit
:param trade: Trade instance
:param limit: limit rate for the sell order
:param sell_reason: Reason the sell was triggered
@@ -1080,6 +1086,17 @@ class FreqtradeBot(LoggingMixin):
and self.strategy.order_types['stoploss_on_exchange']:
limit = trade.stop_loss
# set custom_exit_price if available
proposed_limit_rate = limit
current_profit = trade.calc_profit_ratio(limit)
custom_exit_price = strategy_safe_wrapper(self.strategy.custom_exit_price,
default_retval=proposed_limit_rate)(
pair=trade.pair, trade=trade,
current_time=datetime.now(timezone.utc),
proposed_rate=proposed_limit_rate, current_profit=current_profit)
limit = self.get_valid_price(custom_exit_price, proposed_limit_rate)
# First cancelling stoploss on exchange ...
if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
try:
@@ -1129,7 +1146,7 @@ class FreqtradeBot(LoggingMixin):
trade.close_rate_requested = limit
trade.sell_reason = sell_reason.sell_reason
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') == 'closed':
if order.get('status', 'unknown') in ('closed', 'expired'):
self.update_trade_state(trade, trade.open_order_id, order)
Trade.commit()
@@ -1368,7 +1385,9 @@ class FreqtradeBot(LoggingMixin):
if fee_currency:
# fee_rate should use mean
fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None
trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
if fee_rate is not None and fee_rate < 0.02:
# Only update if fee-rate is < 2%
trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
logger.warning(f"Amount {amount} does not match amount {trade.amount}")
@@ -1379,3 +1398,26 @@ class FreqtradeBot(LoggingMixin):
amount=amount, fee_abs=fee_abs)
else:
return amount
def get_valid_price(self, custom_price: float, proposed_price: float) -> float:
"""
Return the valid price.
Check if the custom price is of the good type if not return proposed_price
:return: valid price for the order
"""
if custom_price:
try:
valid_custom_price = float(custom_price)
except ValueError:
valid_custom_price = proposed_price
else:
valid_custom_price = proposed_price
cust_p_max_dist_r = self.config.get('custom_price_max_distance_ratio', 0.02)
min_custom_price_allowed = proposed_price - (proposed_price * cust_p_max_dist_r)
max_custom_price_allowed = proposed_price + (proposed_price * cust_p_max_dist_r)
# Bracket between min_custom_price_allowed and max_custom_price_allowed
return max(
min(valid_custom_price, max_custom_price_allowed),
min_custom_price_allowed)

94
freqtrade/optimize/backtesting.py
View File

@@ -86,6 +86,17 @@ class Backtesting:
"configuration or as cli argument `--timeframe 5m`")
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
# Load detail timeframe if specified
self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
if self.timeframe_detail:
self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
if self.timeframe_min <= self.timeframe_detail_min:
raise OperationalException(
"Detail timeframe must be smaller than strategy timeframe.")
else:
self.timeframe_detail_min = 0
self.detail_data: Dict[str, DataFrame] = {}
self.pairlists = PairListManager(self.exchange, self.config)
if 'VolumePairList' in self.pairlists.name_list:
@@ -130,6 +141,9 @@ class Backtesting:
self.abort = False
def __del__(self):
self.cleanup()
def cleanup(self):
LoggingMixin.show_output = True
PairLocks.use_db = True
Trade.use_db = True
@@ -185,6 +199,23 @@ class Backtesting:
self.progress.set_new_value(1)
return data, self.timerange
def load_bt_data_detail(self) -> None:
"""
Loads backtest detail data (smaller timeframe) if necessary.
"""
if self.timeframe_detail:
self.detail_data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe_detail,
timerange=self.timerange,
startup_candles=0,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
else:
self.detail_data = {}
def prepare_backtest(self, enable_protections):
"""
Backtesting setup method - called once for every call to "backtest()".
@@ -215,7 +246,7 @@ class Backtesting:
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_tag']
data: Dict = {}
self.progress.init_step(BacktestState.CONVERT, len(processed))
@@ -223,13 +254,10 @@ class Backtesting:
for pair, pair_data in processed.items():
self.check_abort()
self.progress.increment()
has_buy_tag = 'buy_tag' in pair_data
headers = headers + ['buy_tag'] if has_buy_tag else headers
if not pair_data.empty:
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
if has_buy_tag:
pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist
pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}),
@@ -242,14 +270,13 @@ class Backtesting:
# from the previous candle
df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
if has_buy_tag:
df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
# Update dataprovider cache
self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed)
df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = df_analyzed[headers].values.tolist()
@@ -321,15 +348,16 @@ class Backtesting:
else:
return sell_row[OPEN_IDX]
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
def _get_sell_trade_entry_for_candle(self, trade: LocalTrade,
sell_row: Tuple) -> Optional[LocalTrade]:
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore
sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
sell_candle_time, sell_row[BUY_IDX],
sell_row[SELL_IDX],
low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
if sell.sell_flag:
trade.close_date = sell_row[DATE_IDX].to_pydatetime()
trade.close_date = sell_candle_time
trade.sell_reason = sell.sell_reason
trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
@@ -341,7 +369,7 @@ class Backtesting:
rate=closerate,
time_in_force=time_in_force,
sell_reason=sell.sell_reason,
current_time=sell_row[DATE_IDX].to_pydatetime()):
current_time=sell_candle_time):
return None
trade.close(closerate, show_msg=False)
@@ -349,6 +377,32 @@ class Backtesting:
return None
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
if self.timeframe_detail and trade.pair in self.detail_data:
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min)
detail_data = self.detail_data[trade.pair]
detail_data = detail_data.loc[
(detail_data['date'] >= sell_candle_time) &
(detail_data['date'] < sell_candle_end)
]
if len(detail_data) == 0:
# Fall back to "regular" data if no detail data was found for this candle
return self._get_sell_trade_entry_for_candle(trade, sell_row)
detail_data['buy'] = sell_row[BUY_IDX]
detail_data['sell'] = sell_row[SELL_IDX]
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
for det_row in detail_data[headers].values.tolist():
res = self._get_sell_trade_entry_for_candle(trade, det_row)
if res:
return res
return None
else:
return self._get_sell_trade_entry_for_candle(trade, sell_row)
def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
try:
stake_amount = self.wallets.get_trade_stake_amount(pair, None)
@@ -465,6 +519,8 @@ class Backtesting:
for i, pair in enumerate(data):
row_index = indexes[pair]
try:
# Row is treated as "current incomplete candle".
# Buy / sell signals are shifted by 1 to compensate for this.
row = data[pair][row_index]
except IndexError:
# missing Data for one pair at the end.
@@ -476,8 +532,8 @@ class Backtesting:
continue
row_index += 1
self.dataprovider._set_dataframe_max_index(row_index)
indexes[pair] = row_index
self.dataprovider._set_dataframe_max_index(row_index)
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
@@ -501,7 +557,7 @@ class Backtesting:
open_trades[pair].append(trade)
LocalTrade.add_bt_trade(trade)
for trade in open_trades[pair]:
for trade in list(open_trades[pair]):
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(trade, row)
# Sell occurred
@@ -532,7 +588,8 @@ class Backtesting:
'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
}
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, DataFrame],
timerange: TimeRange):
self.progress.init_step(BacktestState.ANALYZE, 0)
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
@@ -551,7 +608,7 @@ class Backtesting:
max_open_trades = 0
# need to reprocess data every time to populate signals
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
preprocessed = self.strategy.advise_all_indicators(data)
# Trim startup period from analyzed dataframe
preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup)
@@ -592,6 +649,7 @@ class Backtesting:
data: Dict[str, Any] = {}
data, timerange = self.load_bt_data()
self.load_bt_data_detail()
logger.info("Dataload complete. Calculating indicators")
for strat in self.strategylist:

20
freqtrade/optimize/hyperopt.py
View File

@@ -107,13 +107,25 @@ class Hyperopt:
# Populate "fallback" functions here
# (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = ( # type: ignore
logger.warning(
"DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. "
"Please move these methods to your strategy."
)
self.backtesting.strategy.populate_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
self.backtesting.strategy.advise_buy = ( # type: ignore
logger.warning(
"DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. "
"Please move these methods to your strategy."
)
self.backtesting.strategy.populate_buy_trend = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.backtesting.strategy.advise_sell = ( # type: ignore
logger.warning(
"DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. "
"Please move these methods to your strategy."
)
self.backtesting.strategy.populate_sell_trend = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
@@ -394,7 +406,7 @@ class Hyperopt:
data, timerange = self.backtesting.load_bt_data()
logger.info("Dataload complete. Calculating indicators")
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
preprocessed = self.backtesting.strategy.advise_all_indicators(data)
# Trim startup period from analyzed dataframe to get correct dates for output.
processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)

2
freqtrade/optimize/hyperopt_auto.py
View File

@@ -74,7 +74,7 @@ class HyperOptAuto(IHyperOpt):
return self._get_indicator_space('sell', 'sell_indicator_space')
def protection_space(self) -> List['Dimension']:
return self._get_indicator_space('protection', 'indicator_space')
return self._get_indicator_space('protection', 'protection_space')
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
return self._get_func('generate_roi_table')(params)

128
freqtrade/optimize/hyperopt_epoch_filters.py Normal file
View File

@@ -0,0 +1,128 @@
import logging
from typing import List
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) -> List:
"""
Filter our items from the list of hyperopt results
"""
if filteroptions['only_best']:
epochs = [x for x in epochs if x['is_best']]
if filteroptions['only_profitable']:
epochs = [x for x in epochs
if x['results_metrics'].get('profit_total', 0) > 0]
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
if log:
logger.info(f"{len(epochs)} " +
("best " if filteroptions['only_best'] else "") +
("profitable " if filteroptions['only_profitable'] else "") +
"epochs found.")
return epochs
def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
"""
Filter epochs with trade-counts > trades
"""
return [
x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count
]
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_trades'] > 0:
epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
if filteroptions['filter_max_trades'] > 0:
epochs = [
x for x in epochs
if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades']
]
return epochs
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
def get_duration_value(x):
# Duration in minutes ...
if 'holding_avg_s' in x['results_metrics']:
avg = x['results_metrics']['holding_avg_s']
return avg // 60
raise OperationalException(
"Holding-average not available. Please omit the filter on average time, "
"or rerun hyperopt with this version")
if filteroptions['filter_min_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) > filteroptions['filter_min_avg_time']
]
if filteroptions['filter_max_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) < filteroptions['filter_max_avg_time']
]
return epochs
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_mean', 0) * 100
> filteroptions['filter_min_avg_profit']
]
if filteroptions['filter_max_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_mean', 0) * 100
< filteroptions['filter_max_avg_profit']
]
if filteroptions['filter_min_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_total_abs', 0)
> filteroptions['filter_min_total_profit']
]
if filteroptions['filter_max_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_total_abs', 0)
< filteroptions['filter_max_total_profit']
]
return epochs
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
if filteroptions['filter_max_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
return epochs

0
freqtrade/optimize/default_hyperopt_loss.py → freqtrade/optimize/hyperopt_loss_short_trade_dur.py
View File

111
freqtrade/optimize/hyperopt_tools.py
View File

@@ -4,7 +4,7 @@ import logging
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, Iterator, List, Optional, Tuple
import numpy as np
import rapidjson
@@ -15,6 +15,7 @@ from pandas import isna, json_normalize
from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
logger = logging.getLogger(__name__)
@@ -89,46 +90,70 @@ class HyperoptTools():
return any(s in config['spaces'] for s in [space, 'all', 'default'])
@staticmethod
def _read_results_pickle(results_file: Path) -> List:
def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]:
"""
Read hyperopt results from pickle file
LEGACY method - new files are written as json and cannot be read with this method.
"""
from joblib import load
logger.info(f"Reading pickled epochs from '{results_file}'")
data = load(results_file)
return data
@staticmethod
def _read_results(results_file: Path) -> List:
"""
Read hyperopt results from file
Stream hyperopt results from file
"""
import rapidjson
logger.info(f"Reading epochs from '{results_file}'")
with results_file.open('r') as f:
data = [rapidjson.loads(line) for line in f]
return data
data = []
for line in f:
data += [rapidjson.loads(line)]
if len(data) >= batch_size:
yield data
data = []
yield data
@staticmethod
def load_previous_results(results_file: Path) -> List:
"""
Load data for epochs from the file if we have one
"""
epochs: List = []
def _test_hyperopt_results_exist(results_file) -> bool:
if results_file.is_file() and results_file.stat().st_size > 0:
if results_file.suffix == '.pickle':
epochs = HyperoptTools._read_results_pickle(results_file)
else:
epochs = HyperoptTools._read_results(results_file)
# Detection of some old format, without 'is_best' field saved
if epochs[0].get('is_best') is None:
raise OperationalException(
"Legacy hyperopt results are no longer supported."
"Please rerun hyperopt or use an older version to load this file."
)
return True
else:
# No file found.
return False
@staticmethod
def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]:
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
}
if not HyperoptTools._test_hyperopt_results_exist(results_file):
# No file found.
return [], 0
epochs = []
total_epochs = 0
for epochs_tmp in HyperoptTools._read_results(results_file):
if total_epochs == 0 and epochs_tmp[0].get('is_best') is None:
raise OperationalException(
"The file with HyperoptTools results is incompatible with this version "
"of Freqtrade and cannot be loaded.")
logger.info(f"Loaded {len(epochs)} previous evaluations from disk.")
return epochs
total_epochs += len(epochs_tmp)
epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False)
logger.info(f"Loaded {total_epochs} previous evaluations from disk.")
# Final filter run ...
epochs = hyperopt_filter_epochs(epochs, filteroptions, log=True)
return epochs, total_epochs
@staticmethod
def show_epoch_details(results, total_epochs: int, print_json: bool,
@@ -433,21 +458,14 @@ class HyperoptTools():
trials['Best'] = ''
trials['Stake currency'] = config['stake_currency']
if 'results_metrics.total_trades' in trials:
base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.profit_mean', 'results_metrics.profit_median',
'results_metrics.profit_total',
'Stake currency',
'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
'loss', 'is_initial_point', 'is_best']
perc_multi = 100
else:
perc_multi = 1
base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.median_profit',
'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']
base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.profit_mean', 'results_metrics.profit_median',
'results_metrics.profit_total',
'Stake currency',
'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
'loss', 'is_initial_point', 'is_best']
perc_multi = 100
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
trials = trials[base_metrics + param_metrics]
@@ -475,11 +493,6 @@ class HyperoptTools():
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
)
if perc_multi == 1:
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: f'{x:,.1f} m' if isinstance(
x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
)
trials['Objective'] = trials['Objective'].apply(
lambda x: f'{x:,.5f}' if x != 100000 else ""
)

1
freqtrade/optimize/optimize_reports.py
View File

@@ -368,6 +368,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'max_open_trades_setting': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
'timeframe_detail': config.get('timeframe_detail', ''),
'timerange': config.get('timerange', ''),
'enable_protections': config.get('enable_protections', False),
'strategy_name': strategy,

10
freqtrade/persistence/models.py
View File

@@ -13,7 +13,7 @@ from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session
from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.schema import UniqueConstraint
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
from freqtrade.enums import SellType
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.leverage import interest
@@ -164,9 +164,9 @@ class Order(_DECL_BASE):
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
self.ft_is_open = True
if self.status in ('closed', 'canceled', 'cancelled'):
if self.status in NON_OPEN_EXCHANGE_STATES:
self.ft_is_open = False
if order.get('filled', 0) > 0:
if (order.get('filled', 0.0) or 0.0) > 0:
self.order_filled_date = datetime.now(timezone.utc)
self.order_update_date = datetime.now(timezone.utc)
@@ -451,12 +451,12 @@ class LocalTrade():
LocalTrade.trades_open = []
LocalTrade.total_profit = 0
def adjust_min_max_rates(self, current_price: float) -> None:
def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
"""
Adjust the max_rate and min_rate.
"""
self.max_rate = max(current_price, self.max_rate or self.open_rate)
self.min_rate = min(current_price, self.min_rate or self.open_rate)
self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
def adjust_stop_loss(self, current_price: float, stoploss: float,
initial: bool = False) -> None:

2
freqtrade/plot/plotting.py
View File

@@ -538,7 +538,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
- Initializes plot-script
- Get candle (OHLCV) data
- Generate Dafaframes populated with indicators and signals based on configured strategy
- Load trades excecuted during the selected period
- Load trades executed during the selected period
- Generate Plotly plot objects
- Generate plot files
:return: None

14
freqtrade/plugins/pairlist/IPairList.py
View File

@@ -150,18 +150,20 @@ class IPairList(LoggingMixin, ABC):
for pair in pairlist:
# pair is not in the generated dynamic market or has the wrong stake currency
if pair not in markets:
logger.warning(f"Pair {pair} is not compatible with exchange "
f"{self._exchange.name}. Removing it from whitelist..")
self.log_once(f"Pair {pair} is not compatible with exchange "
f"{self._exchange.name}. Removing it from whitelist..",
logger.warning)
continue
if not self._exchange.market_is_tradable(markets[pair]):
logger.warning(f"Pair {pair} is not tradable with Freqtrade."
"Removing it from whitelist..")
self.log_once(f"Pair {pair} is not tradable with Freqtrade."
"Removing it from whitelist..", logger.warning)
continue
if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
logger.warning(f"Pair {pair} is not compatible with your stake currency "
f"{self._config['stake_currency']}. Removing it from whitelist..")
self.log_once(f"Pair {pair} is not compatible with your stake currency "
f"{self._config['stake_currency']}. Removing it from whitelist..",
logger.warning)
continue
# Check if market is active

7
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -4,6 +4,7 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from functools import partial
from typing import Any, Dict, List
import arrow
@@ -115,7 +116,7 @@ class VolumePairList(IPairList):
pairlist = self._pair_cache.get('pairlist')
if pairlist:
# Item found - no refresh necessary
return pairlist
return pairlist.copy()
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
@@ -126,7 +127,7 @@ class VolumePairList(IPairList):
pairlist = [s['symbol'] for s in filtered_tickers]
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache['pairlist'] = pairlist
self._pair_cache['pairlist'] = pairlist.copy()
return pairlist
@@ -203,7 +204,7 @@ class VolumePairList(IPairList):
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
pairs = self.verify_blacklist(pairs, logger.info)
pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]

3
freqtrade/plugins/pairlist/rangestabilityfilter.py
View File

@@ -120,5 +120,6 @@ class RangeStabilityFilter(IPairList):
logger.info)
result = False
self._pair_cache[pair] = result
else:
self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
return result

3
freqtrade/rpc/api_server/api_backtest.py
View File

@@ -46,11 +46,14 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
if (
not ApiServer._bt
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail')
or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
or lastconfig.get('timerange') != btconfig['timerange']
):
from freqtrade.optimize.backtesting import Backtesting
ApiServer._bt = Backtesting(btconfig)
if ApiServer._bt.timeframe_detail:
ApiServer._bt.load_bt_data_detail()
# Only reload data if timeframe changed.
if (

1
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -324,6 +324,7 @@ class PairHistory(BaseModel):
class BacktestRequest(BaseModel):
strategy: str
timeframe: Optional[str]
timeframe_detail: Optional[str]
timerange: Optional[str]
max_open_trades: Optional[int]
stake_amount: Optional[Union[float, str]]

6
freqtrade/rpc/api_server/api_v1.py
View File

@@ -223,11 +223,11 @@ def list_strategies(config=Depends(get_config)):
@router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
def get_strategy(strategy: str, config=Depends(get_config)):
config = deepcopy(config)
config_ = deepcopy(config)
from freqtrade.resolvers.strategy_resolver import StrategyResolver
try:
strategy_obj = StrategyResolver._load_strategy(strategy, config,
extra_dir=config.get('strategy_path'))
strategy_obj = StrategyResolver._load_strategy(strategy, config_,
extra_dir=config_.get('strategy_path'))
except OperationalException:
raise HTTPException(status_code=404, detail='Strategy not found')

7
freqtrade/rpc/api_server/uvicorn_threaded.py
View File

@@ -32,8 +32,11 @@ class UvicornServer(uvicorn.Server):
asyncio_setup()
else:
asyncio.set_event_loop(uvloop.new_event_loop())
loop = asyncio.get_event_loop()
try:
loop = asyncio.get_event_loop()
except RuntimeError:
# When running in a thread, we'll not have an eventloop yet.
loop = asyncio.new_event_loop()
loop.run_until_complete(self.serve(sockets=sockets))
@contextlib.contextmanager

17
freqtrade/rpc/api_server/web_ui.py
View File

@@ -29,6 +29,16 @@ async def ui_version():
}
def is_relative_to(path, base) -> bool:
# Helper function simulating behaviour of is_relative_to, which was only added in python 3.9
try:
path.relative_to(base)
return True
except ValueError:
pass
return False
@router_ui.get('/{rest_of_path:path}', include_in_schema=False)
async def index_html(rest_of_path: str):
"""
@@ -37,8 +47,11 @@ async def index_html(rest_of_path: str):
if rest_of_path.startswith('api') or rest_of_path.startswith('.'):
raise HTTPException(status_code=404, detail="Not Found")
uibase = Path(__file__).parent / 'ui/installed/'
if (uibase / rest_of_path).is_file():
return FileResponse(str(uibase / rest_of_path))
filename = uibase / rest_of_path
# It's security relevant to check "relative_to".
# Without this, Directory-traversal is possible.
if filename.is_file() and is_relative_to(filename, uibase):
return FileResponse(str(filename))
index_file = uibase / 'index.html'
if not index_file.is_file():

40
freqtrade/rpc/fiat_convert.py
View File

@@ -5,7 +5,7 @@ e.g BTC to USD
import datetime
import logging
from typing import Dict
from typing import Dict, List
from cachetools.ttl import TTLCache
from pycoingecko import CoinGeckoAPI
@@ -25,8 +25,7 @@ class CryptoToFiatConverter:
"""
__instance = None
_coingekko: CoinGeckoAPI = None
_cryptomap: Dict = {}
_coinlistings: List[Dict] = []
_backoff: float = 0.0
def __new__(cls):
@@ -49,9 +48,8 @@ class CryptoToFiatConverter:
def _load_cryptomap(self) -> None:
try:
coinlistings = self._coingekko.get_coins_list()
# Create mapping table from symbol to coingekko_id
self._cryptomap = {x['symbol']: x['id'] for x in coinlistings}
# Use list-comprehension to ensure we get a list.
self._coinlistings = [x for x in self._coingekko.get_coins_list()]
except RequestException as request_exception:
if "429" in str(request_exception):
logger.warning(
@@ -69,6 +67,24 @@ class CryptoToFiatConverter:
logger.error(
f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
def _get_gekko_id(self, crypto_symbol):
if not self._coinlistings:
if self._backoff <= datetime.datetime.now().timestamp():
self._load_cryptomap()
# Still not loaded.
if not self._coinlistings:
return None
else:
return None
found = [x for x in self._coinlistings if x['symbol'] == crypto_symbol]
if len(found) == 1:
return found[0]['id']
if len(found) > 0:
# Wrong!
logger.warning(f"Found multiple mappings in goingekko for {crypto_symbol}.")
return None
def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float:
"""
Convert an amount of crypto-currency to fiat
@@ -143,22 +159,14 @@ class CryptoToFiatConverter:
if crypto_symbol == fiat_symbol:
return 1.0
if self._cryptomap == {}:
if self._backoff <= datetime.datetime.now().timestamp():
self._load_cryptomap()
# return 0.0 if we still don't have data to check, no reason to proceed
if self._cryptomap == {}:
return 0.0
else:
return 0.0
_gekko_id = self._get_gekko_id(crypto_symbol)
if crypto_symbol not in self._cryptomap:
if not _gekko_id:
# return 0 for unsupported stake currencies (fiat-convert should not break the bot)
logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)
return 0.0
try:
_gekko_id = self._cryptomap[crypto_symbol]
return float(
self._coingekko.get_price(
ids=_gekko_id,

6
freqtrade/rpc/rpc.py
View File

@@ -557,7 +557,7 @@ class RPC:
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
self._freqtrade.execute_sell(trade, current_rate, sell_reason)
self._freqtrade.execute_trade_exit(trade, current_rate, sell_reason)
# ---- EOF def _exec_forcesell ----
if self._freqtrade.state != State.RUNNING:
@@ -613,7 +613,7 @@ class RPC:
stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
# execute buy
if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True):
if self._freqtrade.execute_entry(pair, stakeamount, price, forcebuy=True):
Trade.commit()
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
return trade
@@ -776,7 +776,7 @@ class RPC:
if has_content:
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
# Move open to seperate column when signal for easy plotting
# Move open to separate column when signal for easy plotting
if 'buy' in dataframe.columns:
buy_mask = (dataframe['buy'] == 1)
buy_signals = int(buy_mask.sum())

43
freqtrade/strategy/interface.py
View File

@@ -120,6 +120,8 @@ class IStrategy(ABC, HyperStrategyMixin):
# and wallets - access to the current balance.
dp: Optional[DataProvider] = None
wallets: Optional[Wallets] = None
# Filled from configuration
stake_currency: str
# container variable for strategy source code
__source__: str = ''
@@ -280,6 +282,43 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
return self.stoploss
def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float,
**kwargs) -> float:
"""
Custom entry price logic, returning the new entry price.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns None, orderbook is used to set entry price
:param pair: Pair that's currently analyzed
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in ask_strategy.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New entry price value if provided
"""
return proposed_rate
def custom_exit_price(self, pair: str, trade: Trade,
current_time: datetime, proposed_rate: float,
current_profit: float, **kwargs) -> float:
"""
Custom exit price logic, returning the new exit price.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns None, orderbook is used to set exit price
:param pair: Pair that's currently analyzed
:param trade: trade object.
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in ask_strategy.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New exit price value if provided
"""
return proposed_rate
def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float,
current_profit: float, **kwargs) -> Optional[Union[str, bool]]:
"""
@@ -577,7 +616,7 @@ class IStrategy(ABC, HyperStrategyMixin):
current_rate = rate
current_profit = trade.calc_profit_ratio(current_rate)
trade.adjust_min_max_rates(high or current_rate)
trade.adjust_min_max_rates(high or current_rate, low or current_rate)
stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade,
current_time=date, current_profit=current_profit,
@@ -741,7 +780,7 @@ class IStrategy(ABC, HyperStrategyMixin):
else:
return current_profit > roi
def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
def advise_all_indicators(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
"""
Populates indicators for given candle (OHLCV) data (for multiple pairs)
Does not run advise_buy or advise_sell!

2
freqtrade/templates/base_config.json.j2
View File

@@ -25,7 +25,7 @@
"ask_strategy": {
"price_side": "ask",
"use_order_book": true,
"order_book_top": 1,
"order_book_top": 1
},
{{ exchange | indent(4) }},
"pairlists": [

2
freqtrade/templates/subtemplates/exchange_binance.j2
View File

@@ -36,6 +36,6 @@
"BNB/TUSD",
"BNB/USDC",
"BNB/USDS",
"BNB/USDT",
"BNB/USDT"
]
}