Merge branch 'develop' into pr/rextea/4606
This commit is contained in:
@@ -22,7 +22,7 @@ if __version__ == 'develop':
|
||||
# subprocess.check_output(
|
||||
# ['git', 'log', '--format="%h"', '-n 1'],
|
||||
# stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
|
||||
except Exception:
|
||||
except Exception: # pragma: no cover
|
||||
# git not available, ignore
|
||||
try:
|
||||
# Try Fallback to freqtrade_commit file (created by CI while building docker image)
|
||||
|
@@ -8,15 +8,16 @@ Note: Be careful with file-scoped imports in these subfiles.
|
||||
"""
|
||||
from freqtrade.commands.arguments import Arguments
|
||||
from freqtrade.commands.build_config_commands import start_new_config
|
||||
from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
|
||||
start_list_data)
|
||||
from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
|
||||
start_download_data, start_list_data)
|
||||
from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
|
||||
start_new_hyperopt, start_new_strategy)
|
||||
start_new_strategy)
|
||||
from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
|
||||
from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts,
|
||||
start_list_markets, start_list_strategies,
|
||||
start_list_timeframes, start_show_trades)
|
||||
from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets,
|
||||
start_list_strategies, start_list_timeframes,
|
||||
start_show_trades)
|
||||
from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt
|
||||
from freqtrade.commands.pairlist_commands import start_test_pairlist
|
||||
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
|
||||
from freqtrade.commands.trade_commands import start_trading
|
||||
from freqtrade.commands.webserver_commands import start_webserver
|
||||
|
@@ -16,11 +16,13 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
|
||||
|
||||
ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
|
||||
|
||||
ARGS_WEBSERVER: List[str] = []
|
||||
|
||||
ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
|
||||
"max_open_trades", "stake_amount", "fee"]
|
||||
"max_open_trades", "stake_amount", "fee", "pairs"]
|
||||
|
||||
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
|
||||
"enable_protections", "dry_run_wallet",
|
||||
"enable_protections", "dry_run_wallet", "timeframe_detail",
|
||||
"strategy_list", "export", "exportfilename", "show_days"]
|
||||
|
||||
ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
|
||||
@@ -29,7 +31,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
|
||||
"epochs", "spaces", "print_all",
|
||||
"print_colorized", "print_json", "hyperopt_jobs",
|
||||
"hyperopt_random_state", "hyperopt_min_trades",
|
||||
"hyperopt_loss"]
|
||||
"hyperopt_loss", "disableparamexport",
|
||||
"hyperopt_ignore_missing_space"]
|
||||
|
||||
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
|
||||
|
||||
@@ -53,24 +56,25 @@ ARGS_BUILD_CONFIG = ["config"]
|
||||
|
||||
ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
|
||||
|
||||
ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
|
||||
|
||||
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
|
||||
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
|
||||
|
||||
ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
|
||||
|
||||
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
|
||||
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
|
||||
"timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
|
||||
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
|
||||
"download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
|
||||
"dataformat_trades"]
|
||||
|
||||
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
|
||||
"db_url", "trade_source", "export", "exportfilename",
|
||||
"timerange", "timeframe", "no_trades"]
|
||||
|
||||
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
|
||||
"trade_source", "timeframe"]
|
||||
"trade_source", "timeframe", "plot_auto_open"]
|
||||
|
||||
ARGS_INSTALL_UI = ["erase_ui_only"]
|
||||
ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version']
|
||||
|
||||
ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
|
||||
|
||||
@@ -84,14 +88,15 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
|
||||
"hyperoptexportfilename", "export_csv"]
|
||||
|
||||
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
|
||||
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header"]
|
||||
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
|
||||
"disableparamexport"]
|
||||
|
||||
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
|
||||
"list-markets", "list-pairs", "list-strategies", "list-data",
|
||||
"list-hyperopts", "hyperopt-list", "hyperopt-show",
|
||||
"plot-dataframe", "plot-profit", "show-trades"]
|
||||
"hyperopt-list", "hyperopt-show",
|
||||
"plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
|
||||
|
||||
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]
|
||||
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
|
||||
|
||||
|
||||
class Arguments:
|
||||
@@ -167,14 +172,14 @@ class Arguments:
|
||||
self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
|
||||
self._build_args(optionlist=['version'], parser=self.parser)
|
||||
|
||||
from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir,
|
||||
start_download_data, start_edge, start_hyperopt,
|
||||
start_hyperopt_list, start_hyperopt_show, start_install_ui,
|
||||
start_list_data, start_list_exchanges, start_list_hyperopts,
|
||||
from freqtrade.commands import (start_backtesting, start_convert_data, start_convert_trades,
|
||||
start_create_userdir, start_download_data, start_edge,
|
||||
start_hyperopt, start_hyperopt_list, start_hyperopt_show,
|
||||
start_install_ui, start_list_data, start_list_exchanges,
|
||||
start_list_markets, start_list_strategies,
|
||||
start_list_timeframes, start_new_config, start_new_hyperopt,
|
||||
start_new_strategy, start_plot_dataframe, start_plot_profit,
|
||||
start_show_trades, start_test_pairlist, start_trading)
|
||||
start_list_timeframes, start_new_config, start_new_strategy,
|
||||
start_plot_dataframe, start_plot_profit, start_show_trades,
|
||||
start_test_pairlist, start_trading, start_webserver)
|
||||
|
||||
subparsers = self.parser.add_subparsers(dest='command',
|
||||
# Use custom message when no subhandler is added
|
||||
@@ -201,12 +206,6 @@ class Arguments:
|
||||
build_config_cmd.set_defaults(func=start_new_config)
|
||||
self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
|
||||
|
||||
# add new-hyperopt subcommand
|
||||
build_hyperopt_cmd = subparsers.add_parser('new-hyperopt',
|
||||
help="Create new hyperopt")
|
||||
build_hyperopt_cmd.set_defaults(func=start_new_hyperopt)
|
||||
self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd)
|
||||
|
||||
# add new-strategy subcommand
|
||||
build_strategy_cmd = subparsers.add_parser('new-strategy',
|
||||
help="Create new strategy")
|
||||
@@ -240,6 +239,15 @@ class Arguments:
|
||||
convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
|
||||
self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
|
||||
|
||||
# Add trades-to-ohlcv subcommand
|
||||
convert_trade_data_cmd = subparsers.add_parser(
|
||||
'trades-to-ohlcv',
|
||||
help='Convert trade data to OHLCV data.',
|
||||
parents=[_common_parser],
|
||||
)
|
||||
convert_trade_data_cmd.set_defaults(func=start_convert_trades)
|
||||
self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd)
|
||||
|
||||
# Add list-data subcommand
|
||||
list_data_cmd = subparsers.add_parser(
|
||||
'list-data',
|
||||
@@ -295,15 +303,6 @@ class Arguments:
|
||||
list_exchanges_cmd.set_defaults(func=start_list_exchanges)
|
||||
self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd)
|
||||
|
||||
# Add list-hyperopts subcommand
|
||||
list_hyperopts_cmd = subparsers.add_parser(
|
||||
'list-hyperopts',
|
||||
help='Print available hyperopt classes.',
|
||||
parents=[_common_parser],
|
||||
)
|
||||
list_hyperopts_cmd.set_defaults(func=start_list_hyperopts)
|
||||
self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd)
|
||||
|
||||
# Add list-markets subcommand
|
||||
list_markets_cmd = subparsers.add_parser(
|
||||
'list-markets',
|
||||
@@ -382,3 +381,9 @@ class Arguments:
|
||||
)
|
||||
plot_profit_cmd.set_defaults(func=start_plot_profit)
|
||||
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
|
||||
|
||||
# Add webserver subcommand
|
||||
webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
|
||||
parents=[_common_parser])
|
||||
webserver_cmd.set_defaults(func=start_webserver)
|
||||
self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
|
||||
|
@@ -1,9 +1,11 @@
|
||||
import logging
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from questionary import Separator, prompt
|
||||
|
||||
from freqtrade.configuration.directory_operations import chown_user_directory
|
||||
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
|
||||
@@ -59,21 +61,27 @@ def ask_user_config() -> Dict[str, Any]:
|
||||
"type": "text",
|
||||
"name": "stake_currency",
|
||||
"message": "Please insert your stake currency:",
|
||||
"default": 'BTC',
|
||||
"default": 'USDT',
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"name": "stake_amount",
|
||||
"message": "Please insert your stake amount:",
|
||||
"default": "0.01",
|
||||
"message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
|
||||
"default": "100",
|
||||
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
|
||||
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
|
||||
if val == UNLIMITED_STAKE_AMOUNT
|
||||
else val
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"name": "max_open_trades",
|
||||
"message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):",
|
||||
"default": "3",
|
||||
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val)
|
||||
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val),
|
||||
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
|
||||
if val == UNLIMITED_STAKE_AMOUNT
|
||||
else val
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
@@ -97,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]:
|
||||
"bittrex",
|
||||
"kraken",
|
||||
"ftx",
|
||||
"kucoin",
|
||||
"gateio",
|
||||
Separator(),
|
||||
"other",
|
||||
],
|
||||
@@ -120,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]:
|
||||
"message": "Insert Exchange Secret",
|
||||
"when": lambda x: not x['dry_run']
|
||||
},
|
||||
{
|
||||
"type": "password",
|
||||
"name": "exchange_key_password",
|
||||
"message": "Insert Exchange API Key password",
|
||||
"when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin'
|
||||
},
|
||||
{
|
||||
"type": "confirm",
|
||||
"name": "telegram",
|
||||
@@ -138,6 +154,33 @@ def ask_user_config() -> Dict[str, Any]:
|
||||
"message": "Insert Telegram chat id",
|
||||
"when": lambda x: x['telegram']
|
||||
},
|
||||
{
|
||||
"type": "confirm",
|
||||
"name": "api_server",
|
||||
"message": "Do you want to enable the Rest API (includes FreqUI)?",
|
||||
"default": False,
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"name": "api_server_listen_addr",
|
||||
"message": ("Insert Api server Listen Address (0.0.0.0 for docker, "
|
||||
"otherwise best left untouched)"),
|
||||
"default": "127.0.0.1",
|
||||
"when": lambda x: x['api_server']
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"name": "api_server_username",
|
||||
"message": "Insert api-server username",
|
||||
"default": "freqtrader",
|
||||
"when": lambda x: x['api_server']
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"name": "api_server_password",
|
||||
"message": "Insert api-server password",
|
||||
"when": lambda x: x['api_server']
|
||||
},
|
||||
]
|
||||
answers = prompt(questions)
|
||||
|
||||
@@ -145,6 +188,9 @@ def ask_user_config() -> Dict[str, Any]:
|
||||
# Interrupted questionary sessions return an empty dict.
|
||||
raise OperationalException("User interrupted interactive questions.")
|
||||
|
||||
# Force JWT token to be a random string
|
||||
answers['api_server_jwt_key'] = secrets.token_hex()
|
||||
|
||||
return answers
|
||||
|
||||
|
||||
@@ -152,7 +198,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Applies selections to the template and writes the result to config_path
|
||||
:param config_path: Path object for new config file. Should not exist yet
|
||||
:param selecions: Dict containing selections taken by the user.
|
||||
:param selections: Dict containing selections taken by the user.
|
||||
"""
|
||||
from jinja2.exceptions import TemplateNotFound
|
||||
try:
|
||||
@@ -162,7 +208,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
|
||||
selections['exchange'] = render_template(
|
||||
templatefile=f"subtemplates/exchange_{exchange_template}.j2",
|
||||
arguments=selections
|
||||
)
|
||||
)
|
||||
except TemplateNotFound:
|
||||
selections['exchange'] = render_template(
|
||||
templatefile="subtemplates/exchange_generic.j2",
|
||||
@@ -182,10 +228,11 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
|
||||
def start_new_config(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Create a new strategy from a template
|
||||
Asking the user questions to fill out the templateaccordingly.
|
||||
Asking the user questions to fill out the template accordingly.
|
||||
"""
|
||||
|
||||
config_path = Path(args['config'][0])
|
||||
chown_user_directory(config_path.parent)
|
||||
if config_path.exists():
|
||||
overwrite = ask_user_overwrite(config_path)
|
||||
if overwrite:
|
||||
|
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
Definition of cli arguments used in arguments.py
|
||||
"""
|
||||
from argparse import ArgumentTypeError
|
||||
from argparse import SUPPRESS, ArgumentTypeError
|
||||
|
||||
from freqtrade import __version__, constants
|
||||
from freqtrade.constants import HYPEROPT_LOSS_BUILTIN
|
||||
@@ -118,7 +118,7 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
# Optimize common
|
||||
"timeframe": Arg(
|
||||
'-i', '--timeframe', '--ticker-interval',
|
||||
help='Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).',
|
||||
help='Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).',
|
||||
),
|
||||
"timerange": Arg(
|
||||
'--timerange',
|
||||
@@ -135,6 +135,10 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
help='Override the value of the `stake_amount` configuration setting.',
|
||||
),
|
||||
# Backtesting
|
||||
"timeframe_detail": Arg(
|
||||
'--timeframe-detail',
|
||||
help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).',
|
||||
),
|
||||
"position_stacking": Arg(
|
||||
'--eps', '--enable-position-stacking',
|
||||
help='Allow buying the same pair multiple times (position stacking).',
|
||||
@@ -162,13 +166,14 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
'Please note that ticker-interval needs to be set either in config '
|
||||
'or via command line. When using this together with `--export trades`, '
|
||||
'the strategy-name is injected into the filename '
|
||||
'(so `backtest-data.json` becomes `backtest-data-DefaultStrategy.json`',
|
||||
'(so `backtest-data.json` becomes `backtest-data-SampleStrategy.json`',
|
||||
nargs='+',
|
||||
),
|
||||
"export": Arg(
|
||||
'--export',
|
||||
help='Export backtest results, argument are: trades. '
|
||||
'Example: `--export=trades`',
|
||||
help='Export backtest results (default: trades).',
|
||||
choices=constants.EXPORT_OPTIONS,
|
||||
|
||||
),
|
||||
"exportfilename": Arg(
|
||||
'--export-filename',
|
||||
@@ -177,6 +182,11 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
'Example: `--export-filename=user_data/backtest_results/backtest_today.json`',
|
||||
metavar='PATH',
|
||||
),
|
||||
"disableparamexport": Arg(
|
||||
'--disable-param-export',
|
||||
help="Disable automatic hyperopt parameter export.",
|
||||
action='store_true',
|
||||
),
|
||||
"fee": Arg(
|
||||
'--fee',
|
||||
help='Specify fee ratio. Will be applied twice (on trade entry and exit).',
|
||||
@@ -199,12 +209,13 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
# Hyperopt
|
||||
"hyperopt": Arg(
|
||||
'--hyperopt',
|
||||
help='Specify hyperopt class name which will be used by the bot.',
|
||||
help=SUPPRESS,
|
||||
metavar='NAME',
|
||||
required=False,
|
||||
),
|
||||
"hyperopt_path": Arg(
|
||||
'--hyperopt-path',
|
||||
help='Specify additional lookup path for Hyperopt and Hyperopt Loss functions.',
|
||||
help='Specify additional lookup path for Hyperopt Loss functions.',
|
||||
metavar='PATH',
|
||||
),
|
||||
"epochs": Arg(
|
||||
@@ -217,7 +228,7 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
"spaces": Arg(
|
||||
'--spaces',
|
||||
help='Specify which parameters to hyperopt. Space-separated list.',
|
||||
choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'],
|
||||
choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'default'],
|
||||
nargs='+',
|
||||
default='default',
|
||||
),
|
||||
@@ -272,7 +283,7 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
default=1,
|
||||
),
|
||||
"hyperopt_loss": Arg(
|
||||
'--hyperopt-loss',
|
||||
'--hyperopt-loss', '--hyperoptloss',
|
||||
help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). '
|
||||
'Different functions can generate completely different results, '
|
||||
'since the target for optimization is different. Built-in Hyperopt-loss-functions are: '
|
||||
@@ -335,7 +346,7 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
# Script options
|
||||
"pairs": Arg(
|
||||
'-p', '--pairs',
|
||||
help='Show profits for only these pairs. Pairs are space-separated.',
|
||||
help='Limit command to these pairs. Pairs are space-separated.',
|
||||
nargs='+',
|
||||
),
|
||||
# Download data
|
||||
@@ -350,6 +361,12 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
type=check_int_positive,
|
||||
metavar='INT',
|
||||
),
|
||||
"new_pairs_days": Arg(
|
||||
'--new-pairs-days',
|
||||
help='Download data of new pairs for given number of days. Default: `%(default)s`.',
|
||||
type=check_int_positive,
|
||||
metavar='INT',
|
||||
),
|
||||
"download_trades": Arg(
|
||||
'--dl-trades',
|
||||
help='Download trades instead of OHLCV data. The bot will resample trades to the '
|
||||
@@ -370,12 +387,12 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
),
|
||||
"dataformat_ohlcv": Arg(
|
||||
'--data-format-ohlcv',
|
||||
help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
|
||||
help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
|
||||
choices=constants.AVAILABLE_DATAHANDLERS,
|
||||
),
|
||||
"dataformat_trades": Arg(
|
||||
'--data-format-trades',
|
||||
help='Storage format for downloaded trades data. (default: `%(default)s`).',
|
||||
help='Storage format for downloaded trades data. (default: `jsongz`).',
|
||||
choices=constants.AVAILABLE_DATAHANDLERS,
|
||||
),
|
||||
"exchange": Arg(
|
||||
@@ -403,6 +420,12 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
action='store_true',
|
||||
default=False,
|
||||
),
|
||||
"ui_version": Arg(
|
||||
'--ui-version',
|
||||
help=('Specify a specific version of FreqUI to install. '
|
||||
'Not specifying this installs the latest version.'),
|
||||
type=str,
|
||||
),
|
||||
# Templating options
|
||||
"template": Arg(
|
||||
'--template',
|
||||
@@ -432,6 +455,11 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
metavar='INT',
|
||||
default=750,
|
||||
),
|
||||
"plot_auto_open": Arg(
|
||||
'--auto-open',
|
||||
help='Automatically open generated plot.',
|
||||
action='store_true',
|
||||
),
|
||||
"no_trades": Arg(
|
||||
'--no-trades',
|
||||
help='Skip using trades from backtesting file and DB.',
|
||||
@@ -536,4 +564,10 @@ AVAILABLE_CLI_OPTIONS = {
|
||||
help='Do not print epoch details header.',
|
||||
action='store_true',
|
||||
),
|
||||
"hyperopt_ignore_missing_space": Arg(
|
||||
"--ignore-missing-spaces", "--ignore-unparameterized-spaces",
|
||||
help=("Suppress errors for any requested Hyperopt spaces "
|
||||
"that do not contain any parameters."),
|
||||
action="store_true",
|
||||
),
|
||||
}
|
||||
|
@@ -8,11 +8,11 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration
|
||||
from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
|
||||
from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
|
||||
refresh_backtest_trades_data)
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import timeframe_to_minutes
|
||||
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
|
||||
from freqtrade.resolvers import ExchangeResolver
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -48,7 +48,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
# Init exchange
|
||||
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
|
||||
# Manual validations of relevant settings
|
||||
exchange.validate_pairs(config['pairs'])
|
||||
if not config['exchange'].get('skip_pair_validation', False):
|
||||
exchange.validate_pairs(config['pairs'])
|
||||
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
|
||||
|
||||
logger.info(f"About to download pairs: {expanded_pairs}, "
|
||||
@@ -62,8 +63,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
if config.get('download_trades'):
|
||||
pairs_not_available = refresh_backtest_trades_data(
|
||||
exchange, pairs=expanded_pairs, datadir=config['datadir'],
|
||||
timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format=config['dataformat_trades'])
|
||||
timerange=timerange, new_pairs_days=config['new_pairs_days'],
|
||||
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
|
||||
|
||||
# Convert downloaded trade data to different timeframes
|
||||
convert_trades_to_ohlcv(
|
||||
@@ -75,8 +76,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
else:
|
||||
pairs_not_available = refresh_backtest_ohlcv_data(
|
||||
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format=config['dataformat_ohlcv'])
|
||||
datadir=config['datadir'], timerange=timerange,
|
||||
new_pairs_days=config['new_pairs_days'],
|
||||
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
|
||||
|
||||
except KeyboardInterrupt:
|
||||
sys.exit("SIGINT received, aborting ...")
|
||||
@@ -87,6 +89,41 @@ def start_download_data(args: Dict[str, Any]) -> None:
|
||||
f"on exchange {exchange.name}.")
|
||||
|
||||
|
||||
def start_convert_trades(args: Dict[str, Any]) -> None:
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
|
||||
|
||||
timerange = TimeRange()
|
||||
|
||||
# Remove stake-currency to skip checks which are not relevant for datadownload
|
||||
config['stake_currency'] = ''
|
||||
|
||||
if 'pairs' not in config:
|
||||
raise OperationalException(
|
||||
"Downloading data requires a list of pairs. "
|
||||
"Please check the documentation on how to configure this.")
|
||||
|
||||
# Init exchange
|
||||
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
|
||||
# Manual validations of relevant settings
|
||||
if not config['exchange'].get('skip_pair_validation', False):
|
||||
exchange.validate_pairs(config['pairs'])
|
||||
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
|
||||
|
||||
logger.info(f"About to Convert pairs: {expanded_pairs}, "
|
||||
f"intervals: {config['timeframes']} to {config['datadir']}")
|
||||
|
||||
for timeframe in config['timeframes']:
|
||||
exchange.validate_timeframes(timeframe)
|
||||
# Convert downloaded trade data to different timeframes
|
||||
convert_trades_to_ohlcv(
|
||||
pairs=expanded_pairs, timeframes=config['timeframes'],
|
||||
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
|
||||
data_format_ohlcv=config['dataformat_ohlcv'],
|
||||
data_format_trades=config['dataformat_trades'],
|
||||
)
|
||||
|
||||
|
||||
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
|
||||
"""
|
||||
Convert data from one format to another
|
||||
|
@@ -7,10 +7,10 @@ import requests
|
||||
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir
|
||||
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
|
||||
from freqtrade.constants import USERPATH_STRATEGIES
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import render_template, render_template_with_fallback
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,15 +38,15 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
|
||||
indicators = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/indicators_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/indicators_{fallback}.j2",
|
||||
)
|
||||
)
|
||||
buy_trend = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2",
|
||||
)
|
||||
)
|
||||
sell_trend = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2",
|
||||
)
|
||||
)
|
||||
plot_config = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/plot_config_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2",
|
||||
@@ -74,8 +74,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
|
||||
|
||||
if "strategy" in args and args["strategy"]:
|
||||
if args["strategy"] == "DefaultStrategy":
|
||||
raise OperationalException("DefaultStrategy is not allowed as name.")
|
||||
|
||||
new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
|
||||
|
||||
@@ -89,58 +87,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
|
||||
raise OperationalException("`new-strategy` requires --strategy to be set.")
|
||||
|
||||
|
||||
def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None:
|
||||
"""
|
||||
Deploys a new hyperopt template to hyperopt_path
|
||||
"""
|
||||
fallback = 'full'
|
||||
buy_guards = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2",
|
||||
)
|
||||
sell_guards = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2",
|
||||
)
|
||||
buy_space = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2",
|
||||
)
|
||||
sell_space = render_template_with_fallback(
|
||||
templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",
|
||||
templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2",
|
||||
)
|
||||
|
||||
strategy_text = render_template(templatefile='base_hyperopt.py.j2',
|
||||
arguments={"hyperopt": hyperopt_name,
|
||||
"buy_guards": buy_guards,
|
||||
"sell_guards": sell_guards,
|
||||
"buy_space": buy_space,
|
||||
"sell_space": sell_space,
|
||||
})
|
||||
|
||||
logger.info(f"Writing hyperopt to `{hyperopt_path}`.")
|
||||
hyperopt_path.write_text(strategy_text)
|
||||
|
||||
|
||||
def start_new_hyperopt(args: Dict[str, Any]) -> None:
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
|
||||
|
||||
if 'hyperopt' in args and args['hyperopt']:
|
||||
if args['hyperopt'] == 'DefaultHyperopt':
|
||||
raise OperationalException("DefaultHyperopt is not allowed as name.")
|
||||
|
||||
new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
|
||||
|
||||
if new_path.exists():
|
||||
raise OperationalException(f"`{new_path}` already exists. "
|
||||
"Please choose another Hyperopt Name.")
|
||||
deploy_new_hyperopt(args['hyperopt'], new_path, args['template'])
|
||||
else:
|
||||
raise OperationalException("`new-hyperopt` requires --hyperopt to be set.")
|
||||
|
||||
|
||||
def clean_ui_subdir(directory: Path):
|
||||
if directory.is_dir():
|
||||
logger.info("Removing UI directory content.")
|
||||
@@ -182,7 +128,7 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str):
|
||||
f.write(version)
|
||||
|
||||
|
||||
def get_ui_download_url() -> Tuple[str, str]:
|
||||
def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]:
|
||||
base_url = 'https://api.github.com/repos/freqtrade/frequi/'
|
||||
# Get base UI Repo path
|
||||
|
||||
@@ -190,8 +136,16 @@ def get_ui_download_url() -> Tuple[str, str]:
|
||||
resp.raise_for_status()
|
||||
r = resp.json()
|
||||
|
||||
latest_version = r[0]['name']
|
||||
assets = r[0].get('assets', [])
|
||||
if version:
|
||||
tmp = [x for x in r if x['name'] == version]
|
||||
if tmp:
|
||||
latest_version = tmp[0]['name']
|
||||
assets = tmp[0].get('assets', [])
|
||||
else:
|
||||
raise ValueError("UI-Version not found.")
|
||||
else:
|
||||
latest_version = r[0]['name']
|
||||
assets = r[0].get('assets', [])
|
||||
dl_url = ''
|
||||
if assets and len(assets) > 0:
|
||||
dl_url = assets[0]['browser_download_url']
|
||||
@@ -210,7 +164,7 @@ def start_install_ui(args: Dict[str, Any]) -> None:
|
||||
|
||||
dest_folder = Path(__file__).parents[1] / 'rpc/api_server/ui/installed/'
|
||||
# First make sure the assets are removed.
|
||||
dl_url, latest_version = get_ui_download_url()
|
||||
dl_url, latest_version = get_ui_download_url(args.get('ui_version'))
|
||||
|
||||
curr_version = read_ui_version(dest_folder)
|
||||
if curr_version == latest_version and not args.get('erase_ui_only'):
|
||||
|
@@ -1,13 +1,14 @@
|
||||
import logging
|
||||
from operator import itemgetter
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict
|
||||
|
||||
from colorama import init as colorama_init
|
||||
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.data.btanalysis import get_latest_hyperopt_file
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.state import RunMode
|
||||
from freqtrade.optimize.optimize_reports import show_backtest_result
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -17,7 +18,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
List hyperopt epochs previously evaluated
|
||||
"""
|
||||
from freqtrade.optimize.hyperopt import Hyperopt
|
||||
from freqtrade.optimize.hyperopt_tools import HyperoptTools
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
|
||||
|
||||
@@ -27,49 +28,32 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
|
||||
no_details = config.get('hyperopt_list_no_details', False)
|
||||
no_header = False
|
||||
|
||||
filteroptions = {
|
||||
'only_best': config.get('hyperopt_list_best', False),
|
||||
'only_profitable': config.get('hyperopt_list_profitable', False),
|
||||
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
|
||||
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
|
||||
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
|
||||
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
|
||||
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
|
||||
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
|
||||
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
|
||||
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
|
||||
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
|
||||
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
|
||||
}
|
||||
|
||||
results_file = get_latest_hyperopt_file(
|
||||
config['user_data_dir'] / 'hyperopt_results',
|
||||
config.get('hyperoptexportfilename'))
|
||||
|
||||
# Previous evaluations
|
||||
epochs = Hyperopt.load_previous_results(results_file)
|
||||
total_epochs = len(epochs)
|
||||
|
||||
epochs = hyperopt_filter_epochs(epochs, filteroptions)
|
||||
epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
|
||||
|
||||
if print_colorized:
|
||||
colorama_init(autoreset=True)
|
||||
|
||||
if not export_csv:
|
||||
try:
|
||||
print(Hyperopt.get_result_table(config, epochs, total_epochs,
|
||||
not filteroptions['only_best'], print_colorized, 0))
|
||||
print(HyperoptTools.get_result_table(config, epochs, total_epochs,
|
||||
not config.get('hyperopt_list_best', False),
|
||||
print_colorized, 0))
|
||||
except KeyboardInterrupt:
|
||||
print('User interrupted..')
|
||||
|
||||
if epochs and not no_details:
|
||||
sorted_epochs = sorted(epochs, key=itemgetter('loss'))
|
||||
results = sorted_epochs[0]
|
||||
Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header)
|
||||
HyperoptTools.show_epoch_details(results, total_epochs, print_json, no_header)
|
||||
|
||||
if epochs and export_csv:
|
||||
Hyperopt.export_csv_file(
|
||||
config, epochs, total_epochs, not filteroptions['only_best'], export_csv
|
||||
HyperoptTools.export_csv_file(
|
||||
config, epochs, export_csv
|
||||
)
|
||||
|
||||
|
||||
@@ -77,7 +61,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Show details of a hyperopt epoch previously evaluated
|
||||
"""
|
||||
from freqtrade.optimize.hyperopt import Hyperopt
|
||||
from freqtrade.optimize.hyperopt_tools import HyperoptTools
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
|
||||
|
||||
@@ -89,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
|
||||
|
||||
n = config.get('hyperopt_show_index', -1)
|
||||
|
||||
filteroptions = {
|
||||
'only_best': config.get('hyperopt_list_best', False),
|
||||
'only_profitable': config.get('hyperopt_list_profitable', False),
|
||||
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
|
||||
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
|
||||
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
|
||||
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
|
||||
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
|
||||
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
|
||||
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
|
||||
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
|
||||
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
|
||||
'filter_max_objective': config.get('hyperopt_list_max_objective', None)
|
||||
}
|
||||
|
||||
# Previous evaluations
|
||||
epochs = Hyperopt.load_previous_results(results_file)
|
||||
total_epochs = len(epochs)
|
||||
epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
|
||||
|
||||
epochs = hyperopt_filter_epochs(epochs, filteroptions)
|
||||
filtered_epochs = len(epochs)
|
||||
|
||||
if n > filtered_epochs:
|
||||
@@ -124,105 +91,14 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
|
||||
|
||||
if epochs:
|
||||
val = epochs[n]
|
||||
Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header,
|
||||
header_str="Epoch details")
|
||||
|
||||
metrics = val['results_metrics']
|
||||
if 'strategy_name' in metrics:
|
||||
strategy_name = metrics['strategy_name']
|
||||
show_backtest_result(strategy_name, metrics,
|
||||
metrics['stake_currency'])
|
||||
|
||||
def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
|
||||
"""
|
||||
Filter our items from the list of hyperopt results
|
||||
"""
|
||||
if filteroptions['only_best']:
|
||||
epochs = [x for x in epochs if x['is_best']]
|
||||
if filteroptions['only_profitable']:
|
||||
epochs = [x for x in epochs if x['results_metrics']['profit'] > 0]
|
||||
HyperoptTools.try_export_params(config, strategy_name, val)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
|
||||
|
||||
logger.info(f"{len(epochs)} " +
|
||||
("best " if filteroptions['only_best'] else "") +
|
||||
("profitable " if filteroptions['only_profitable'] else "") +
|
||||
"epochs found.")
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_trades'] > 0:
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades']
|
||||
]
|
||||
if filteroptions['filter_max_trades'] > 0:
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
|
||||
]
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_avg_time'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']
|
||||
]
|
||||
if filteroptions['filter_max_avg_time'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
|
||||
]
|
||||
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_avg_profit'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit']
|
||||
]
|
||||
if filteroptions['filter_max_avg_profit'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit']
|
||||
]
|
||||
if filteroptions['filter_min_total_profit'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']
|
||||
]
|
||||
if filteroptions['filter_max_total_profit'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
|
||||
]
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_objective'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
|
||||
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
|
||||
if filteroptions['filter_max_objective'] is not None:
|
||||
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
|
||||
|
||||
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
|
||||
|
||||
return epochs
|
||||
HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
|
||||
header_str="Epoch details")
|
||||
|
@@ -1,7 +1,6 @@
|
||||
import csv
|
||||
import logging
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
@@ -11,12 +10,12 @@ from colorama import init as colorama_init
|
||||
from tabulate import tabulate
|
||||
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
|
||||
from freqtrade.constants import USERPATH_STRATEGIES
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import available_exchanges, ccxt_exchanges, market_is_active
|
||||
from freqtrade.misc import plural
|
||||
from freqtrade.exchange import market_is_active, validate_exchanges
|
||||
from freqtrade.misc import parse_db_uri_for_logging, plural
|
||||
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -28,14 +27,18 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
|
||||
:param args: Cli args from Arguments()
|
||||
:return: None
|
||||
"""
|
||||
exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
|
||||
exchanges = validate_exchanges(args['list_exchanges_all'])
|
||||
|
||||
if args['print_one_column']:
|
||||
print('\n'.join(exchanges))
|
||||
print('\n'.join([e[0] for e in exchanges]))
|
||||
else:
|
||||
if args['list_exchanges_all']:
|
||||
print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
|
||||
print("All exchanges supported by the ccxt library:")
|
||||
else:
|
||||
print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
|
||||
print("Exchanges available for Freqtrade:")
|
||||
exchanges = [e for e in exchanges if e[1] is not False]
|
||||
|
||||
print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
|
||||
|
||||
|
||||
def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
|
||||
@@ -50,15 +53,21 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
|
||||
reset = ''
|
||||
|
||||
names = [s['name'] for s in objs]
|
||||
objss_to_print = [{
|
||||
objs_to_print = [{
|
||||
'name': s['name'] if s['name'] else "--",
|
||||
'location': s['location'].name,
|
||||
'status': (red + "LOAD FAILED" + reset if s['class'] is None
|
||||
else "OK" if names.count(s['name']) == 1
|
||||
else yellow + "DUPLICATE NAME" + reset)
|
||||
} for s in objs]
|
||||
|
||||
print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right'))
|
||||
for idx, s in enumerate(objs):
|
||||
if 'hyperoptable' in s:
|
||||
objs_to_print[idx].update({
|
||||
'hyperoptable': "Yes" if s['hyperoptable']['count'] > 0 else "No",
|
||||
'buy-Params': len(s['hyperoptable'].get('buy', [])),
|
||||
'sell-Params': len(s['hyperoptable'].get('sell', [])),
|
||||
})
|
||||
print(tabulate(objs_to_print, headers='keys', tablefmt='psql', stralign='right'))
|
||||
|
||||
|
||||
def start_list_strategies(args: Dict[str, Any]) -> None:
|
||||
@@ -71,6 +80,11 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
|
||||
strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column'])
|
||||
# Sort alphabetically
|
||||
strategy_objs = sorted(strategy_objs, key=lambda x: x['name'])
|
||||
for obj in strategy_objs:
|
||||
if obj['class']:
|
||||
obj['hyperoptable'] = obj['class'].detect_all_parameters()
|
||||
else:
|
||||
obj['hyperoptable'] = {'count': 0}
|
||||
|
||||
if args['print_one_column']:
|
||||
print('\n'.join([s['name'] for s in strategy_objs]))
|
||||
@@ -78,28 +92,9 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
|
||||
_print_objs_tabular(strategy_objs, config.get('print_colorized', False))
|
||||
|
||||
|
||||
def start_list_hyperopts(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Print files with HyperOpt custom classes available in the directory
|
||||
"""
|
||||
from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver
|
||||
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
|
||||
|
||||
directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS))
|
||||
hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column'])
|
||||
# Sort alphabetically
|
||||
hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name'])
|
||||
|
||||
if args['print_one_column']:
|
||||
print('\n'.join([s['name'] for s in hyperopt_objs]))
|
||||
else:
|
||||
_print_objs_tabular(hyperopt_objs, config.get('print_colorized', False))
|
||||
|
||||
|
||||
def start_list_timeframes(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Print ticker intervals (timeframes) available on Exchange
|
||||
Print timeframes available on Exchange
|
||||
"""
|
||||
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
|
||||
# Do not use timeframe set in the config
|
||||
@@ -139,7 +134,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
|
||||
pairs_only=pairs_only,
|
||||
active_only=active_only)
|
||||
# Sort the pairs/markets by symbol
|
||||
pairs = OrderedDict(sorted(pairs.items()))
|
||||
pairs = dict(sorted(pairs.items()))
|
||||
except Exception as e:
|
||||
raise OperationalException(f"Cannot get markets. Reason: {e}") from e
|
||||
|
||||
@@ -177,7 +172,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
|
||||
# human-readable formats.
|
||||
print()
|
||||
|
||||
if len(pairs):
|
||||
if pairs:
|
||||
if args.get('print_list', False):
|
||||
# print data as a list, with human-readable summary
|
||||
print(f"{summary_str}: {', '.join(pairs.keys())}.")
|
||||
@@ -211,7 +206,7 @@ def start_show_trades(args: Dict[str, Any]) -> None:
|
||||
if 'db_url' not in config:
|
||||
raise OperationalException("--db-url is required for this command.")
|
||||
|
||||
logger.info(f'Using DB: "{config["db_url"]}"')
|
||||
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
|
||||
init_db(config['db_url'], clean_open_orders=False)
|
||||
tfilter = []
|
||||
|
||||
|
@@ -3,9 +3,9 @@ from typing import Any, Dict
|
||||
|
||||
from freqtrade import constants
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import round_coin_value
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -15,6 +15,7 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
|
||||
"""
|
||||
Prepare the configuration for the Hyperopt module
|
||||
:param args: Cli args from Arguments()
|
||||
:param method: Bot running mode
|
||||
:return: Configuration
|
||||
"""
|
||||
config = setup_utils_configuration(args, method)
|
||||
|
@@ -4,8 +4,8 @@ from typing import Any, Dict
|
||||
import rapidjson
|
||||
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.resolvers import ExchangeResolver
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -31,7 +31,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
|
||||
results[curr] = pairlists.whitelist
|
||||
|
||||
for curr, pairlist in results.items():
|
||||
if not args.get('print_one_column', False):
|
||||
if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False):
|
||||
print(f"Pairs for {curr}: ")
|
||||
|
||||
if args.get('print_one_column', False):
|
||||
|
@@ -1,8 +1,8 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.configuration import setup_utils_configuration
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
def validate_plot_args(args: Dict[str, Any]) -> None:
|
||||
|
15
freqtrade/commands/webserver_commands.py
Normal file
15
freqtrade/commands/webserver_commands.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.enums import RunMode
|
||||
|
||||
|
||||
def start_webserver(args: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Main entry point for webserver mode
|
||||
"""
|
||||
from freqtrade.configuration import Configuration
|
||||
from freqtrade.rpc.api_server import ApiServer
|
||||
|
||||
# Initialize configuration
|
||||
config = Configuration(args, RunMode.WEBSERVER).get_config()
|
||||
ApiServer(config, standalone=True)
|
19
freqtrade/configuration/PeriodicCache.py
Normal file
19
freqtrade/configuration/PeriodicCache.py
Normal file
@@ -0,0 +1,19 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from cachetools.ttl import TTLCache
|
||||
|
||||
|
||||
class PeriodicCache(TTLCache):
|
||||
"""
|
||||
Special cache that expires at "straight" times
|
||||
A timer with ttl of 3600 (1h) will expire at every full hour (:00).
|
||||
"""
|
||||
|
||||
def __init__(self, maxsize, ttl, getsizeof=None):
|
||||
def local_timer():
|
||||
ts = datetime.now(timezone.utc).timestamp()
|
||||
offset = (ts % ttl)
|
||||
return ts - offset
|
||||
|
||||
# Init with smlight offset
|
||||
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)
|
@@ -1,7 +1,8 @@
|
||||
# flake8: noqa: F401
|
||||
|
||||
from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
|
||||
from freqtrade.configuration.check_exchange import check_exchange
|
||||
from freqtrade.configuration.config_setup import setup_utils_configuration
|
||||
from freqtrade.configuration.config_validation import validate_config_consistency
|
||||
from freqtrade.configuration.configuration import Configuration
|
||||
from freqtrade.configuration.PeriodicCache import PeriodicCache
|
||||
from freqtrade.configuration.timerange import TimeRange
|
||||
|
@@ -1,28 +1,15 @@
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad,
|
||||
is_exchange_known_ccxt, is_exchange_officially_supported)
|
||||
from freqtrade.state import RunMode
|
||||
from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
|
||||
is_exchange_officially_supported, validate_exchange)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def remove_credentials(config: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Removes exchange keys from the configuration and specifies dry-run
|
||||
Used for backtesting / hyperopt / edge and utils.
|
||||
Modifies the input dict!
|
||||
"""
|
||||
config['exchange']['key'] = ''
|
||||
config['exchange']['secret'] = ''
|
||||
config['exchange']['password'] = ''
|
||||
config['exchange']['uid'] = ''
|
||||
config['dry_run'] = True
|
||||
|
||||
|
||||
def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
|
||||
"""
|
||||
Check if the exchange name in the config file is supported by Freqtrade
|
||||
@@ -51,15 +38,19 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
|
||||
|
||||
if not is_exchange_known_ccxt(exchange):
|
||||
raise OperationalException(
|
||||
f'Exchange "{exchange}" is not known to the ccxt library '
|
||||
f'and therefore not available for the bot.\n'
|
||||
f'The following exchanges are available for Freqtrade: '
|
||||
f'{", ".join(available_exchanges())}'
|
||||
f'Exchange "{exchange}" is not known to the ccxt library '
|
||||
f'and therefore not available for the bot.\n'
|
||||
f'The following exchanges are available for Freqtrade: '
|
||||
f'{", ".join(available_exchanges())}'
|
||||
)
|
||||
|
||||
if check_for_bad and is_exchange_bad(exchange):
|
||||
raise OperationalException(f'Exchange "{exchange}" is known to not work with the bot yet. '
|
||||
f'Reason: {get_exchange_bad_reason(exchange)}')
|
||||
valid, reason = validate_exchange(exchange)
|
||||
if not valid:
|
||||
if check_for_bad:
|
||||
raise OperationalException(f'Exchange "{exchange}" will not work with Freqtrade. '
|
||||
f'Reason: {reason}')
|
||||
else:
|
||||
logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}')
|
||||
|
||||
if is_exchange_officially_supported(exchange):
|
||||
logger.info(f'Exchange "{exchange}" is officially supported '
|
||||
|
@@ -1,9 +1,8 @@
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.state import RunMode
|
||||
from freqtrade.enums import RunMode
|
||||
|
||||
from .check_exchange import remove_credentials
|
||||
from .config_validation import validate_config_consistency
|
||||
from .configuration import Configuration
|
||||
|
||||
@@ -15,13 +14,14 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
|
||||
"""
|
||||
Prepare the configuration for utils subcommands
|
||||
:param args: Cli args from Arguments()
|
||||
:param method: Bot running mode
|
||||
:return: Configuration
|
||||
"""
|
||||
configuration = Configuration(args, method)
|
||||
config = configuration.get_config()
|
||||
|
||||
# Ensure we do not use Exchange credentials
|
||||
remove_credentials(config)
|
||||
# Ensure these modes are using Dry-run
|
||||
config['dry_run'] = True
|
||||
validate_config_consistency(config)
|
||||
|
||||
return config
|
||||
|
@@ -6,8 +6,8 @@ from jsonschema import Draft4Validator, validators
|
||||
from jsonschema.exceptions import ValidationError, best_match
|
||||
|
||||
from freqtrade import constants
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.state import RunMode
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -74,10 +74,12 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
|
||||
|
||||
# validating trailing stoploss
|
||||
_validate_trailing_stoploss(conf)
|
||||
_validate_price_config(conf)
|
||||
_validate_edge(conf)
|
||||
_validate_whitelist(conf)
|
||||
_validate_protections(conf)
|
||||
_validate_unlimited_amount(conf)
|
||||
_validate_ask_orderbook(conf)
|
||||
|
||||
# validate configuration before returning
|
||||
logger.info('Validating configuration ...')
|
||||
@@ -95,12 +97,25 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
|
||||
raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.")
|
||||
|
||||
|
||||
def _validate_price_config(conf: Dict[str, Any]) -> None:
|
||||
"""
|
||||
When using market orders, price sides must be using the "other" side of the price
|
||||
"""
|
||||
if (conf.get('order_types', {}).get('buy') == 'market'
|
||||
and conf.get('bid_strategy', {}).get('price_side') != 'ask'):
|
||||
raise OperationalException('Market buy orders require bid_strategy.price_side = "ask".')
|
||||
|
||||
if (conf.get('order_types', {}).get('sell') == 'market'
|
||||
and conf.get('ask_strategy', {}).get('price_side') != 'bid'):
|
||||
raise OperationalException('Market sell orders require ask_strategy.price_side = "bid".')
|
||||
|
||||
|
||||
def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
|
||||
|
||||
if conf.get('stoploss') == 0.0:
|
||||
raise OperationalException(
|
||||
'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
|
||||
)
|
||||
)
|
||||
# Skip if trailing stoploss is not activated
|
||||
if not conf.get('trailing_stop', False):
|
||||
return
|
||||
@@ -135,12 +150,7 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
|
||||
if not conf.get('edge', {}).get('enabled'):
|
||||
return
|
||||
|
||||
if conf.get('pairlist', {}).get('method') == 'VolumePairList':
|
||||
raise OperationalException(
|
||||
"Edge and VolumePairList are incompatible, "
|
||||
"Edge will override whatever pairs VolumePairlist selects."
|
||||
)
|
||||
if not conf.get('ask_strategy', {}).get('use_sell_signal', True):
|
||||
if not conf.get('use_sell_signal', True):
|
||||
raise OperationalException(
|
||||
"Edge requires `use_sell_signal` to be True, otherwise no sells will happen."
|
||||
)
|
||||
@@ -170,10 +180,30 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
|
||||
raise OperationalException(
|
||||
"Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
|
||||
f"Please fix the protection {prot.get('method')}"
|
||||
)
|
||||
)
|
||||
|
||||
if ('lookback_period' in prot and 'lookback_period_candles' in prot):
|
||||
raise OperationalException(
|
||||
"Protections must specify either `lookback_period` or `lookback_period_candles`.\n"
|
||||
f"Please fix the protection {prot.get('method')}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
|
||||
ask_strategy = conf.get('ask_strategy', {})
|
||||
ob_min = ask_strategy.get('order_book_min')
|
||||
ob_max = ask_strategy.get('order_book_max')
|
||||
if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
|
||||
if ob_min != ob_max:
|
||||
raise OperationalException(
|
||||
"Using order_book_max != order_book_min in ask_strategy is no longer supported."
|
||||
"Please pick one value and use `order_book_top` in the future."
|
||||
)
|
||||
else:
|
||||
# Move value to order_book_top
|
||||
ask_strategy['order_book_top'] = ob_min
|
||||
logger.warning(
|
||||
"DEPRECATED: "
|
||||
"Please use `order_book_top` instead of `order_book_min` and `order_book_max` "
|
||||
"for your `ask_strategy` configuration."
|
||||
)
|
||||
|
@@ -11,11 +11,12 @@ from freqtrade import constants
|
||||
from freqtrade.configuration.check_exchange import check_exchange
|
||||
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
|
||||
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
|
||||
from freqtrade.configuration.load_config import load_config_file
|
||||
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
|
||||
from freqtrade.configuration.load_config import load_config_file, load_file
|
||||
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.loggers import setup_logging
|
||||
from freqtrade.misc import deep_merge_dicts, json_load
|
||||
from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode
|
||||
from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -72,11 +73,14 @@ class Configuration:
|
||||
# Merge config options, overwriting old values
|
||||
config = deep_merge_dicts(load_config_file(path), config)
|
||||
|
||||
# Load environment variables
|
||||
env_data = enironment_vars_to_dict()
|
||||
config = deep_merge_dicts(env_data, config)
|
||||
|
||||
config['config_files'] = files
|
||||
# Normalize config
|
||||
if 'internals' not in config:
|
||||
config['internals'] = {}
|
||||
# TODO: This can be deleted along with removal of deprecated
|
||||
# experimental settings
|
||||
if 'ask_strategy' not in config:
|
||||
config['ask_strategy'] = {}
|
||||
|
||||
@@ -108,6 +112,8 @@ class Configuration:
|
||||
|
||||
self._process_plot_options(config)
|
||||
|
||||
self._process_data_options(config)
|
||||
|
||||
# Check if the exchange set by the user is supported
|
||||
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
|
||||
|
||||
@@ -144,7 +150,7 @@ class Configuration:
|
||||
config['db_url'] = constants.DEFAULT_DB_PROD_URL
|
||||
logger.info('Dry run is disabled')
|
||||
|
||||
logger.info(f'Using DB: "{config["db_url"]}"')
|
||||
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
|
||||
|
||||
def _process_common_options(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
@@ -236,6 +242,9 @@ class Configuration:
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
self._args_to_config(config, argname='timeframe_detail',
|
||||
logstring='Parameter --timeframe-detail detected, '
|
||||
'using {} for intra-candle backtesting ...')
|
||||
self._args_to_config(config, argname='stake_amount',
|
||||
logstring='Parameter --stake-amount detected, '
|
||||
'overriding stake_amount to: {} ...')
|
||||
@@ -263,6 +272,9 @@ class Configuration:
|
||||
self._args_to_config(config, argname='show_days',
|
||||
logstring='Parameter --show-days detected ...')
|
||||
|
||||
self._args_to_config(config, argname='disableparamexport',
|
||||
logstring='Parameter --disableparamexport detected: {} ...')
|
||||
|
||||
# Edge section:
|
||||
if 'stoploss_range' in self.args and self.args["stoploss_range"]:
|
||||
txt_range = eval(self.args["stoploss_range"])
|
||||
@@ -361,6 +373,9 @@ class Configuration:
|
||||
self._args_to_config(config, argname='hyperopt_show_no_header',
|
||||
logstring='Parameter --no-header detected: {}')
|
||||
|
||||
self._args_to_config(config, argname="hyperopt_ignore_missing_space",
|
||||
logstring="Paramter --ignore-missing-space detected: {}")
|
||||
|
||||
def _process_plot_options(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='pairs',
|
||||
@@ -378,6 +393,9 @@ class Configuration:
|
||||
self._args_to_config(config, argname='plot_limit',
|
||||
logstring='Limiting plot to: {}')
|
||||
|
||||
self._args_to_config(config, argname='plot_auto_open',
|
||||
logstring='Parameter --auto-open detected.')
|
||||
|
||||
self._args_to_config(config, argname='trade_source',
|
||||
logstring='Using trades from: {}')
|
||||
|
||||
@@ -402,6 +420,11 @@ class Configuration:
|
||||
self._args_to_config(config, argname='dataformat_trades',
|
||||
logstring='Using "{}" to store trades data.')
|
||||
|
||||
def _process_data_options(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='new_pairs_days',
|
||||
logstring='Detected --new-pairs-days: {}')
|
||||
|
||||
def _process_runmode(self, config: Dict[str, Any]) -> None:
|
||||
|
||||
self._args_to_config(config, argname='dry_run',
|
||||
@@ -448,18 +471,18 @@ class Configuration:
|
||||
"""
|
||||
|
||||
if "pairs" in config:
|
||||
config['exchange']['pair_whitelist'] = config['pairs']
|
||||
return
|
||||
|
||||
if "pairs_file" in self.args and self.args["pairs_file"]:
|
||||
pairs_file = Path(self.args["pairs_file"])
|
||||
logger.info(f'Reading pairs file "{pairs_file}".')
|
||||
# Download pairs from the pairs file if no config is specified
|
||||
# or if pairs file is specified explicitely
|
||||
# or if pairs file is specified explicitly
|
||||
if not pairs_file.exists():
|
||||
raise OperationalException(f'No pairs file found with path "{pairs_file}".')
|
||||
with pairs_file.open('r') as f:
|
||||
config['pairs'] = json_load(f)
|
||||
config['pairs'].sort()
|
||||
config['pairs'] = load_file(pairs_file)
|
||||
config['pairs'].sort()
|
||||
return
|
||||
|
||||
if 'config' in self.args and self.args['config']:
|
||||
@@ -469,7 +492,6 @@ class Configuration:
|
||||
# Fall back to /dl_path/pairs.json
|
||||
pairs_file = config['datadir'] / 'pairs.json'
|
||||
if pairs_file.exists():
|
||||
with pairs_file.open('r') as f:
|
||||
config['pairs'] = json_load(f)
|
||||
config['pairs'] = load_file(pairs_file)
|
||||
if 'pairs' in config:
|
||||
config['pairs'].sort()
|
||||
|
@@ -3,7 +3,7 @@ Functions to handle deprecated settings
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
|
||||
@@ -12,23 +12,24 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_conflicting_settings(config: Dict[str, Any],
|
||||
section1: str, name1: str,
|
||||
section2: str, name2: str) -> None:
|
||||
section1_config = config.get(section1, {})
|
||||
section2_config = config.get(section2, {})
|
||||
if name1 in section1_config and name2 in section2_config:
|
||||
section_old: str, name_old: str,
|
||||
section_new: Optional[str], name_new: str) -> None:
|
||||
section_new_config = config.get(section_new, {}) if section_new else config
|
||||
section_old_config = config.get(section_old, {})
|
||||
if name_new in section_new_config and name_old in section_old_config:
|
||||
new_name = f"{section_new}.{name_new}" if section_new else f"{name_new}"
|
||||
raise OperationalException(
|
||||
f"Conflicting settings `{section1}.{name1}` and `{section2}.{name2}` "
|
||||
f"Conflicting settings `{new_name}` and `{section_old}.{name_old}` "
|
||||
"(DEPRECATED) detected in the configuration file. "
|
||||
"This deprecated setting will be removed in the next versions of Freqtrade. "
|
||||
f"Please delete it from your configuration and use the `{section1}.{name1}` "
|
||||
f"Please delete it from your configuration and use the `{new_name}` "
|
||||
"setting instead."
|
||||
)
|
||||
|
||||
|
||||
def process_removed_setting(config: Dict[str, Any],
|
||||
section1: str, name1: str,
|
||||
section2: str, name2: str) -> None:
|
||||
section2: Optional[str], name2: str) -> None:
|
||||
"""
|
||||
:param section1: Removed section
|
||||
:param name1: Removed setting name
|
||||
@@ -37,27 +38,32 @@ def process_removed_setting(config: Dict[str, Any],
|
||||
"""
|
||||
section1_config = config.get(section1, {})
|
||||
if name1 in section1_config:
|
||||
section_2 = f"{section2}.{name2}" if section2 else f"{name2}"
|
||||
raise OperationalException(
|
||||
f"Setting `{section1}.{name1}` has been moved to `{section2}.{name2}. "
|
||||
f"Please delete it from your configuration and use the `{section2}.{name2}` "
|
||||
f"Setting `{section1}.{name1}` has been moved to `{section_2}. "
|
||||
f"Please delete it from your configuration and use the `{section_2}` "
|
||||
"setting instead."
|
||||
)
|
||||
|
||||
|
||||
def process_deprecated_setting(config: Dict[str, Any],
|
||||
section1: str, name1: str,
|
||||
section2: str, name2: str) -> None:
|
||||
section2_config = config.get(section2, {})
|
||||
section_old: str, name_old: str,
|
||||
section_new: Optional[str], name_new: str
|
||||
) -> None:
|
||||
check_conflicting_settings(config, section_old, name_old, section_new, name_new)
|
||||
section_old_config = config.get(section_old, {})
|
||||
|
||||
if name2 in section2_config:
|
||||
if name_old in section_old_config:
|
||||
section_2 = f"{section_new}.{name_new}" if section_new else f"{name_new}"
|
||||
logger.warning(
|
||||
"DEPRECATED: "
|
||||
f"The `{section2}.{name2}` setting is deprecated and "
|
||||
f"The `{section_old}.{name_old}` setting is deprecated and "
|
||||
"will be removed in the next versions of Freqtrade. "
|
||||
f"Please use the `{section1}.{name1}` setting in your configuration instead."
|
||||
f"Please use the `{section_2}` setting in your configuration instead."
|
||||
)
|
||||
section1_config = config.get(section1, {})
|
||||
section1_config[name1] = section2_config[name2]
|
||||
|
||||
section_new_config = config.get(section_new, {}) if section_new else config
|
||||
section_new_config[name_new] = section_old_config[name_old]
|
||||
|
||||
|
||||
def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
|
||||
@@ -65,15 +71,24 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
|
||||
# Kept for future deprecated / moved settings
|
||||
# check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
|
||||
# 'experimental', 'use_sell_signal')
|
||||
# process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
|
||||
# 'experimental', 'use_sell_signal')
|
||||
process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
|
||||
None, 'use_sell_signal')
|
||||
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only',
|
||||
None, 'sell_profit_only')
|
||||
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_offset',
|
||||
None, 'sell_profit_offset')
|
||||
process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
|
||||
None, 'ignore_roi_if_buy_signal')
|
||||
process_deprecated_setting(config, 'ask_strategy', 'ignore_buying_expired_candle_after',
|
||||
None, 'ignore_buying_expired_candle_after')
|
||||
|
||||
# Legacy way - having them in experimental ...
|
||||
process_removed_setting(config, 'experimental', 'use_sell_signal',
|
||||
'ask_strategy', 'use_sell_signal')
|
||||
None, 'use_sell_signal')
|
||||
process_removed_setting(config, 'experimental', 'sell_profit_only',
|
||||
'ask_strategy', 'sell_profit_only')
|
||||
None, 'sell_profit_only')
|
||||
process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
|
||||
'ask_strategy', 'ignore_roi_if_buy_signal')
|
||||
None, 'ignore_roi_if_buy_signal')
|
||||
|
||||
if (config.get('edge', {}).get('enabled', False)
|
||||
and 'capital_available_percentage' in config.get('edge', {})):
|
||||
@@ -93,5 +108,8 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
|
||||
raise OperationalException(
|
||||
"Both 'timeframe' and 'ticker_interval' detected."
|
||||
"Please remove 'ticker_interval' from your configuration to continue operating."
|
||||
)
|
||||
)
|
||||
config['timeframe'] = config['ticker_interval']
|
||||
|
||||
if 'protections' in config:
|
||||
logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")
|
||||
|
@@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat
|
||||
return folder
|
||||
|
||||
|
||||
def chown_user_directory(directory: Path) -> None:
|
||||
"""
|
||||
Use Sudo to change permissions of the home-directory if necessary
|
||||
Only applies when running in docker!
|
||||
"""
|
||||
import os
|
||||
if os.environ.get('FT_APP_ENV') == 'docker':
|
||||
try:
|
||||
import subprocess
|
||||
subprocess.check_output(
|
||||
['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())])
|
||||
except Exception:
|
||||
logger.warning(f"Could not chown {directory}")
|
||||
|
||||
|
||||
def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
|
||||
"""
|
||||
Create userdata directory structure.
|
||||
@@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
|
||||
sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs",
|
||||
"notebooks", "plot", "strategies", ]
|
||||
folder = Path(directory)
|
||||
chown_user_directory(folder)
|
||||
if not folder.is_dir():
|
||||
if create_dir:
|
||||
folder.mkdir(parents=True)
|
||||
@@ -72,6 +88,5 @@ def copy_sample_files(directory: Path, overwrite: bool = False) -> None:
|
||||
if not overwrite:
|
||||
logger.warning(f"File `{targetfile}` exists already, not deploying sample file.")
|
||||
continue
|
||||
else:
|
||||
logger.warning(f"File `{targetfile}` exists already, overwriting.")
|
||||
logger.warning(f"File `{targetfile}` exists already, overwriting.")
|
||||
shutil.copy(str(sourcedir / source), str(targetfile))
|
||||
|
54
freqtrade/configuration/environment_vars.py
Normal file
54
freqtrade/configuration/environment_vars.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.constants import ENV_VAR_PREFIX
|
||||
from freqtrade.misc import deep_merge_dicts
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_var_typed(val):
|
||||
try:
|
||||
return int(val)
|
||||
except ValueError:
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
if val.lower() in ('t', 'true'):
|
||||
return True
|
||||
elif val.lower() in ('f', 'false'):
|
||||
return False
|
||||
# keep as string
|
||||
return val
|
||||
|
||||
|
||||
def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Environment variables must be prefixed with FREQTRADE.
|
||||
FREQTRADE__{section}__{key}
|
||||
:param env_dict: Dictionary to validate - usually os.environ
|
||||
:param prefix: Prefix to consider (usually FREQTRADE__)
|
||||
:return: Nested dict based on available and relevant variables.
|
||||
"""
|
||||
relevant_vars: Dict[str, Any] = {}
|
||||
|
||||
for env_var, val in sorted(env_dict.items()):
|
||||
if env_var.startswith(prefix):
|
||||
logger.info(f"Loading variable '{env_var}'")
|
||||
key = env_var.replace(prefix, '')
|
||||
for k in reversed(key.split('__')):
|
||||
val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
|
||||
relevant_vars = deep_merge_dicts(val, relevant_vars)
|
||||
|
||||
return relevant_vars
|
||||
|
||||
|
||||
def enironment_vars_to_dict() -> Dict[str, Any]:
|
||||
"""
|
||||
Read environment variables and return a nested dict for relevant variables
|
||||
Relevant variables must follow the FREQTRADE__{section}__{key} pattern
|
||||
:return: Nested dict based on available and relevant variables.
|
||||
"""
|
||||
return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)
|
@@ -38,6 +38,15 @@ def log_config_error_range(path: str, errmsg: str) -> str:
|
||||
return ''
|
||||
|
||||
|
||||
def load_file(path: Path) -> Dict[str, Any]:
|
||||
try:
|
||||
with path.open('r') as file:
|
||||
config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
|
||||
except FileNotFoundError:
|
||||
raise OperationalException(f'File "{path}" not found!')
|
||||
return config
|
||||
|
||||
|
||||
def load_config_file(path: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Loads a config file from the given path
|
||||
|
@@ -3,10 +3,13 @@ This module contains the argument manager class
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import arrow
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -41,7 +44,7 @@ class TimeRange:
|
||||
self.startts = self.startts - seconds
|
||||
|
||||
def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
|
||||
min_date: arrow.Arrow) -> None:
|
||||
min_date: datetime) -> None:
|
||||
"""
|
||||
Adjust startts by <startup_candles> candles.
|
||||
Applies only if no startup-candles have been available.
|
||||
@@ -52,11 +55,11 @@ class TimeRange:
|
||||
:return: None (Modifies the object in place)
|
||||
"""
|
||||
if (not self.starttype or (startup_candles
|
||||
and min_date.int_timestamp >= self.startts)):
|
||||
and min_date.timestamp() >= self.startts)):
|
||||
# If no startts was defined, or backtest-data starts at the defined backtest-date
|
||||
logger.warning("Moving start-date by %s candles to account for startup time.",
|
||||
startup_candles)
|
||||
self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles)
|
||||
self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
|
||||
self.starttype = 'date'
|
||||
|
||||
@staticmethod
|
||||
@@ -103,5 +106,8 @@ class TimeRange:
|
||||
stop = int(stops) // 1000
|
||||
else:
|
||||
stop = int(stops)
|
||||
if start > stop > 0:
|
||||
raise OperationalException(
|
||||
f'Start date is after stop date for timerange "{text}"')
|
||||
return TimeRange(stype[0], stype[1], start, stop)
|
||||
raise Exception('Incorrect syntax for timerange "%s"' % text)
|
||||
raise OperationalException(f'Incorrect syntax for timerange "{text}"')
|
||||
|
@@ -11,6 +11,8 @@ DEFAULT_EXCHANGE = 'bittrex'
|
||||
PROCESS_THROTTLE_SECS = 5 # sec
|
||||
HYPEROPT_EPOCH = 100 # epochs
|
||||
RETRY_TIMEOUT = 30 # sec
|
||||
TIMEOUT_UNITS = ['minutes', 'seconds']
|
||||
EXPORT_OPTIONS = ['none', 'trades']
|
||||
DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
|
||||
DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
|
||||
UNLIMITED_STAKE_AMOUNT = 'unlimited'
|
||||
@@ -22,11 +24,12 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market']
|
||||
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
|
||||
HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
|
||||
'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
|
||||
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
|
||||
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily',
|
||||
'MaxDrawDownHyperOptLoss']
|
||||
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
|
||||
'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
|
||||
'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
|
||||
'SpreadFilter']
|
||||
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
|
||||
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
|
||||
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
|
||||
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
|
||||
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
|
||||
DRY_RUN_WALLET = 1000
|
||||
@@ -38,12 +41,16 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
|
||||
DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
|
||||
|
||||
LAST_BT_RESULT_FN = '.last_result.json'
|
||||
FTHYPT_FILEVERSION = 'fthypt_fileversion'
|
||||
|
||||
USERPATH_HYPEROPTS = 'hyperopts'
|
||||
USERPATH_STRATEGIES = 'strategies'
|
||||
USERPATH_NOTEBOOKS = 'notebooks'
|
||||
|
||||
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
|
||||
ENV_VAR_PREFIX = 'FREQTRADE__'
|
||||
|
||||
NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
|
||||
|
||||
|
||||
# Define decimals per coin for outputs
|
||||
@@ -60,12 +67,10 @@ DUST_PER_COIN = {
|
||||
}
|
||||
|
||||
|
||||
# Soure files with destination directories within user-directory
|
||||
# Source files with destination directories within user-directory
|
||||
USER_DATA_FILES = {
|
||||
'sample_strategy.py': USERPATH_STRATEGIES,
|
||||
'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS,
|
||||
'sample_hyperopt_loss.py': USERPATH_HYPEROPTS,
|
||||
'sample_hyperopt.py': USERPATH_HYPEROPTS,
|
||||
'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS,
|
||||
}
|
||||
|
||||
@@ -96,6 +101,7 @@ CONF_SCHEMA = {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
|
||||
'new_pairs_days': {'type': 'integer', 'default': 30},
|
||||
'timeframe': {'type': 'string'},
|
||||
'stake_currency': {'type': 'string'},
|
||||
'stake_amount': {
|
||||
@@ -105,10 +111,14 @@ CONF_SCHEMA = {
|
||||
},
|
||||
'tradable_balance_ratio': {
|
||||
'type': 'number',
|
||||
'minimum': 0.1,
|
||||
'minimum': 0.0,
|
||||
'maximum': 1,
|
||||
'default': 0.99
|
||||
},
|
||||
'available_capital': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
},
|
||||
'amend_last_stake_amount': {'type': 'boolean', 'default': False},
|
||||
'last_stake_amount_min_ratio': {
|
||||
'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5
|
||||
@@ -131,12 +141,18 @@ CONF_SCHEMA = {
|
||||
'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1},
|
||||
'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1},
|
||||
'trailing_only_offset_is_reached': {'type': 'boolean'},
|
||||
'use_sell_signal': {'type': 'boolean'},
|
||||
'sell_profit_only': {'type': 'boolean'},
|
||||
'sell_profit_offset': {'type': 'number'},
|
||||
'ignore_roi_if_buy_signal': {'type': 'boolean'},
|
||||
'ignore_buying_expired_candle_after': {'type': 'number'},
|
||||
'bot_name': {'type': 'string'},
|
||||
'unfilledtimeout': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'buy': {'type': 'number', 'minimum': 1},
|
||||
'sell': {'type': 'number', 'minimum': 1}
|
||||
'sell': {'type': 'number', 'minimum': 1},
|
||||
'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'}
|
||||
}
|
||||
},
|
||||
'bid_strategy': {
|
||||
@@ -150,7 +166,7 @@ CONF_SCHEMA = {
|
||||
},
|
||||
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'},
|
||||
'use_order_book': {'type': 'boolean'},
|
||||
'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1},
|
||||
'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
|
||||
'check_depth_of_market': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
@@ -159,20 +175,25 @@ CONF_SCHEMA = {
|
||||
}
|
||||
},
|
||||
},
|
||||
'required': ['ask_last_balance']
|
||||
'required': ['price_side']
|
||||
},
|
||||
'ask_strategy': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'ask'},
|
||||
'bid_last_balance': {
|
||||
'type': 'number',
|
||||
'minimum': 0,
|
||||
'maximum': 1,
|
||||
'exclusiveMaximum': False,
|
||||
},
|
||||
'use_order_book': {'type': 'boolean'},
|
||||
'order_book_min': {'type': 'integer', 'minimum': 1},
|
||||
'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50},
|
||||
'use_sell_signal': {'type': 'boolean'},
|
||||
'sell_profit_only': {'type': 'boolean'},
|
||||
'sell_profit_offset': {'type': 'number', 'minimum': 0.0},
|
||||
'ignore_roi_if_buy_signal': {'type': 'boolean'}
|
||||
}
|
||||
'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
|
||||
},
|
||||
'required': ['price_side']
|
||||
},
|
||||
'custom_price_max_distance_ratio': {
|
||||
'type': 'number', 'minimum': 0.0
|
||||
},
|
||||
'order_types': {
|
||||
'type': 'object',
|
||||
@@ -240,16 +261,42 @@ CONF_SCHEMA = {
|
||||
'balance_dust_level': {'type': 'number', 'minimum': 0.0},
|
||||
'notification_settings': {
|
||||
'type': 'object',
|
||||
'default': {},
|
||||
'properties': {
|
||||
'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}
|
||||
'buy_fill': {'type': 'string',
|
||||
'enum': TELEGRAM_SETTING_OPTIONS,
|
||||
'default': 'off'
|
||||
},
|
||||
'sell': {
|
||||
'type': ['string', 'object'],
|
||||
'additionalProperties': {
|
||||
'type': 'string',
|
||||
'enum': TELEGRAM_SETTING_OPTIONS
|
||||
}
|
||||
},
|
||||
'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
|
||||
'sell_fill': {
|
||||
'type': 'string',
|
||||
'enum': TELEGRAM_SETTING_OPTIONS,
|
||||
'default': 'off'
|
||||
},
|
||||
'protection_trigger': {
|
||||
'type': 'string',
|
||||
'enum': TELEGRAM_SETTING_OPTIONS,
|
||||
'default': 'off'
|
||||
},
|
||||
'protection_trigger_global': {
|
||||
'type': 'string',
|
||||
'enum': TELEGRAM_SETTING_OPTIONS,
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
'reload': {'type': 'boolean'},
|
||||
},
|
||||
'required': ['enabled', 'token', 'chat_id'],
|
||||
},
|
||||
@@ -283,6 +330,8 @@ CONF_SCHEMA = {
|
||||
'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password']
|
||||
},
|
||||
'db_url': {'type': 'string'},
|
||||
'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
|
||||
'disableparamexport': {'type': 'boolean'},
|
||||
'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
|
||||
'forcebuy_enable': {'type': 'boolean'},
|
||||
'disable_dataframe_checks': {'type': 'boolean'},
|
||||
|
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
|
||||
BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
|
||||
"trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
|
||||
|
||||
# Mid-term format, crated by BacktestResult Named Tuple
|
||||
# Mid-term format, created by BacktestResult Named Tuple
|
||||
BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
|
||||
'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
|
||||
'fee_close', 'amount', 'profit_abs', 'profit_ratio']
|
||||
@@ -30,7 +30,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
|
||||
'fee_open', 'fee_close', 'trade_duration',
|
||||
'profit_ratio', 'profit_abs', 'sell_reason',
|
||||
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
|
||||
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
|
||||
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag']
|
||||
|
||||
|
||||
def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
|
||||
@@ -156,33 +156,35 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
|
||||
|
||||
data = data['strategy'][strategy]['trades']
|
||||
df = pd.DataFrame(data)
|
||||
df['open_date'] = pd.to_datetime(df['open_date'],
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
df['close_date'] = pd.to_datetime(df['close_date'],
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
if not df.empty:
|
||||
df['open_date'] = pd.to_datetime(df['open_date'],
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
df['close_date'] = pd.to_datetime(df['close_date'],
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
else:
|
||||
# old format - only with lists.
|
||||
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
|
||||
|
||||
df['open_date'] = pd.to_datetime(df['open_date'],
|
||||
unit='s',
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
df['close_date'] = pd.to_datetime(df['close_date'],
|
||||
unit='s',
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
# Create compatibility with new format
|
||||
df['profit_abs'] = df['close_rate'] - df['open_rate']
|
||||
if 'profit_ratio' not in df.columns:
|
||||
df['profit_ratio'] = df['profit_percent']
|
||||
df = df.sort_values("open_date").reset_index(drop=True)
|
||||
if not df.empty:
|
||||
df['open_date'] = pd.to_datetime(df['open_date'],
|
||||
unit='s',
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
df['close_date'] = pd.to_datetime(df['close_date'],
|
||||
unit='s',
|
||||
utc=True,
|
||||
infer_datetime_format=True
|
||||
)
|
||||
# Create compatibility with new format
|
||||
df['profit_abs'] = df['close_rate'] - df['open_rate']
|
||||
if not df.empty:
|
||||
if 'profit_ratio' not in df.columns:
|
||||
df['profit_ratio'] = df['profit_percent']
|
||||
df = df.sort_values("open_date").reset_index(drop=True)
|
||||
return df
|
||||
|
||||
|
||||
@@ -337,7 +339,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
|
||||
"""
|
||||
Adds a column `col_name` with the cumulative profit for the given trades array.
|
||||
:param df: DataFrame with date index
|
||||
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
|
||||
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
|
||||
:param col_name: Column name that will be assigned the results
|
||||
:param timeframe: Timeframe used during the operations
|
||||
:return: Returns df with one additional column, col_name, containing the cumulative profit.
|
||||
@@ -349,8 +351,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
|
||||
timeframe_minutes = timeframe_to_minutes(timeframe)
|
||||
# Resample to timeframe to make sure trades match candles
|
||||
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
|
||||
)[['profit_ratio']].sum()
|
||||
df.loc[:, col_name] = _trades_sum['profit_ratio'].cumsum()
|
||||
)[['profit_abs']].sum()
|
||||
df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
|
||||
# Set first value to 0
|
||||
df.loc[df.iloc[0].name, col_name] = 0
|
||||
# FFill to get continuous
|
||||
|
@@ -49,7 +49,7 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
|
||||
fill_missing: bool = True,
|
||||
drop_incomplete: bool = True) -> DataFrame:
|
||||
"""
|
||||
Clense a OHLCV dataframe by
|
||||
Cleanse a OHLCV dataframe by
|
||||
* Grouping it by date (removes duplicate tics)
|
||||
* dropping last candles if requested
|
||||
* Filling up missing data (if requested)
|
||||
@@ -110,28 +110,62 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
|
||||
df.reset_index(inplace=True)
|
||||
len_before = len(dataframe)
|
||||
len_after = len(df)
|
||||
pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
|
||||
if len_before != len_after:
|
||||
logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}")
|
||||
message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
|
||||
f" - {round(pct_missing * 100, 2)}%")
|
||||
if pct_missing > 0.01:
|
||||
logger.info(message)
|
||||
else:
|
||||
# Don't be verbose if only a small amount is missing
|
||||
logger.debug(message)
|
||||
return df
|
||||
|
||||
|
||||
def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame:
|
||||
def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
|
||||
startup_candles: int = 0) -> DataFrame:
|
||||
"""
|
||||
Trim dataframe based on given timerange
|
||||
:param df: Dataframe to trim
|
||||
:param timerange: timerange (use start and end date if available)
|
||||
:param: df_date_col: Column in the dataframe to use as Date column
|
||||
:param df_date_col: Column in the dataframe to use as Date column
|
||||
:param startup_candles: When not 0, is used instead the timerange start date
|
||||
:return: trimmed dataframe
|
||||
"""
|
||||
if timerange.starttype == 'date':
|
||||
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
|
||||
df = df.loc[df[df_date_col] >= start, :]
|
||||
if startup_candles:
|
||||
# Trim candles instead of timeframe in case of given startup_candle count
|
||||
df = df.iloc[startup_candles:, :]
|
||||
else:
|
||||
if timerange.starttype == 'date':
|
||||
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
|
||||
df = df.loc[df[df_date_col] >= start, :]
|
||||
if timerange.stoptype == 'date':
|
||||
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
|
||||
df = df.loc[df[df_date_col] <= stop, :]
|
||||
return df
|
||||
|
||||
|
||||
def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange,
|
||||
startup_candles: int) -> Dict[str, DataFrame]:
|
||||
"""
|
||||
Trim startup period from analyzed dataframes
|
||||
:param preprocessed: Dict of pair: dataframe
|
||||
:param timerange: timerange (use start and end date if available)
|
||||
:param startup_candles: Startup-candles that should be removed
|
||||
:return: Dict of trimmed dataframes
|
||||
"""
|
||||
processed: Dict[str, DataFrame] = {}
|
||||
|
||||
for pair, df in preprocessed.items():
|
||||
trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles)
|
||||
if not trimed_df.empty:
|
||||
processed[pair] = trimed_df
|
||||
else:
|
||||
logger.warning(f'{pair} has no data left after adjusting for startup candles, '
|
||||
f'skipping.')
|
||||
return processed
|
||||
|
||||
|
||||
def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
|
||||
"""
|
||||
TODO: This should get a dedicated test
|
||||
@@ -208,7 +242,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
|
||||
:param config: Config dictionary
|
||||
:param convert_from: Source format
|
||||
:param convert_to: Target format
|
||||
:param erase: Erase souce data (does not apply if source and target format are identical)
|
||||
:param erase: Erase source data (does not apply if source and target format are identical)
|
||||
"""
|
||||
from freqtrade.data.history.idatahandler import get_datahandler
|
||||
src = get_datahandler(config['datadir'], convert_from)
|
||||
@@ -233,7 +267,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
|
||||
:param config: Config dictionary
|
||||
:param convert_from: Source format
|
||||
:param convert_to: Target format
|
||||
:param erase: Erase souce data (does not apply if source and target format are identical)
|
||||
:param erase: Erase source data (does not apply if source and target format are identical)
|
||||
"""
|
||||
from freqtrade.data.history.idatahandler import get_datahandler
|
||||
src = get_datahandler(config['datadir'], convert_from)
|
||||
|
@@ -10,23 +10,36 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
|
||||
from freqtrade.data.history import load_pair_history
|
||||
from freqtrade.enums import RunMode
|
||||
from freqtrade.exceptions import ExchangeError, OperationalException
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.state import RunMode
|
||||
from freqtrade.exchange import Exchange, timeframe_to_seconds
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.'
|
||||
MAX_DATAFRAME_CANDLES = 1000
|
||||
|
||||
|
||||
class DataProvider:
|
||||
|
||||
def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
|
||||
def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None:
|
||||
self._config = config
|
||||
self._exchange = exchange
|
||||
self._pairlists = pairlists
|
||||
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
|
||||
self.__slice_index: Optional[int] = None
|
||||
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
|
||||
|
||||
def _set_dataframe_max_index(self, limit_index: int):
|
||||
"""
|
||||
Limit analyzed dataframe to max specified index.
|
||||
:param limit_index: dataframe index.
|
||||
"""
|
||||
self.__slice_index = limit_index
|
||||
|
||||
def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
|
||||
"""
|
||||
@@ -45,51 +58,28 @@ class DataProvider:
|
||||
"""
|
||||
self._pairlists = pairlists
|
||||
|
||||
def refresh(self,
|
||||
pairlist: ListPairsWithTimeframes,
|
||||
helping_pairs: ListPairsWithTimeframes = None) -> None:
|
||||
"""
|
||||
Refresh data, called with each cycle
|
||||
"""
|
||||
if helping_pairs:
|
||||
self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
|
||||
else:
|
||||
self._exchange.refresh_latest_ohlcv(pairlist)
|
||||
|
||||
@property
|
||||
def available_pairs(self) -> ListPairsWithTimeframes:
|
||||
"""
|
||||
Return a list of tuples containing (pair, timeframe) for which data is currently cached.
|
||||
Should be whitelist + open trades.
|
||||
"""
|
||||
return list(self._exchange._klines.keys())
|
||||
|
||||
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
|
||||
"""
|
||||
Get candle (OHLCV) data for the given pair as DataFrame
|
||||
Please use the `available_pairs` method to verify which pairs are currently cached.
|
||||
:param pair: pair to get the data for
|
||||
:param timeframe: Timeframe to get data for
|
||||
:param copy: copy dataframe before returning if True.
|
||||
Use False only for read-only operations (where the dataframe is not modified)
|
||||
"""
|
||||
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
|
||||
return self._exchange.klines((pair, timeframe or self._config['timeframe']),
|
||||
copy=copy)
|
||||
else:
|
||||
return DataFrame()
|
||||
|
||||
def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
|
||||
"""
|
||||
Get stored historical candle (OHLCV) data
|
||||
:param pair: pair to get the data for
|
||||
:param timeframe: timeframe to get data for
|
||||
"""
|
||||
return load_pair_history(pair=pair,
|
||||
timeframe=timeframe or self._config['timeframe'],
|
||||
datadir=self._config['datadir'],
|
||||
data_format=self._config.get('dataformat_ohlcv', 'json')
|
||||
)
|
||||
saved_pair = (pair, str(timeframe))
|
||||
if saved_pair not in self.__cached_pairs_backtesting:
|
||||
timerange = TimeRange.parse_timerange(None if self._config.get(
|
||||
'timerange') is None else str(self._config.get('timerange')))
|
||||
# Move informative start time respecting startup_candle_count
|
||||
timerange.subtract_start(
|
||||
timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
|
||||
)
|
||||
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
|
||||
pair=pair,
|
||||
timeframe=timeframe or self._config['timeframe'],
|
||||
datadir=self._config['datadir'],
|
||||
timerange=timerange,
|
||||
data_format=self._config.get('dataformat_ohlcv', 'json')
|
||||
)
|
||||
return self.__cached_pairs_backtesting[saved_pair].copy()
|
||||
|
||||
def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
|
||||
"""
|
||||
@@ -111,47 +101,27 @@ class DataProvider:
|
||||
|
||||
def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
|
||||
"""
|
||||
Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry),
|
||||
and the last 1000 candles (up to the time evaluated at this moment) in all other modes.
|
||||
:param pair: pair to get the data for
|
||||
:param timeframe: timeframe to get data for
|
||||
:return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe
|
||||
combination.
|
||||
Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
|
||||
"""
|
||||
if (pair, timeframe) in self.__cached_pairs:
|
||||
return self.__cached_pairs[(pair, timeframe)]
|
||||
pair_key = (pair, timeframe)
|
||||
if pair_key in self.__cached_pairs:
|
||||
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
|
||||
df, date = self.__cached_pairs[pair_key]
|
||||
else:
|
||||
df, date = self.__cached_pairs[pair_key]
|
||||
if self.__slice_index is not None:
|
||||
max_index = self.__slice_index
|
||||
df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index]
|
||||
return df, date
|
||||
else:
|
||||
|
||||
return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
|
||||
|
||||
def market(self, pair: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Return market data for the pair
|
||||
:param pair: Pair to get the data for
|
||||
:return: Market data dict from ccxt or None if market info is not available for the pair
|
||||
"""
|
||||
return self._exchange.markets.get(pair)
|
||||
|
||||
def ticker(self, pair: str):
|
||||
"""
|
||||
Return last ticker data from exchange
|
||||
:param pair: Pair to get the data for
|
||||
:return: Ticker dict from exchange or empty dict if ticker is not available for the pair
|
||||
"""
|
||||
try:
|
||||
return self._exchange.fetch_ticker(pair)
|
||||
except ExchangeError:
|
||||
return {}
|
||||
|
||||
def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
|
||||
"""
|
||||
Fetch latest l2 orderbook data
|
||||
Warning: Does a network request - so use with common sense.
|
||||
:param pair: pair to get the data for
|
||||
:param maximum: Maximum number of orderbook entries to query
|
||||
:return: dict including bids/asks with a total of `maximum` entries.
|
||||
"""
|
||||
return self._exchange.fetch_l2_order_book(pair, maximum)
|
||||
|
||||
@property
|
||||
def runmode(self) -> RunMode:
|
||||
"""
|
||||
@@ -170,6 +140,91 @@ class DataProvider:
|
||||
"""
|
||||
|
||||
if self._pairlists:
|
||||
return self._pairlists.whitelist
|
||||
return self._pairlists.whitelist.copy()
|
||||
else:
|
||||
raise OperationalException("Dataprovider was not initialized with a pairlist provider.")
|
||||
|
||||
def clear_cache(self):
|
||||
"""
|
||||
Clear pair dataframe cache.
|
||||
"""
|
||||
self.__cached_pairs = {}
|
||||
self.__cached_pairs_backtesting = {}
|
||||
self.__slice_index = 0
|
||||
|
||||
# Exchange functions
|
||||
|
||||
def refresh(self,
|
||||
pairlist: ListPairsWithTimeframes,
|
||||
helping_pairs: ListPairsWithTimeframes = None) -> None:
|
||||
"""
|
||||
Refresh data, called with each cycle
|
||||
"""
|
||||
if self._exchange is None:
|
||||
raise OperationalException(NO_EXCHANGE_EXCEPTION)
|
||||
if helping_pairs:
|
||||
self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
|
||||
else:
|
||||
self._exchange.refresh_latest_ohlcv(pairlist)
|
||||
|
||||
@property
|
||||
def available_pairs(self) -> ListPairsWithTimeframes:
|
||||
"""
|
||||
Return a list of tuples containing (pair, timeframe) for which data is currently cached.
|
||||
Should be whitelist + open trades.
|
||||
"""
|
||||
if self._exchange is None:
|
||||
raise OperationalException(NO_EXCHANGE_EXCEPTION)
|
||||
return list(self._exchange._klines.keys())
|
||||
|
||||
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
|
||||
"""
|
||||
Get candle (OHLCV) data for the given pair as DataFrame
|
||||
Please use the `available_pairs` method to verify which pairs are currently cached.
|
||||
:param pair: pair to get the data for
|
||||
:param timeframe: Timeframe to get data for
|
||||
:param copy: copy dataframe before returning if True.
|
||||
Use False only for read-only operations (where the dataframe is not modified)
|
||||
"""
|
||||
if self._exchange is None:
|
||||
raise OperationalException(NO_EXCHANGE_EXCEPTION)
|
||||
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
|
||||
return self._exchange.klines((pair, timeframe or self._config['timeframe']),
|
||||
copy=copy)
|
||||
else:
|
||||
return DataFrame()
|
||||
|
||||
def market(self, pair: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Return market data for the pair
|
||||
:param pair: Pair to get the data for
|
||||
:return: Market data dict from ccxt or None if market info is not available for the pair
|
||||
"""
|
||||
if self._exchange is None:
|
||||
raise OperationalException(NO_EXCHANGE_EXCEPTION)
|
||||
return self._exchange.markets.get(pair)
|
||||
|
||||
def ticker(self, pair: str):
|
||||
"""
|
||||
Return last ticker data from exchange
|
||||
:param pair: Pair to get the data for
|
||||
:return: Ticker dict from exchange or empty dict if ticker is not available for the pair
|
||||
"""
|
||||
if self._exchange is None:
|
||||
raise OperationalException(NO_EXCHANGE_EXCEPTION)
|
||||
try:
|
||||
return self._exchange.fetch_ticker(pair)
|
||||
except ExchangeError:
|
||||
return {}
|
||||
|
||||
def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
|
||||
"""
|
||||
Fetch latest l2 orderbook data
|
||||
Warning: Does a network request - so use with common sense.
|
||||
:param pair: pair to get the data for
|
||||
:param maximum: Maximum number of orderbook entries to query
|
||||
:return: dict including bids/asks with a total of `maximum` entries.
|
||||
"""
|
||||
if self._exchange is None:
|
||||
raise OperationalException(NO_EXCHANGE_EXCEPTION)
|
||||
return self._exchange.fetch_l2_order_book(pair, maximum)
|
||||
|
@@ -52,8 +52,8 @@ class HDF5DataHandler(IDataHandler):
|
||||
"""
|
||||
Store data in hdf5 file.
|
||||
:param pair: Pair - used to generate filename
|
||||
:timeframe: Timeframe - used to generate filename
|
||||
:data: Dataframe containing OHLCV data
|
||||
:param timeframe: Timeframe - used to generate filename
|
||||
:param data: Dataframe containing OHLCV data
|
||||
:return: None
|
||||
"""
|
||||
key = self._pair_ohlcv_key(pair, timeframe)
|
||||
@@ -89,7 +89,7 @@ class HDF5DataHandler(IDataHandler):
|
||||
if timerange.starttype == 'date':
|
||||
where.append(f"date >= Timestamp({timerange.startts * 1e9})")
|
||||
if timerange.stoptype == 'date':
|
||||
where.append(f"date < Timestamp({timerange.stopts * 1e9})")
|
||||
where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
|
||||
|
||||
pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
|
||||
|
||||
|
@@ -113,13 +113,15 @@ def refresh_data(datadir: Path,
|
||||
:param timeframe: Timeframe (e.g. "5m")
|
||||
:param pairs: List of pairs to load
|
||||
:param exchange: Exchange object
|
||||
:param data_format: dataformat to use
|
||||
:param timerange: Limit data to be loaded to this timerange
|
||||
"""
|
||||
data_handler = get_datahandler(datadir, data_format)
|
||||
for pair in pairs:
|
||||
_download_pair_history(pair=pair, timeframe=timeframe,
|
||||
datadir=datadir, timerange=timerange,
|
||||
exchange=exchange, data_handler=data_handler)
|
||||
for idx, pair in enumerate(pairs):
|
||||
process = f'{idx}/{len(pairs)}'
|
||||
_download_pair_history(pair=pair, process=process,
|
||||
timeframe=timeframe, datadir=datadir,
|
||||
timerange=timerange, exchange=exchange, data_handler=data_handler)
|
||||
|
||||
|
||||
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
|
||||
@@ -152,12 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
|
||||
return data, start_ms
|
||||
|
||||
|
||||
def _download_pair_history(datadir: Path,
|
||||
def _download_pair_history(pair: str, *,
|
||||
datadir: Path,
|
||||
exchange: Exchange,
|
||||
pair: str, *,
|
||||
timeframe: str = '5m',
|
||||
timerange: Optional[TimeRange] = None,
|
||||
data_handler: IDataHandler = None) -> bool:
|
||||
process: str = '',
|
||||
new_pairs_days: int = 30,
|
||||
data_handler: IDataHandler = None,
|
||||
timerange: Optional[TimeRange] = None) -> bool:
|
||||
"""
|
||||
Download latest candles from the exchange for the pair and timeframe passed in parameters
|
||||
The data is downloaded starting from the last correct data that
|
||||
@@ -175,7 +179,7 @@ def _download_pair_history(datadir: Path,
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
|
||||
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
|
||||
f'and store in {datadir}.'
|
||||
)
|
||||
|
||||
@@ -192,8 +196,9 @@ def _download_pair_history(datadir: Path,
|
||||
new_data = exchange.get_historic_ohlcv(pair=pair,
|
||||
timeframe=timeframe,
|
||||
since_ms=since_ms if since_ms else
|
||||
int(arrow.utcnow().shift(
|
||||
days=-30).float_timestamp) * 1000
|
||||
arrow.utcnow().shift(
|
||||
days=-new_pairs_days).int_timestamp * 1000,
|
||||
is_new_pair=data.empty
|
||||
)
|
||||
# TODO: Maybe move parsing to exchange class (?)
|
||||
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
|
||||
@@ -223,7 +228,8 @@ def _download_pair_history(datadir: Path,
|
||||
|
||||
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
|
||||
datadir: Path, timerange: Optional[TimeRange] = None,
|
||||
erase: bool = False, data_format: str = None) -> List[str]:
|
||||
new_pairs_days: int = 30, erase: bool = False,
|
||||
data_format: str = None) -> List[str]:
|
||||
"""
|
||||
Refresh stored ohlcv data for backtesting and hyperopt operations.
|
||||
Used by freqtrade download-data subcommand.
|
||||
@@ -231,7 +237,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
|
||||
"""
|
||||
pairs_not_available = []
|
||||
data_handler = get_datahandler(datadir, data_format)
|
||||
for pair in pairs:
|
||||
for idx, pair in enumerate(pairs, start=1):
|
||||
if pair not in exchange.markets:
|
||||
pairs_not_available.append(pair)
|
||||
logger.info(f"Skipping pair {pair}...")
|
||||
@@ -244,14 +250,17 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
|
||||
f'Deleting existing data for pair {pair}, interval {timeframe}.')
|
||||
|
||||
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
|
||||
_download_pair_history(datadir=datadir, exchange=exchange,
|
||||
pair=pair, timeframe=str(timeframe),
|
||||
timerange=timerange, data_handler=data_handler)
|
||||
process = f'{idx}/{len(pairs)}'
|
||||
_download_pair_history(pair=pair, process=process,
|
||||
datadir=datadir, exchange=exchange,
|
||||
timerange=timerange, data_handler=data_handler,
|
||||
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
|
||||
return pairs_not_available
|
||||
|
||||
|
||||
def _download_trades_history(exchange: Exchange,
|
||||
pair: str, *,
|
||||
new_pairs_days: int = 30,
|
||||
timerange: Optional[TimeRange] = None,
|
||||
data_handler: IDataHandler
|
||||
) -> bool:
|
||||
@@ -261,9 +270,13 @@ def _download_trades_history(exchange: Exchange,
|
||||
"""
|
||||
try:
|
||||
|
||||
since = timerange.startts * 1000 if \
|
||||
(timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
|
||||
days=-30).float_timestamp) * 1000
|
||||
until = None
|
||||
if (timerange and timerange.starttype == 'date'):
|
||||
since = timerange.startts * 1000
|
||||
if timerange.stoptype == 'date':
|
||||
until = timerange.stopts * 1000
|
||||
else:
|
||||
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
|
||||
|
||||
trades = data_handler.trades_load(pair)
|
||||
|
||||
@@ -291,6 +304,7 @@ def _download_trades_history(exchange: Exchange,
|
||||
# Default since_ms to 30 days if nothing is given
|
||||
new_trades = exchange.get_historic_trades(pair=pair,
|
||||
since=since,
|
||||
until=until,
|
||||
from_id=from_id,
|
||||
)
|
||||
trades.extend(new_trades[1])
|
||||
@@ -311,8 +325,8 @@ def _download_trades_history(exchange: Exchange,
|
||||
|
||||
|
||||
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
|
||||
timerange: TimeRange, erase: bool = False,
|
||||
data_format: str = 'jsongz') -> List[str]:
|
||||
timerange: TimeRange, new_pairs_days: int = 30,
|
||||
erase: bool = False, data_format: str = 'jsongz') -> List[str]:
|
||||
"""
|
||||
Refresh stored trades data for backtesting and hyperopt operations.
|
||||
Used by freqtrade download-data subcommand.
|
||||
@@ -333,6 +347,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
|
||||
logger.info(f'Downloading trades for pair {pair}.')
|
||||
_download_trades_history(exchange=exchange,
|
||||
pair=pair,
|
||||
new_pairs_days=new_pairs_days,
|
||||
timerange=timerange,
|
||||
data_handler=data_handler)
|
||||
return pairs_not_available
|
||||
@@ -362,7 +377,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
|
||||
logger.exception(f'Could not convert {pair} to OHLCV.')
|
||||
|
||||
|
||||
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
|
||||
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
|
||||
"""
|
||||
Get the maximum common timerange for the given backtest data.
|
||||
|
||||
@@ -370,7 +385,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
|
||||
:return: tuple containing min_date, max_date
|
||||
"""
|
||||
timeranges = [
|
||||
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
|
||||
(frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime())
|
||||
for frame in data.values()
|
||||
]
|
||||
return (min(timeranges, key=operator.itemgetter(0))[0],
|
||||
|
@@ -49,8 +49,8 @@ class IDataHandler(ABC):
|
||||
"""
|
||||
Store ohlcv data.
|
||||
:param pair: Pair - used to generate filename
|
||||
:timeframe: Timeframe - used to generate filename
|
||||
:data: Dataframe containing OHLCV data
|
||||
:param timeframe: Timeframe - used to generate filename
|
||||
:param data: Dataframe containing OHLCV data
|
||||
:return: None
|
||||
"""
|
||||
|
||||
@@ -245,8 +245,8 @@ def get_datahandler(datadir: Path, data_format: str = None,
|
||||
data_handler: IDataHandler = None) -> IDataHandler:
|
||||
"""
|
||||
:param datadir: Folder to save data
|
||||
:data_format: dataformat to use
|
||||
:data_handler: returns this datahandler if it exists or initializes a new one
|
||||
:param data_format: dataformat to use
|
||||
:param data_handler: returns this datahandler if it exists or initializes a new one
|
||||
"""
|
||||
|
||||
if not data_handler:
|
||||
|
@@ -55,14 +55,14 @@ class JsonDataHandler(IDataHandler):
|
||||
format looks as follows:
|
||||
[[<date>,<open>,<high>,<low>,<close>]]
|
||||
:param pair: Pair - used to generate filename
|
||||
:timeframe: Timeframe - used to generate filename
|
||||
:data: Dataframe containing OHLCV data
|
||||
:param timeframe: Timeframe - used to generate filename
|
||||
:param data: Dataframe containing OHLCV data
|
||||
:return: None
|
||||
"""
|
||||
filename = self._pair_data_filename(self._datadir, pair, timeframe)
|
||||
_data = data.copy()
|
||||
# Convert date to int
|
||||
_data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
|
||||
_data['date'] = _data['date'].view(np.int64) // 1000 // 1000
|
||||
|
||||
# Reset index, select only appropriate columns and save as json
|
||||
_data.reset_index(drop=True).loc[:, self._columns].to_json(
|
||||
|
@@ -1,6 +1,8 @@
|
||||
# pragma pylint: disable=W0603
|
||||
""" Edge positioning package """
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, NamedTuple
|
||||
|
||||
import arrow
|
||||
@@ -11,9 +13,11 @@ from pandas import DataFrame
|
||||
from freqtrade.configuration import TimeRange
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
|
||||
from freqtrade.data.history import get_timerange, load_data, refresh_data
|
||||
from freqtrade.enums import RunMode, SellType
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange.exchange import timeframe_to_seconds
|
||||
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
|
||||
from freqtrade.strategy.interface import SellType
|
||||
from freqtrade.strategy.interface import IStrategy
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -45,7 +49,7 @@ class Edge:
|
||||
|
||||
self.config = config
|
||||
self.exchange = exchange
|
||||
self.strategy = strategy
|
||||
self.strategy: IStrategy = strategy
|
||||
|
||||
self.edge_config = self.config.get('edge', {})
|
||||
self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs
|
||||
@@ -81,12 +85,16 @@ class Edge:
|
||||
if config.get('fee'):
|
||||
self.fee = config['fee']
|
||||
else:
|
||||
self.fee = self.exchange.get_fee(symbol=expand_pairlist(
|
||||
self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
|
||||
try:
|
||||
self.fee = self.exchange.get_fee(symbol=expand_pairlist(
|
||||
self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
|
||||
except IndexError:
|
||||
self.fee = None
|
||||
|
||||
def calculate(self, pairs: List[str]) -> bool:
|
||||
if self.fee is None and pairs:
|
||||
self.fee = self.exchange.get_fee(pairs[0])
|
||||
|
||||
def calculate(self) -> bool:
|
||||
pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
|
||||
list(self.exchange.markets))
|
||||
heartbeat = self.edge_config.get('process_throttle_secs')
|
||||
|
||||
if (self._last_updated > 0) and (
|
||||
@@ -98,14 +106,33 @@ class Edge:
|
||||
logger.info('Using local backtesting data (using whitelist in given config) ...')
|
||||
|
||||
if self._refresh_pairs:
|
||||
timerange_startup = deepcopy(self._timerange)
|
||||
timerange_startup.subtract_start(timeframe_to_seconds(
|
||||
self.strategy.timeframe) * self.strategy.startup_candle_count)
|
||||
refresh_data(
|
||||
datadir=self.config['datadir'],
|
||||
pairs=pairs,
|
||||
exchange=self.exchange,
|
||||
timeframe=self.strategy.timeframe,
|
||||
timerange=self._timerange,
|
||||
timerange=timerange_startup,
|
||||
data_format=self.config.get('dataformat_ohlcv', 'json'),
|
||||
)
|
||||
# Download informative pairs too
|
||||
res = defaultdict(list)
|
||||
for p, t in self.strategy.gather_informative_pairs():
|
||||
res[t].append(p)
|
||||
for timeframe, inf_pairs in res.items():
|
||||
timerange_startup = deepcopy(self._timerange)
|
||||
timerange_startup.subtract_start(timeframe_to_seconds(
|
||||
timeframe) * self.strategy.startup_candle_count)
|
||||
refresh_data(
|
||||
datadir=self.config['datadir'],
|
||||
pairs=inf_pairs,
|
||||
exchange=self.exchange,
|
||||
timeframe=timeframe,
|
||||
timerange=timerange_startup,
|
||||
data_format=self.config.get('dataformat_ohlcv', 'json'),
|
||||
)
|
||||
|
||||
data = load_data(
|
||||
datadir=self.config['datadir'],
|
||||
@@ -121,8 +148,11 @@ class Edge:
|
||||
self._cached_pairs = {}
|
||||
logger.critical("No data found. Edge is stopped ...")
|
||||
return False
|
||||
|
||||
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
|
||||
# Fake run-mode to Edge
|
||||
prior_rm = self.config['runmode']
|
||||
self.config['runmode'] = RunMode.EDGE
|
||||
preprocessed = self.strategy.advise_all_indicators(data)
|
||||
self.config['runmode'] = prior_rm
|
||||
|
||||
# Print timeframe
|
||||
min_date, max_date = get_timerange(preprocessed)
|
||||
@@ -179,7 +209,7 @@ class Edge:
|
||||
if pair in self._cached_pairs:
|
||||
return self._cached_pairs[pair].stoploss
|
||||
else:
|
||||
logger.warning('tried to access stoploss of a non-existing pair, '
|
||||
logger.warning(f'Tried to access stoploss of non-existing pair {pair}, '
|
||||
'strategy stoploss is returned instead.')
|
||||
return self.strategy.stoploss
|
||||
|
||||
@@ -201,23 +231,23 @@ class Edge:
|
||||
'Minimum expectancy and minimum winrate are met only for %s,'
|
||||
' so other pairs are filtered out.',
|
||||
self._final_pairs
|
||||
)
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
'Edge removed all pairs as no pair with minimum expectancy '
|
||||
'and minimum winrate was found !'
|
||||
)
|
||||
)
|
||||
|
||||
return self._final_pairs
|
||||
|
||||
def accepted_pairs(self) -> list:
|
||||
def accepted_pairs(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
return a list of accepted pairs along with their winrate, expectancy and stoploss
|
||||
"""
|
||||
final = []
|
||||
for pair, info in self._cached_pairs.items():
|
||||
if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
|
||||
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
|
||||
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
|
||||
final.append({
|
||||
'Pair': pair,
|
||||
'Winrate': info.winrate,
|
||||
@@ -271,7 +301,7 @@ class Edge:
|
||||
def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
|
||||
"""
|
||||
This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
|
||||
The calulation will be done per pair and per strategy.
|
||||
The calculation will be done per pair and per strategy.
|
||||
"""
|
||||
# Removing pairs having less than min_trades_number
|
||||
min_trades_number = self.edge_config.get('min_trade_number', 10)
|
||||
|
7
freqtrade/enums/__init__.py
Normal file
7
freqtrade/enums/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
# flake8: noqa: F401
|
||||
from freqtrade.enums.backteststate import BacktestState
|
||||
from freqtrade.enums.rpcmessagetype import RPCMessageType
|
||||
from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
|
||||
from freqtrade.enums.selltype import SellType
|
||||
from freqtrade.enums.signaltype import SignalTagType, SignalType
|
||||
from freqtrade.enums.state import State
|
15
freqtrade/enums/backteststate.py
Normal file
15
freqtrade/enums/backteststate.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class BacktestState(Enum):
|
||||
"""
|
||||
Bot application states
|
||||
"""
|
||||
STARTUP = 1
|
||||
DATALOAD = 2
|
||||
ANALYZE = 3
|
||||
CONVERT = 4
|
||||
BACKTEST = 5
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name.lower()}"
|
21
freqtrade/enums/rpcmessagetype.py
Normal file
21
freqtrade/enums/rpcmessagetype.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class RPCMessageType(Enum):
|
||||
STATUS = 'status'
|
||||
WARNING = 'warning'
|
||||
STARTUP = 'startup'
|
||||
BUY = 'buy'
|
||||
BUY_FILL = 'buy_fill'
|
||||
BUY_CANCEL = 'buy_cancel'
|
||||
SELL = 'sell'
|
||||
SELL_FILL = 'sell_fill'
|
||||
SELL_CANCEL = 'sell_cancel'
|
||||
PROTECTION_TRIGGER = 'protection_trigger'
|
||||
PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global'
|
||||
|
||||
def __repr__(self):
|
||||
return self.value
|
||||
|
||||
def __str__(self):
|
||||
return self.value
|
@@ -1,23 +1,6 @@
|
||||
# pragma pylint: disable=too-few-public-methods
|
||||
|
||||
"""
|
||||
Bot state constant
|
||||
"""
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class State(Enum):
|
||||
"""
|
||||
Bot application states
|
||||
"""
|
||||
RUNNING = 1
|
||||
STOPPED = 2
|
||||
RELOAD_CONFIG = 3
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name.lower()}"
|
||||
|
||||
|
||||
class RunMode(Enum):
|
||||
"""
|
||||
Bot running mode (backtest, hyperopt, ...)
|
||||
@@ -31,6 +14,7 @@ class RunMode(Enum):
|
||||
UTIL_EXCHANGE = "util_exchange"
|
||||
UTIL_NO_EXCHANGE = "util_no_exchange"
|
||||
PLOT = "plot"
|
||||
WEBSERVER = "webserver"
|
||||
OTHER = "other"
|
||||
|
||||
|
20
freqtrade/enums/selltype.py
Normal file
20
freqtrade/enums/selltype.py
Normal file
@@ -0,0 +1,20 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class SellType(Enum):
|
||||
"""
|
||||
Enum to distinguish between sell reasons
|
||||
"""
|
||||
ROI = "roi"
|
||||
STOP_LOSS = "stop_loss"
|
||||
STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange"
|
||||
TRAILING_STOP_LOSS = "trailing_stop_loss"
|
||||
SELL_SIGNAL = "sell_signal"
|
||||
FORCE_SELL = "force_sell"
|
||||
EMERGENCY_SELL = "emergency_sell"
|
||||
CUSTOM_SELL = "custom_sell"
|
||||
NONE = ""
|
||||
|
||||
def __str__(self):
|
||||
# explicitly convert to String to help with exporting data.
|
||||
return self.value
|
16
freqtrade/enums/signaltype.py
Normal file
16
freqtrade/enums/signaltype.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class SignalType(Enum):
|
||||
"""
|
||||
Enum to distinguish between buy and sell signals
|
||||
"""
|
||||
BUY = "buy"
|
||||
SELL = "sell"
|
||||
|
||||
|
||||
class SignalTagType(Enum):
|
||||
"""
|
||||
Enum for signal columns
|
||||
"""
|
||||
BUY_TAG = "buy_tag"
|
13
freqtrade/enums/state.py
Normal file
13
freqtrade/enums/state.py
Normal file
@@ -0,0 +1,13 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class State(Enum):
|
||||
"""
|
||||
Bot application states
|
||||
"""
|
||||
RUNNING = 1
|
||||
STOPPED = 2
|
||||
RELOAD_CONFIG = 3
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name.lower()}"
|
@@ -47,7 +47,7 @@ class InvalidOrderException(ExchangeError):
|
||||
class RetryableOrderError(InvalidOrderException):
|
||||
"""
|
||||
This is returned when the order is not found.
|
||||
This Error will be repeated with increasing backof (in line with DDosError).
|
||||
This Error will be repeated with increasing backoff (in line with DDosError).
|
||||
"""
|
||||
|
||||
|
||||
@@ -75,6 +75,6 @@ class DDosProtection(TemporaryError):
|
||||
|
||||
class StrategyError(FreqtradeException):
|
||||
"""
|
||||
Errors with custom user-code deteced.
|
||||
Errors with custom user-code detected.
|
||||
Usually caused by errors in the strategy.
|
||||
"""
|
||||
|
@@ -1,17 +1,21 @@
|
||||
# flake8: noqa: F401
|
||||
# isort: off
|
||||
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS
|
||||
from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS
|
||||
from freqtrade.exchange.exchange import Exchange
|
||||
# isort: on
|
||||
from freqtrade.exchange.bibox import Bibox
|
||||
from freqtrade.exchange.binance import Binance
|
||||
from freqtrade.exchange.bittrex import Bittrex
|
||||
from freqtrade.exchange.bybit import Bybit
|
||||
from freqtrade.exchange.coinbasepro import Coinbasepro
|
||||
from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
|
||||
get_exchange_bad_reason, is_exchange_bad,
|
||||
is_exchange_known_ccxt, is_exchange_officially_supported,
|
||||
market_is_active, timeframe_to_minutes, timeframe_to_msecs,
|
||||
timeframe_to_next_date, timeframe_to_prev_date,
|
||||
timeframe_to_seconds)
|
||||
timeframe_to_seconds, validate_exchange,
|
||||
validate_exchanges)
|
||||
from freqtrade.exchange.ftx import Ftx
|
||||
from freqtrade.exchange.gateio import Gateio
|
||||
from freqtrade.exchange.hitbtc import Hitbtc
|
||||
from freqtrade.exchange.kraken import Kraken
|
||||
from freqtrade.exchange.kucoin import Kucoin
|
||||
|
@@ -1,7 +1,8 @@
|
||||
""" Binance exchange subclass """
|
||||
import logging
|
||||
from typing import Dict
|
||||
from typing import Dict, List
|
||||
|
||||
import arrow
|
||||
import ccxt
|
||||
|
||||
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
|
||||
@@ -18,6 +19,7 @@ class Binance(Exchange):
|
||||
_ft_has: Dict = {
|
||||
"stoploss_on_exchange": True,
|
||||
"order_time_in_force": ['gtc', 'fok', 'ioc'],
|
||||
"time_in_force_parameter": "timeInForce",
|
||||
"ohlcv_candle_limit": 1000,
|
||||
"trades_pagination": "id",
|
||||
"trades_pagination_arg": "fromId",
|
||||
@@ -52,7 +54,7 @@ class Binance(Exchange):
|
||||
'In stoploss limit order, stop price should be more than limit price')
|
||||
|
||||
if self._config['dry_run']:
|
||||
dry_order = self.dry_run_order(
|
||||
dry_order = self.create_dry_run_order(
|
||||
pair, ordertype, "sell", amount, stop_price)
|
||||
return dry_order
|
||||
|
||||
@@ -68,6 +70,7 @@ class Binance(Exchange):
|
||||
amount=amount, price=rate, params=params)
|
||||
logger.info('stoploss limit order added for %s. '
|
||||
'stop price: %s. limit: %s', pair, stop_price, rate)
|
||||
self._log_exchange_response('create_stoploss_order', order)
|
||||
return order
|
||||
except ccxt.InsufficientFunds as e:
|
||||
raise InsufficientFundsError(
|
||||
@@ -88,3 +91,20 @@ class Binance(Exchange):
|
||||
f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
|
||||
except ccxt.BaseError as e:
|
||||
raise OperationalException(e) from e
|
||||
|
||||
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
|
||||
since_ms: int, is_new_pair: bool
|
||||
) -> List:
|
||||
"""
|
||||
Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
|
||||
Does not work for other exchanges, which don't return the earliest data when called with "0"
|
||||
"""
|
||||
if is_new_pair:
|
||||
x = await self._async_get_candle_history(pair, timeframe, 0)
|
||||
if x and x[2] and x[2][0] and x[2][0][0] > since_ms:
|
||||
# Set starting date to first available candle.
|
||||
since_ms = x[2][0][0]
|
||||
logger.info(f"Candle-data for {pair} available starting with "
|
||||
f"{arrow.get(since_ms // 1000).isoformat()}.")
|
||||
return await super()._async_get_historic_ohlcv(
|
||||
pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair)
|
||||
|
@@ -12,10 +12,6 @@ class Bittrex(Exchange):
|
||||
"""
|
||||
Bittrex exchange class. Contains adjustments needed for Freqtrade to work
|
||||
with this exchange.
|
||||
|
||||
Please note that this exchange is not included in the list of exchanges
|
||||
officially supported by the Freqtrade development team. So some features
|
||||
may still not work as expected.
|
||||
"""
|
||||
|
||||
_ft_has: Dict = {
|
||||
|
@@ -18,7 +18,6 @@ class Bybit(Exchange):
|
||||
may still not work as expected.
|
||||
"""
|
||||
|
||||
# fetchCurrencies API point requires authentication for Bybit,
|
||||
_ft_has: Dict = {
|
||||
"ohlcv_candle_limit": 200,
|
||||
}
|
||||
|
23
freqtrade/exchange/coinbasepro.py
Normal file
23
freqtrade/exchange/coinbasepro.py
Normal file
@@ -0,0 +1,23 @@
|
||||
""" CoinbasePro exchange subclass """
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from freqtrade.exchange import Exchange
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Coinbasepro(Exchange):
|
||||
"""
|
||||
CoinbasePro exchange class. Contains adjustments needed for Freqtrade to work
|
||||
with this exchange.
|
||||
|
||||
Please note that this exchange is not included in the list of exchanges
|
||||
officially supported by the Freqtrade development team. So some features
|
||||
may still not work as expected.
|
||||
"""
|
||||
|
||||
_ft_has: Dict = {
|
||||
"ohlcv_candle_limit": 300,
|
||||
}
|
@@ -18,78 +18,8 @@ BAD_EXCHANGES = {
|
||||
"bitmex": "Various reasons.",
|
||||
"bitstamp": "Does not provide history. "
|
||||
"Details in https://github.com/freqtrade/freqtrade/issues/1983",
|
||||
"hitbtc": "This API cannot be used with Freqtrade. "
|
||||
"Use `hitbtc2` exchange id to access this exchange.",
|
||||
"phemex": "Does not provide history. ",
|
||||
"poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
|
||||
**dict.fromkeys([
|
||||
'adara',
|
||||
'anxpro',
|
||||
'bigone',
|
||||
'coinbase',
|
||||
'coinexchange',
|
||||
'coinmarketcap',
|
||||
'lykke',
|
||||
'xbtce',
|
||||
], "Does not provide timeframes. ccxt fetchOHLCV: False"),
|
||||
**dict.fromkeys([
|
||||
'bcex',
|
||||
'bit2c',
|
||||
'bitbay',
|
||||
'bitflyer',
|
||||
'bitforex',
|
||||
'bithumb',
|
||||
'bitso',
|
||||
'bitstamp1',
|
||||
'bl3p',
|
||||
'braziliex',
|
||||
'btcbox',
|
||||
'btcchina',
|
||||
'btctradeim',
|
||||
'btctradeua',
|
||||
'bxinth',
|
||||
'chilebit',
|
||||
'coincheck',
|
||||
'coinegg',
|
||||
'coinfalcon',
|
||||
'coinfloor',
|
||||
'coingi',
|
||||
'coinmate',
|
||||
'coinone',
|
||||
'coinspot',
|
||||
'coolcoin',
|
||||
'crypton',
|
||||
'deribit',
|
||||
'exmo',
|
||||
'exx',
|
||||
'flowbtc',
|
||||
'foxbit',
|
||||
'fybse',
|
||||
# 'hitbtc',
|
||||
'ice3x',
|
||||
'independentreserve',
|
||||
'indodax',
|
||||
'itbit',
|
||||
'lakebtc',
|
||||
'latoken',
|
||||
'liquid',
|
||||
'livecoin',
|
||||
'luno',
|
||||
'mixcoins',
|
||||
'negociecoins',
|
||||
'nova',
|
||||
'paymium',
|
||||
'southxchange',
|
||||
'stronghold',
|
||||
'surbitcoin',
|
||||
'therock',
|
||||
'tidex',
|
||||
'vaultoro',
|
||||
'vbtc',
|
||||
'virwox',
|
||||
'yobit',
|
||||
'zaif',
|
||||
], "Does not provide timeframes. ccxt fetchOHLCV: emulated"),
|
||||
}
|
||||
|
||||
MAP_EXCHANGE_CHILDCLASS = {
|
||||
@@ -98,6 +28,42 @@ MAP_EXCHANGE_CHILDCLASS = {
|
||||
}
|
||||
|
||||
|
||||
EXCHANGE_HAS_REQUIRED = [
|
||||
# Required / private
|
||||
'fetchOrder',
|
||||
'cancelOrder',
|
||||
'createOrder',
|
||||
# 'createLimitOrder', 'createMarketOrder',
|
||||
'fetchBalance',
|
||||
|
||||
# Public endpoints
|
||||
'loadMarkets',
|
||||
'fetchOHLCV',
|
||||
]
|
||||
|
||||
EXCHANGE_HAS_OPTIONAL = [
|
||||
# Private
|
||||
'fetchMyTrades', # Trades for order - fee detection
|
||||
# Public
|
||||
'fetchOrderBook', 'fetchL2OrderBook', 'fetchTicker', # OR for pricing
|
||||
'fetchTickers', # For volumepairlist?
|
||||
'fetchTrades', # Downloading trades data
|
||||
]
|
||||
|
||||
|
||||
def remove_credentials(config) -> None:
|
||||
"""
|
||||
Removes exchange keys from the configuration and specifies dry-run
|
||||
Used for backtesting / hyperopt / edge and utils.
|
||||
Modifies the input dict!
|
||||
"""
|
||||
if config.get('dry_run', False):
|
||||
config['exchange']['key'] = ''
|
||||
config['exchange']['secret'] = ''
|
||||
config['exchange']['password'] = ''
|
||||
config['exchange']['uid'] = ''
|
||||
|
||||
|
||||
def calculate_backoff(retrycount, max_retries):
|
||||
"""
|
||||
Calculate backoff
|
||||
@@ -140,7 +106,7 @@ def retrier(_func=None, retries=API_RETRY_COUNT):
|
||||
logger.warning('retrying %s() still for %s times', f.__name__, count)
|
||||
count -= 1
|
||||
kwargs.update({'count': count})
|
||||
if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError):
|
||||
if isinstance(ex, (DDosProtection, RetryableOrderError)):
|
||||
# increasing backoff
|
||||
backoff_delay = calculate_backoff(count + 1, retries)
|
||||
logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali
|
||||
OperationalException, TemporaryError)
|
||||
from freqtrade.exchange import Exchange
|
||||
from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
|
||||
from freqtrade.misc import safe_value_fallback2
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -53,7 +54,7 @@ class Ftx(Exchange):
|
||||
stop_price = self.price_to_precision(pair, stop_price)
|
||||
|
||||
if self._config['dry_run']:
|
||||
dry_order = self.dry_run_order(
|
||||
dry_order = self.create_dry_run_order(
|
||||
pair, ordertype, "sell", amount, stop_price)
|
||||
return dry_order
|
||||
|
||||
@@ -63,10 +64,12 @@ class Ftx(Exchange):
|
||||
# set orderPrice to place limit order, otherwise it's a market order
|
||||
params['orderPrice'] = limit_rate
|
||||
|
||||
params['stopPrice'] = stop_price
|
||||
amount = self.amount_to_precision(pair, amount)
|
||||
|
||||
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
|
||||
amount=amount, price=stop_price, params=params)
|
||||
amount=amount, params=params)
|
||||
self._log_exchange_response('create_stoploss_order', order)
|
||||
logger.info('stoploss order added for %s. '
|
||||
'stop price: %s.', pair, stop_price)
|
||||
return order
|
||||
@@ -91,18 +94,26 @@ class Ftx(Exchange):
|
||||
@retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
|
||||
def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict:
|
||||
if self._config['dry_run']:
|
||||
try:
|
||||
order = self._dry_run_open_orders[order_id]
|
||||
return order
|
||||
except KeyError as e:
|
||||
# Gracefully handle errors with dry-run orders.
|
||||
raise InvalidOrderException(
|
||||
f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e
|
||||
return self.fetch_dry_run_order(order_id)
|
||||
|
||||
try:
|
||||
orders = self._api.fetch_orders(pair, None, params={'type': 'stop'})
|
||||
|
||||
order = [order for order in orders if order['id'] == order_id]
|
||||
self._log_exchange_response('fetch_stoploss_order', order)
|
||||
if len(order) == 1:
|
||||
if order[0].get('status') == 'closed':
|
||||
# Trigger order was triggered ...
|
||||
real_order_id = order[0].get('info', {}).get('orderId')
|
||||
|
||||
order1 = self._api.fetch_order(real_order_id, pair)
|
||||
self._log_exchange_response('fetch_stoploss_order1', order1)
|
||||
# Fake type to stop - as this was really a stop order.
|
||||
order1['id_stop'] = order1['id']
|
||||
order1['id'] = order_id
|
||||
order1['type'] = 'stop'
|
||||
order1['status_stop'] = 'triggered'
|
||||
return order1
|
||||
return order[0]
|
||||
else:
|
||||
raise InvalidOrderException(f"Could not get stoploss order for id {order_id}")
|
||||
@@ -123,7 +134,9 @@ class Ftx(Exchange):
|
||||
if self._config['dry_run']:
|
||||
return {}
|
||||
try:
|
||||
return self._api.cancel_order(order_id, pair, params={'type': 'stop'})
|
||||
order = self._api.cancel_order(order_id, pair, params={'type': 'stop'})
|
||||
self._log_exchange_response('cancel_stoploss_order', order)
|
||||
return order
|
||||
except ccxt.InvalidOrder as e:
|
||||
raise InvalidOrderException(
|
||||
f'Could not cancel order. Message: {e}') from e
|
||||
@@ -134,3 +147,8 @@ class Ftx(Exchange):
|
||||
f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
|
||||
except ccxt.BaseError as e:
|
||||
raise OperationalException(e) from e
|
||||
|
||||
def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
|
||||
if order['type'] == 'stop':
|
||||
return safe_value_fallback2(order, order, 'id_stop', 'id')
|
||||
return order['id']
|
||||
|
33
freqtrade/exchange/gateio.py
Normal file
33
freqtrade/exchange/gateio.py
Normal file
@@ -0,0 +1,33 @@
|
||||
""" Gate.io exchange subclass """
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import Exchange
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Gateio(Exchange):
|
||||
"""
|
||||
Gate.io exchange class. Contains adjustments needed for Freqtrade to work
|
||||
with this exchange.
|
||||
|
||||
Please note that this exchange is not included in the list of exchanges
|
||||
officially supported by the Freqtrade development team. So some features
|
||||
may still not work as expected.
|
||||
"""
|
||||
|
||||
_ft_has: Dict = {
|
||||
"ohlcv_candle_limit": 1000,
|
||||
}
|
||||
|
||||
_headers = {'X-Gate-Channel-Id': 'freqtrade'}
|
||||
|
||||
def validate_ordertypes(self, order_types: Dict) -> None:
|
||||
super().validate_ordertypes(order_types)
|
||||
|
||||
if any(v == 'market' for k, v in order_types.items()):
|
||||
raise OperationalException(
|
||||
f'Exchange {self.name} does not support market orders.')
|
23
freqtrade/exchange/hitbtc.py
Normal file
23
freqtrade/exchange/hitbtc.py
Normal file
@@ -0,0 +1,23 @@
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from freqtrade.exchange import Exchange
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Hitbtc(Exchange):
|
||||
"""
|
||||
Hitbtc exchange class. Contains adjustments needed for Freqtrade to work
|
||||
with this exchange.
|
||||
|
||||
Please note that this exchange is not included in the list of exchanges
|
||||
officially supported by the Freqtrade development team. So some features
|
||||
may still not work as expected.
|
||||
"""
|
||||
|
||||
_ft_has: Dict = {
|
||||
"ohlcv_candle_limit": 1000,
|
||||
"ohlcv_params": {"sort": "DESC"}
|
||||
}
|
@@ -49,10 +49,12 @@ class Kraken(Exchange):
|
||||
orders = self._api.fetch_open_orders()
|
||||
order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1],
|
||||
x["remaining"] if x["side"] == "sell" else x["remaining"] * x["price"],
|
||||
# Don't remove the below comment, this can be important for debuggung
|
||||
# Don't remove the below comment, this can be important for debugging
|
||||
# x["side"], x["amount"],
|
||||
) for x in orders]
|
||||
for bal in balances:
|
||||
if not isinstance(balances[bal], dict):
|
||||
continue
|
||||
balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal)
|
||||
balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used']
|
||||
|
||||
@@ -92,7 +94,7 @@ class Kraken(Exchange):
|
||||
stop_price = self.price_to_precision(pair, stop_price)
|
||||
|
||||
if self._config['dry_run']:
|
||||
dry_order = self.dry_run_order(
|
||||
dry_order = self.create_dry_run_order(
|
||||
pair, ordertype, "sell", amount, stop_price)
|
||||
return dry_order
|
||||
|
||||
@@ -101,6 +103,7 @@ class Kraken(Exchange):
|
||||
|
||||
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
|
||||
amount=amount, price=stop_price, params=params)
|
||||
self._log_exchange_response('create_stoploss_order', order)
|
||||
logger.info('stoploss order added for %s. '
|
||||
'stop price: %s.', pair, stop_price)
|
||||
return order
|
||||
|
26
freqtrade/exchange/kucoin.py
Normal file
26
freqtrade/exchange/kucoin.py
Normal file
@@ -0,0 +1,26 @@
|
||||
""" Kucoin exchange subclass """
|
||||
import logging
|
||||
from typing import Dict
|
||||
|
||||
from freqtrade.exchange import Exchange
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Kucoin(Exchange):
|
||||
"""
|
||||
Kucoin exchange class. Contains adjustments needed for Freqtrade to work
|
||||
with this exchange.
|
||||
|
||||
Please note that this exchange is not included in the list of exchanges
|
||||
officially supported by the Freqtrade development team. So some features
|
||||
may still not work as expected.
|
||||
"""
|
||||
|
||||
_ft_has: Dict = {
|
||||
"l2_limit_range": [20, 100],
|
||||
"l2_limit_range_required": False,
|
||||
"order_time_in_force": ['gtc', 'fok', 'ioc'],
|
||||
"time_in_force_parameter": "timeInForce",
|
||||
}
|
File diff suppressed because it is too large
Load Diff
@@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None:
|
||||
# syslog config. The messages should be equal for this.
|
||||
handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
|
||||
logging.root.addHandler(handler_sl)
|
||||
elif s[0] == 'journald':
|
||||
elif s[0] == 'journald': # pragma: no cover
|
||||
try:
|
||||
from systemd.journal import JournaldLogHandler
|
||||
except ImportError:
|
||||
|
@@ -9,7 +9,7 @@ from typing import Any, List
|
||||
|
||||
|
||||
# check min. python version
|
||||
if sys.version_info < (3, 7):
|
||||
if sys.version_info < (3, 7): # pragma: no cover
|
||||
sys.exit("Freqtrade requires Python version >= 3.7")
|
||||
|
||||
from freqtrade.commands import Arguments
|
||||
@@ -44,9 +44,9 @@ def main(sysargv: List[str] = None) -> None:
|
||||
"as `freqtrade trade [options...]`.\n"
|
||||
"To see the full list of options available, please use "
|
||||
"`freqtrade --help` or `freqtrade <command> --help`."
|
||||
)
|
||||
)
|
||||
|
||||
except SystemExit as e:
|
||||
except SystemExit as e: # pragma: no cover
|
||||
return_code = e
|
||||
except KeyboardInterrupt:
|
||||
logger.info('SIGINT received, aborting ...')
|
||||
@@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None:
|
||||
sys.exit(return_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == '__main__': # pragma: no cover
|
||||
main()
|
||||
|
@@ -6,8 +6,9 @@ import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Iterator, List
|
||||
from typing.io import IO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import rapidjson
|
||||
|
||||
@@ -56,6 +57,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
|
||||
"""
|
||||
Dump JSON data into a file
|
||||
:param filename: file to create
|
||||
:param is_zip: if file should be zip
|
||||
:param data: JSON Data to save
|
||||
:return:
|
||||
"""
|
||||
@@ -81,7 +83,7 @@ def json_load(datafile: IO) -> Any:
|
||||
"""
|
||||
load data with rapidjson
|
||||
Use this to have a consistent experience,
|
||||
sete number_mode to "NM_NATIVE" for greatest speed
|
||||
set number_mode to "NM_NATIVE" for greatest speed
|
||||
"""
|
||||
return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE)
|
||||
|
||||
@@ -202,3 +204,27 @@ def render_template_with_fallback(templatefile: str, templatefallbackfile: str,
|
||||
return render_template(templatefile, arguments)
|
||||
except TemplateNotFound:
|
||||
return render_template(templatefallbackfile, arguments)
|
||||
|
||||
|
||||
def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
|
||||
"""
|
||||
Split lst into chunks of the size n.
|
||||
:param lst: list to split into chunks
|
||||
:param n: number of max elements per chunk
|
||||
:return: None
|
||||
"""
|
||||
for chunk in range(0, len(lst), n):
|
||||
yield (lst[chunk:chunk + n])
|
||||
|
||||
|
||||
def parse_db_uri_for_logging(uri: str):
|
||||
"""
|
||||
Helper method to parse the DB URI and return the same DB URI with the password censored
|
||||
if it contains it. Otherwise, return the DB URI unchanged
|
||||
:param uri: DB URI to parse for logging
|
||||
"""
|
||||
parsed_db_uri = urlparse(uri)
|
||||
if not parsed_db_uri.netloc: # No need for censoring as no password was provided
|
||||
return uri
|
||||
pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
|
||||
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
|
||||
|
@@ -11,22 +11,24 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
|
||||
from freqtrade.configuration import TimeRange, validate_config_consistency
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT
|
||||
from freqtrade.data import history
|
||||
from freqtrade.data.btanalysis import trade_list_to_dataframe
|
||||
from freqtrade.data.converter import trim_dataframe
|
||||
from freqtrade.data.converter import trim_dataframe, trim_dataframes
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.enums import BacktestState, SellType
|
||||
from freqtrade.exceptions import DependencyException, OperationalException
|
||||
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
|
||||
from freqtrade.mixins import LoggingMixin
|
||||
from freqtrade.optimize.bt_progress import BTProgress
|
||||
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
|
||||
store_backtest_stats)
|
||||
from freqtrade.persistence import LocalTrade, PairLocks, Trade
|
||||
from freqtrade.plugins.pairlistmanager import PairListManager
|
||||
from freqtrade.plugins.protectionmanager import ProtectionManager
|
||||
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
|
||||
from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
|
||||
from freqtrade.strategy.interface import IStrategy, SellCheckTuple
|
||||
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
|
||||
from freqtrade.wallets import Wallets
|
||||
|
||||
@@ -41,6 +43,7 @@ CLOSE_IDX = 3
|
||||
SELL_IDX = 4
|
||||
LOW_IDX = 5
|
||||
HIGH_IDX = 6
|
||||
BUY_TAG_IDX = 7
|
||||
|
||||
|
||||
class Backtesting:
|
||||
@@ -56,16 +59,14 @@ class Backtesting:
|
||||
|
||||
LoggingMixin.show_output = False
|
||||
self.config = config
|
||||
self.results: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Reset keys for backtesting
|
||||
remove_credentials(self.config)
|
||||
config['dry_run'] = True
|
||||
self.strategylist: List[IStrategy] = []
|
||||
self.all_results: Dict[str, Dict] = {}
|
||||
|
||||
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
|
||||
|
||||
dataprovider = DataProvider(self.config, self.exchange)
|
||||
IStrategy.dp = dataprovider
|
||||
self.dataprovider = DataProvider(self.config, None)
|
||||
|
||||
if self.config.get('strategy_list', None):
|
||||
for strat in list(self.config['strategy_list']):
|
||||
@@ -84,7 +85,7 @@ class Backtesting:
|
||||
"configuration or as cli argument `--timeframe 5m`")
|
||||
self.timeframe = str(self.config.get('timeframe'))
|
||||
self.timeframe_min = timeframe_to_minutes(self.timeframe)
|
||||
|
||||
self.init_backtest_detail()
|
||||
self.pairlists = PairListManager(self.exchange, self.config)
|
||||
if 'VolumePairList' in self.pairlists.name_list:
|
||||
raise OperationalException("VolumePairList not allowed for backtesting.")
|
||||
@@ -96,7 +97,7 @@ class Backtesting:
|
||||
"PrecisionFilter not allowed for backtesting multiple strategies."
|
||||
)
|
||||
|
||||
dataprovider.add_pairlisthandler(self.pairlists)
|
||||
self.dataprovider.add_pairlisthandler(self.pairlists)
|
||||
self.pairlists.refresh_pairlist()
|
||||
|
||||
if len(self.pairlists.whitelist) == 0:
|
||||
@@ -107,49 +108,79 @@ class Backtesting:
|
||||
else:
|
||||
self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
|
||||
|
||||
Trade.use_db = False
|
||||
Trade.reset_trades()
|
||||
PairLocks.timeframe = self.config['timeframe']
|
||||
PairLocks.use_db = False
|
||||
PairLocks.reset_locks()
|
||||
if self.config.get('enable_protections', False):
|
||||
self.protections = ProtectionManager(self.config)
|
||||
|
||||
self.wallets = Wallets(self.config, self.exchange, log=False)
|
||||
self.timerange = TimeRange.parse_timerange(
|
||||
None if self.config.get('timerange') is None else str(self.config.get('timerange')))
|
||||
|
||||
# Get maximum required startup period
|
||||
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
|
||||
# Load one (first) strategy
|
||||
self._set_strategy(self.strategylist[0])
|
||||
# Add maximum startup candle count to configuration for informative pairs support
|
||||
self.config['startup_candle_count'] = self.required_startup
|
||||
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
|
||||
self.init_backtest()
|
||||
|
||||
def __del__(self):
|
||||
self.cleanup()
|
||||
|
||||
def cleanup(self):
|
||||
LoggingMixin.show_output = True
|
||||
PairLocks.use_db = True
|
||||
Trade.use_db = True
|
||||
|
||||
def init_backtest_detail(self):
|
||||
# Load detail timeframe if specified
|
||||
self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
|
||||
if self.timeframe_detail:
|
||||
self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
|
||||
if self.timeframe_min <= self.timeframe_detail_min:
|
||||
raise OperationalException(
|
||||
"Detail timeframe must be smaller than strategy timeframe.")
|
||||
|
||||
else:
|
||||
self.timeframe_detail_min = 0
|
||||
self.detail_data: Dict[str, DataFrame] = {}
|
||||
|
||||
def init_backtest(self):
|
||||
|
||||
self.prepare_backtest(False)
|
||||
|
||||
self.wallets = Wallets(self.config, self.exchange, log=False)
|
||||
|
||||
self.progress = BTProgress()
|
||||
self.abort = False
|
||||
|
||||
def _set_strategy(self, strategy: IStrategy):
|
||||
"""
|
||||
Load strategy into backtesting
|
||||
"""
|
||||
self.strategy: IStrategy = strategy
|
||||
strategy.dp = self.dataprovider
|
||||
# Attach Wallets to Strategy baseclass
|
||||
strategy.wallets = self.wallets
|
||||
# Set stoploss_on_exchange to false for backtesting,
|
||||
# since a "perfect" stoploss-sell is assumed anyway
|
||||
# And the regular "stoploss" function would not apply to that case
|
||||
self.strategy.order_types['stoploss_on_exchange'] = False
|
||||
|
||||
def _load_protections(self, strategy: IStrategy):
|
||||
if self.config.get('enable_protections', False):
|
||||
conf = self.config
|
||||
if hasattr(strategy, 'protections'):
|
||||
conf = deepcopy(conf)
|
||||
conf['protections'] = strategy.protections
|
||||
self.protections = ProtectionManager(self.config, strategy.protections)
|
||||
|
||||
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
|
||||
"""
|
||||
Loads backtest data and returns the data combined with the timerange
|
||||
as tuple.
|
||||
"""
|
||||
timerange = TimeRange.parse_timerange(None if self.config.get(
|
||||
'timerange') is None else str(self.config.get('timerange')))
|
||||
self.progress.init_step(BacktestState.DATALOAD, 1)
|
||||
|
||||
data = history.load_data(
|
||||
datadir=self.config['datadir'],
|
||||
pairs=self.pairlists.whitelist,
|
||||
timeframe=self.timeframe,
|
||||
timerange=timerange,
|
||||
timerange=self.timerange,
|
||||
startup_candles=self.required_startup,
|
||||
fail_without_data=True,
|
||||
data_format=self.config.get('dataformat_ohlcv', 'json'),
|
||||
@@ -159,13 +190,31 @@ class Backtesting:
|
||||
|
||||
logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'({(max_date - min_date).days} days)..')
|
||||
f'({(max_date - min_date).days} days).')
|
||||
|
||||
# Adjust startts forward if not enough data is available
|
||||
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
|
||||
self.required_startup, min_date)
|
||||
self.timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
|
||||
self.required_startup, min_date)
|
||||
|
||||
return data, timerange
|
||||
self.progress.set_new_value(1)
|
||||
return data, self.timerange
|
||||
|
||||
def load_bt_data_detail(self) -> None:
|
||||
"""
|
||||
Loads backtest detail data (smaller timeframe) if necessary.
|
||||
"""
|
||||
if self.timeframe_detail:
|
||||
self.detail_data = history.load_data(
|
||||
datadir=self.config['datadir'],
|
||||
pairs=self.pairlists.whitelist,
|
||||
timeframe=self.timeframe_detail,
|
||||
timerange=self.timerange,
|
||||
startup_candles=0,
|
||||
fail_without_data=True,
|
||||
data_format=self.config.get('dataformat_ohlcv', 'json'),
|
||||
)
|
||||
else:
|
||||
self.detail_data = {}
|
||||
|
||||
def prepare_backtest(self, enable_protections):
|
||||
"""
|
||||
@@ -176,6 +225,19 @@ class Backtesting:
|
||||
Trade.use_db = False
|
||||
PairLocks.reset_locks()
|
||||
Trade.reset_trades()
|
||||
self.rejected_trades = 0
|
||||
self.dataprovider.clear_cache()
|
||||
if enable_protections:
|
||||
self._load_protections(self.strategy)
|
||||
|
||||
def check_abort(self):
|
||||
"""
|
||||
Check if abort was requested, raise DependencyException if that's the case
|
||||
Only applies to Interactive backtest mode (webserver mode)
|
||||
"""
|
||||
if self.abort:
|
||||
self.abort = False
|
||||
raise DependencyException("Stop requested")
|
||||
|
||||
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
|
||||
"""
|
||||
@@ -185,26 +247,38 @@ class Backtesting:
|
||||
"""
|
||||
# Every change to this headers list must evaluate further usages of the resulting tuple
|
||||
# and eventually change the constants for indexes at the top
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_tag']
|
||||
data: Dict = {}
|
||||
self.progress.init_step(BacktestState.CONVERT, len(processed))
|
||||
|
||||
# Create dict with data
|
||||
for pair, pair_data in processed.items():
|
||||
pair_data.loc[:, 'buy'] = 0 # cleanup from previous run
|
||||
pair_data.loc[:, 'sell'] = 0 # cleanup from previous run
|
||||
self.check_abort()
|
||||
self.progress.increment()
|
||||
if not pair_data.empty:
|
||||
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
|
||||
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
|
||||
pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist
|
||||
|
||||
df_analyzed = self.strategy.advise_sell(
|
||||
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
|
||||
|
||||
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair}).copy()
|
||||
# Trim startup period from analyzed dataframe
|
||||
df_analyzed = trim_dataframe(df_analyzed, self.timerange,
|
||||
startup_candles=self.required_startup)
|
||||
# To avoid using data from future, we use buy/sell signals shifted
|
||||
# from the previous candle
|
||||
df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
|
||||
df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
|
||||
df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
|
||||
|
||||
df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
|
||||
# Update dataprovider cache
|
||||
self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed)
|
||||
|
||||
df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
|
||||
|
||||
# Convert from Pandas to list for performance reasons
|
||||
# (Looping Pandas is slow.)
|
||||
data[pair] = df_analyzed.values.tolist()
|
||||
data[pair] = df_analyzed[headers].values.tolist()
|
||||
return data
|
||||
|
||||
def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
|
||||
@@ -214,6 +288,32 @@ class Backtesting:
|
||||
"""
|
||||
# Special handling if high or low hit STOP_LOSS or ROI
|
||||
if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
|
||||
if trade.stop_loss > sell_row[HIGH_IDX]:
|
||||
# our stoploss was already higher than candle high,
|
||||
# possibly due to a cancelled trade exit.
|
||||
# sell at open price.
|
||||
return sell_row[OPEN_IDX]
|
||||
|
||||
# Special case: trailing triggers within same candle as trade opened. Assume most
|
||||
# pessimistic price movement, which is moving just enough to arm stoploss and
|
||||
# immediately going down to stop price.
|
||||
if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0:
|
||||
if (
|
||||
not self.strategy.use_custom_stoploss and self.strategy.trailing_stop
|
||||
and self.strategy.trailing_only_offset_is_reached
|
||||
and self.strategy.trailing_stop_positive_offset is not None
|
||||
and self.strategy.trailing_stop_positive
|
||||
):
|
||||
# Worst case: price reaches stop_positive_offset and dives down.
|
||||
stop_rate = (sell_row[OPEN_IDX] *
|
||||
(1 + abs(self.strategy.trailing_stop_positive_offset) -
|
||||
abs(self.strategy.trailing_stop_positive)))
|
||||
else:
|
||||
# Worst case: price ticks tiny bit above open and dives down.
|
||||
stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct))
|
||||
assert stop_rate < sell_row[HIGH_IDX]
|
||||
return stop_rate
|
||||
|
||||
# Set close_rate to stoploss
|
||||
return trade.stop_loss
|
||||
elif sell.sell_type == (SellType.ROI):
|
||||
@@ -239,7 +339,7 @@ class Backtesting:
|
||||
# Use the maximum between close_rate and low as we
|
||||
# cannot sell outside of a candle.
|
||||
# Applies when a new ROI setting comes in place and the whole candle is above that.
|
||||
return max(close_rate, sell_row[LOW_IDX])
|
||||
return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX])
|
||||
|
||||
else:
|
||||
# This should not be reached...
|
||||
@@ -247,41 +347,100 @@ class Backtesting:
|
||||
else:
|
||||
return sell_row[OPEN_IDX]
|
||||
|
||||
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
|
||||
|
||||
def _get_sell_trade_entry_for_candle(self, trade: LocalTrade,
|
||||
sell_row: Tuple) -> Optional[LocalTrade]:
|
||||
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
|
||||
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore
|
||||
sell_row[DATE_IDX], sell_row[BUY_IDX], sell_row[SELL_IDX],
|
||||
sell_candle_time, sell_row[BUY_IDX],
|
||||
sell_row[SELL_IDX],
|
||||
low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
|
||||
if sell.sell_flag:
|
||||
|
||||
trade.close_date = sell_row[DATE_IDX]
|
||||
trade.sell_reason = sell.sell_type.value
|
||||
if sell.sell_flag:
|
||||
trade.close_date = sell_candle_time
|
||||
trade.sell_reason = sell.sell_reason
|
||||
trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
|
||||
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
|
||||
|
||||
# Confirm trade exit:
|
||||
time_in_force = self.strategy.order_time_in_force['sell']
|
||||
if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
|
||||
pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
|
||||
rate=closerate,
|
||||
time_in_force=time_in_force,
|
||||
sell_reason=sell.sell_reason,
|
||||
current_time=sell_candle_time):
|
||||
return None
|
||||
|
||||
trade.close(closerate, show_msg=False)
|
||||
return trade
|
||||
|
||||
return None
|
||||
|
||||
def _enter_trade(self, pair: str, row: List, max_open_trades: int,
|
||||
open_trade_count: int) -> Optional[LocalTrade]:
|
||||
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
|
||||
if self.timeframe_detail and trade.pair in self.detail_data:
|
||||
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
|
||||
sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min)
|
||||
|
||||
detail_data = self.detail_data[trade.pair]
|
||||
detail_data = detail_data.loc[
|
||||
(detail_data['date'] >= sell_candle_time) &
|
||||
(detail_data['date'] < sell_candle_end)
|
||||
].copy()
|
||||
if len(detail_data) == 0:
|
||||
# Fall back to "regular" data if no detail data was found for this candle
|
||||
return self._get_sell_trade_entry_for_candle(trade, sell_row)
|
||||
detail_data.loc[:, 'buy'] = sell_row[BUY_IDX]
|
||||
detail_data.loc[:, 'sell'] = sell_row[SELL_IDX]
|
||||
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
|
||||
for det_row in detail_data[headers].values.tolist():
|
||||
res = self._get_sell_trade_entry_for_candle(trade, det_row)
|
||||
if res:
|
||||
return res
|
||||
|
||||
return None
|
||||
|
||||
else:
|
||||
return self._get_sell_trade_entry_for_candle(trade, sell_row)
|
||||
|
||||
def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
|
||||
try:
|
||||
stake_amount = self.wallets.get_trade_stake_amount(
|
||||
pair, max_open_trades - open_trade_count, None)
|
||||
stake_amount = self.wallets.get_trade_stake_amount(pair, None)
|
||||
except DependencyException:
|
||||
return None
|
||||
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
|
||||
|
||||
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) or 0
|
||||
max_stake_amount = self.wallets.get_available_stake_amount()
|
||||
|
||||
stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
|
||||
default_retval=stake_amount)(
|
||||
pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX],
|
||||
proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount)
|
||||
stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
|
||||
|
||||
if not stake_amount:
|
||||
return None
|
||||
|
||||
order_type = self.strategy.order_types['buy']
|
||||
time_in_force = self.strategy.order_time_in_force['sell']
|
||||
# Confirm trade entry:
|
||||
if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
|
||||
pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX],
|
||||
time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()):
|
||||
return None
|
||||
|
||||
if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
|
||||
# Enter trade
|
||||
has_buy_tag = len(row) >= BUY_TAG_IDX + 1
|
||||
trade = LocalTrade(
|
||||
pair=pair,
|
||||
open_rate=row[OPEN_IDX],
|
||||
open_date=row[DATE_IDX],
|
||||
open_date=row[DATE_IDX].to_pydatetime(),
|
||||
stake_amount=stake_amount,
|
||||
amount=round(stake_amount / row[OPEN_IDX], 8),
|
||||
fee_open=self.fee,
|
||||
fee_close=self.fee,
|
||||
is_open=True,
|
||||
buy_tag=row[BUY_TAG_IDX] if has_buy_tag else None,
|
||||
exchange='backtesting',
|
||||
)
|
||||
return trade
|
||||
@@ -298,7 +457,7 @@ class Backtesting:
|
||||
for trade in open_trades[pair]:
|
||||
sell_row = data[pair][-1]
|
||||
|
||||
trade.close_date = sell_row[DATE_IDX]
|
||||
trade.close_date = sell_row[DATE_IDX].to_pydatetime()
|
||||
trade.sell_reason = SellType.FORCE_SELL.value
|
||||
trade.close(sell_row[OPEN_IDX], show_msg=False)
|
||||
LocalTrade.close_bt_trade(trade)
|
||||
@@ -308,10 +467,18 @@ class Backtesting:
|
||||
trades.append(trade1)
|
||||
return trades
|
||||
|
||||
def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool:
|
||||
# Always allow trades when max_open_trades is enabled.
|
||||
if max_open_trades <= 0 or open_trade_count < max_open_trades:
|
||||
return True
|
||||
# Rejected trade
|
||||
self.rejected_trades += 1
|
||||
return False
|
||||
|
||||
def backtest(self, processed: Dict,
|
||||
start_date: datetime, end_date: datetime,
|
||||
max_open_trades: int = 0, position_stacking: bool = False,
|
||||
enable_protections: bool = False) -> DataFrame:
|
||||
enable_protections: bool = False) -> Dict[str, Any]:
|
||||
"""
|
||||
Implement backtesting functionality
|
||||
|
||||
@@ -335,22 +502,25 @@ class Backtesting:
|
||||
data: Dict = self._get_ohlcv_as_lists(processed)
|
||||
|
||||
# Indexes per pair, so some pairs are allowed to have a missing start.
|
||||
indexes: Dict = {}
|
||||
indexes: Dict = defaultdict(int)
|
||||
tmp = start_date + timedelta(minutes=self.timeframe_min)
|
||||
|
||||
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
|
||||
open_trade_count = 0
|
||||
|
||||
self.progress.init_step(BacktestState.BACKTEST, int(
|
||||
(end_date - start_date) / timedelta(minutes=self.timeframe_min)))
|
||||
|
||||
# Loop timerange and get candle for each pair at that point in time
|
||||
while tmp <= end_date:
|
||||
open_trade_count_start = open_trade_count
|
||||
|
||||
self.check_abort()
|
||||
for i, pair in enumerate(data):
|
||||
if pair not in indexes:
|
||||
indexes[pair] = 0
|
||||
|
||||
row_index = indexes[pair]
|
||||
try:
|
||||
row = data[pair][indexes[pair]]
|
||||
# Row is treated as "current incomplete candle".
|
||||
# Buy / sell signals are shifted by 1 to compensate for this.
|
||||
row = data[pair][row_index]
|
||||
except IndexError:
|
||||
# missing Data for one pair at the end.
|
||||
# Warnings for this are shown during data loading
|
||||
@@ -359,17 +529,23 @@ class Backtesting:
|
||||
# Waits until the time-counter reaches the start of the data for this pair.
|
||||
if row[DATE_IDX] > tmp:
|
||||
continue
|
||||
indexes[pair] += 1
|
||||
|
||||
row_index += 1
|
||||
indexes[pair] = row_index
|
||||
self.dataprovider._set_dataframe_max_index(row_index)
|
||||
|
||||
# without positionstacking, we can only have one open trade per pair.
|
||||
# max_open_trades must be respected
|
||||
# don't open on the last row
|
||||
if ((position_stacking or len(open_trades[pair]) == 0)
|
||||
and (max_open_trades <= 0 or open_trade_count_start < max_open_trades)
|
||||
and tmp != end_date
|
||||
and row[BUY_IDX] == 1 and row[SELL_IDX] != 1
|
||||
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
|
||||
trade = self._enter_trade(pair, row, max_open_trades, open_trade_count_start)
|
||||
if (
|
||||
(position_stacking or len(open_trades[pair]) == 0)
|
||||
and self.trade_slot_available(max_open_trades, open_trade_count_start)
|
||||
and tmp != end_date
|
||||
and row[BUY_IDX] == 1
|
||||
and row[SELL_IDX] != 1
|
||||
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])
|
||||
):
|
||||
trade = self._enter_trade(pair, row)
|
||||
if trade:
|
||||
# TODO: hacky workaround to avoid opening > max_open_trades
|
||||
# This emulates previous behaviour - not sure if this is correct
|
||||
@@ -380,10 +556,10 @@ class Backtesting:
|
||||
open_trades[pair].append(trade)
|
||||
LocalTrade.add_bt_trade(trade)
|
||||
|
||||
for trade in open_trades[pair]:
|
||||
for trade in list(open_trades[pair]):
|
||||
# also check the buying candle for sell conditions.
|
||||
trade_entry = self._get_sell_trade_entry(trade, row)
|
||||
# Sell occured
|
||||
# Sell occurred
|
||||
if trade_entry:
|
||||
# logger.debug(f"{pair} - Backtesting sell {trade}")
|
||||
open_trade_count -= 1
|
||||
@@ -396,14 +572,25 @@ class Backtesting:
|
||||
self.protections.global_stop(tmp)
|
||||
|
||||
# Move time one configured time_interval ahead.
|
||||
self.progress.increment()
|
||||
tmp += timedelta(minutes=self.timeframe_min)
|
||||
|
||||
trades += self.handle_left_open(open_trades, data=data)
|
||||
self.wallets.update()
|
||||
|
||||
return trade_list_to_dataframe(trades)
|
||||
results = trade_list_to_dataframe(trades)
|
||||
return {
|
||||
'results': results,
|
||||
'config': self.strategy.config,
|
||||
'locks': PairLocks.get_all_locks(),
|
||||
'rejected_signals': self.rejected_trades,
|
||||
'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
|
||||
}
|
||||
|
||||
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, DataFrame],
|
||||
timerange: TimeRange):
|
||||
self.progress.init_step(BacktestState.ANALYZE, 0)
|
||||
|
||||
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
|
||||
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
|
||||
backtest_start_time = datetime.now(timezone.utc)
|
||||
self._set_strategy(strat)
|
||||
@@ -420,34 +607,37 @@ class Backtesting:
|
||||
max_open_trades = 0
|
||||
|
||||
# need to reprocess data every time to populate signals
|
||||
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
|
||||
preprocessed = self.strategy.advise_all_indicators(data)
|
||||
|
||||
# Trim startup period from analyzed dataframe
|
||||
for pair, df in preprocessed.items():
|
||||
preprocessed[pair] = trim_dataframe(df, timerange)
|
||||
min_date, max_date = history.get_timerange(preprocessed)
|
||||
preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup)
|
||||
|
||||
if not preprocessed_tmp:
|
||||
raise OperationalException(
|
||||
"No data left after adjusting for startup candles.")
|
||||
|
||||
# Use preprocessed_tmp for date generation (the trimmed dataframe).
|
||||
# Backtesting will re-trim the dataframes after buy/sell signal generation.
|
||||
min_date, max_date = history.get_timerange(preprocessed_tmp)
|
||||
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'({(max_date - min_date).days} days)..')
|
||||
f'({(max_date - min_date).days} days).')
|
||||
# Execute backtest and store results
|
||||
results = self.backtest(
|
||||
processed=preprocessed,
|
||||
start_date=min_date.datetime,
|
||||
end_date=max_date.datetime,
|
||||
start_date=min_date,
|
||||
end_date=max_date,
|
||||
max_open_trades=max_open_trades,
|
||||
position_stacking=self.config.get('position_stacking', False),
|
||||
enable_protections=self.config.get('enable_protections', False),
|
||||
)
|
||||
backtest_end_time = datetime.now(timezone.utc)
|
||||
self.all_results[self.strategy.get_strategy_name()] = {
|
||||
'results': results,
|
||||
'config': self.strategy.config,
|
||||
'locks': PairLocks.get_all_locks(),
|
||||
'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
|
||||
results.update({
|
||||
'backtest_start_time': int(backtest_start_time.timestamp()),
|
||||
'backtest_end_time': int(backtest_end_time.timestamp()),
|
||||
}
|
||||
})
|
||||
self.all_results[self.strategy.get_strategy_name()] = results
|
||||
|
||||
return min_date, max_date
|
||||
|
||||
def start(self) -> None:
|
||||
@@ -458,15 +648,18 @@ class Backtesting:
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
data, timerange = self.load_bt_data()
|
||||
self.load_bt_data_detail()
|
||||
logger.info("Dataload complete. Calculating indicators")
|
||||
|
||||
for strat in self.strategylist:
|
||||
min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
|
||||
if len(self.strategylist) > 0:
|
||||
stats = generate_backtest_stats(data, self.all_results,
|
||||
min_date=min_date, max_date=max_date)
|
||||
|
||||
if self.config.get('export', False):
|
||||
store_backtest_stats(self.config['exportfilename'], stats)
|
||||
self.results = generate_backtest_stats(data, self.all_results,
|
||||
min_date=min_date, max_date=max_date)
|
||||
|
||||
if self.config.get('export', 'none') == 'trades':
|
||||
store_backtest_stats(self.config['exportfilename'], self.results)
|
||||
|
||||
# Show backtest results
|
||||
show_backtest_results(self.config, stats)
|
||||
show_backtest_results(self.config, self.results)
|
||||
|
33
freqtrade/optimize/bt_progress.py
Normal file
33
freqtrade/optimize/bt_progress.py
Normal file
@@ -0,0 +1,33 @@
|
||||
from freqtrade.enums import BacktestState
|
||||
|
||||
|
||||
class BTProgress:
|
||||
_action: BacktestState = BacktestState.STARTUP
|
||||
_progress: float = 0
|
||||
_max_steps: float = 0
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def init_step(self, action: BacktestState, max_steps: float):
|
||||
self._action = action
|
||||
self._max_steps = max_steps
|
||||
self._proress = 0
|
||||
|
||||
def set_new_value(self, new_value: float):
|
||||
self._progress = new_value
|
||||
|
||||
def increment(self):
|
||||
self._progress += 1
|
||||
|
||||
@property
|
||||
def progress(self):
|
||||
"""
|
||||
Get progress as ratio, capped to be between 0 and 1 (to avoid small calculation errors).
|
||||
"""
|
||||
return max(min(round(self._progress / self._max_steps, 5)
|
||||
if self._max_steps > 0 else 0, 1), 0)
|
||||
|
||||
@property
|
||||
def action(self):
|
||||
return str(self._action)
|
@@ -7,7 +7,8 @@ import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade import constants
|
||||
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
|
||||
from freqtrade.configuration import TimeRange, validate_config_consistency
|
||||
from freqtrade.data.dataprovider import DataProvider
|
||||
from freqtrade.edge import Edge
|
||||
from freqtrade.optimize.optimize_reports import generate_edge_table
|
||||
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
|
||||
@@ -28,11 +29,12 @@ class EdgeCli:
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
self.config = config
|
||||
|
||||
# Reset keys for edge
|
||||
remove_credentials(self.config)
|
||||
# Ensure using dry-run
|
||||
self.config['dry_run'] = True
|
||||
self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
|
||||
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
|
||||
self.strategy = StrategyResolver.load_strategy(self.config)
|
||||
self.strategy.dp = DataProvider(config, None)
|
||||
|
||||
validate_config_consistency(self.config)
|
||||
|
||||
@@ -44,7 +46,7 @@ class EdgeCli:
|
||||
'timerange') is None else str(self.config.get('timerange')))
|
||||
|
||||
def start(self) -> None:
|
||||
result = self.edge.calculate()
|
||||
result = self.edge.calculate(self.config['exchange']['pair_whitelist'])
|
||||
if result:
|
||||
print('') # blank line for readability
|
||||
print(generate_edge_table(self.edge._cached_pairs))
|
||||
|
@@ -4,38 +4,34 @@
|
||||
This module contains the hyperopt logic
|
||||
"""
|
||||
|
||||
import io
|
||||
import locale
|
||||
import logging
|
||||
import random
|
||||
import warnings
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from math import ceil
|
||||
from operator import itemgetter
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import progressbar
|
||||
import rapidjson
|
||||
import tabulate
|
||||
from colorama import Fore, Style
|
||||
from colorama import init as colorama_init
|
||||
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
|
||||
from pandas import DataFrame, isna, json_normalize
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
|
||||
from freqtrade.data.converter import trim_dataframe
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
|
||||
from freqtrade.data.converter import trim_dataframes
|
||||
from freqtrade.data.history import get_timerange
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import file_dump_json, plural, round_dict
|
||||
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
|
||||
from freqtrade.optimize.backtesting import Backtesting
|
||||
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
|
||||
from freqtrade.optimize.hyperopt_auto import HyperOptAuto
|
||||
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
|
||||
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
|
||||
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
|
||||
from freqtrade.strategy import IStrategy
|
||||
from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
|
||||
from freqtrade.optimize.optimize_reports import generate_strategy_stats
|
||||
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
|
||||
|
||||
|
||||
# Suppress scikit-learn FutureWarnings from skopt
|
||||
@@ -49,7 +45,7 @@ progressbar.streams.wrap_stdout()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
INITIAL_POINTS = 30
|
||||
INITIAL_POINTS = 5
|
||||
|
||||
# Keep no more than SKOPT_MODEL_QUEUE_SIZE models
|
||||
# in the skopt model queue, to optimize memory consumption
|
||||
@@ -66,22 +62,37 @@ class Hyperopt:
|
||||
hyperopt = Hyperopt(config)
|
||||
hyperopt.start()
|
||||
"""
|
||||
custom_hyperopt: IHyperOpt
|
||||
|
||||
def __init__(self, config: Dict[str, Any]) -> None:
|
||||
self.buy_space: List[Dimension] = []
|
||||
self.sell_space: List[Dimension] = []
|
||||
self.protection_space: List[Dimension] = []
|
||||
self.roi_space: List[Dimension] = []
|
||||
self.stoploss_space: List[Dimension] = []
|
||||
self.trailing_space: List[Dimension] = []
|
||||
self.dimensions: List[Dimension] = []
|
||||
|
||||
self.config = config
|
||||
|
||||
self.backtesting = Backtesting(self.config)
|
||||
|
||||
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
|
||||
self.custom_hyperopt.__class__.strategy = self.backtesting.strategy
|
||||
if not self.config.get('hyperopt'):
|
||||
self.custom_hyperopt = HyperOptAuto(self.config)
|
||||
else:
|
||||
raise OperationalException(
|
||||
"Using separate Hyperopt files has been removed in 2021.9. Please convert "
|
||||
"your existing Hyperopt file to the new Hyperoptable strategy interface")
|
||||
|
||||
self.backtesting._set_strategy(self.backtesting.strategylist[0])
|
||||
self.custom_hyperopt.strategy = self.backtesting.strategy
|
||||
|
||||
self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config)
|
||||
self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
|
||||
time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
strategy = str(self.config['strategy'])
|
||||
self.results_file = (self.config['user_data_dir'] /
|
||||
'hyperopt_results' /
|
||||
f'strategy_{strategy}_hyperopt_results_{time_now}.pickle')
|
||||
self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' /
|
||||
f'strategy_{strategy}_{time_now}.fthypt')
|
||||
self.data_pickle_file = (self.config['user_data_dir'] /
|
||||
'hyperopt_results' / 'hyperopt_tickerdata.pkl')
|
||||
self.total_epochs = config.get('epochs', 0)
|
||||
@@ -91,20 +102,7 @@ class Hyperopt:
|
||||
self.clean_hyperopt()
|
||||
|
||||
self.num_epochs_saved = 0
|
||||
|
||||
# Previous evaluations
|
||||
self.epochs: List = []
|
||||
|
||||
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
|
||||
if hasattr(self.custom_hyperopt, 'populate_indicators'):
|
||||
self.backtesting.strategy.advise_indicators = ( # type: ignore
|
||||
self.custom_hyperopt.populate_indicators) # type: ignore
|
||||
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
|
||||
self.backtesting.strategy.advise_buy = ( # type: ignore
|
||||
self.custom_hyperopt.populate_buy_trend) # type: ignore
|
||||
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
|
||||
self.backtesting.strategy.advise_sell = ( # type: ignore
|
||||
self.custom_hyperopt.populate_sell_trend) # type: ignore
|
||||
self.current_best_epoch: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
|
||||
if self.config.get('use_max_market_positions', True):
|
||||
@@ -114,7 +112,7 @@ class Hyperopt:
|
||||
self.max_open_trades = 0
|
||||
self.position_stacking = self.config.get('position_stacking', False)
|
||||
|
||||
if self.has_space('sell'):
|
||||
if HyperoptTools.has_space(self.config, 'sell'):
|
||||
# Make sure use_sell_signal is enabled
|
||||
if 'ask_strategy' not in self.config:
|
||||
self.config['ask_strategy'] = {}
|
||||
@@ -140,9 +138,7 @@ class Hyperopt:
|
||||
logger.info(f"Removing `{p}`.")
|
||||
p.unlink()
|
||||
|
||||
def _get_params_dict(self, raw_params: List[Any]) -> Dict:
|
||||
|
||||
dimensions: List[Dimension] = self.dimensions
|
||||
def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict:
|
||||
|
||||
# Ensure the number of dimensions match
|
||||
# the number of parameters in the list.
|
||||
@@ -153,30 +149,26 @@ class Hyperopt:
|
||||
# and the values are taken from the list of parameters.
|
||||
return {d.name: v for d, v in zip(dimensions, raw_params)}
|
||||
|
||||
def _save_results(self) -> None:
|
||||
def _save_result(self, epoch: Dict) -> None:
|
||||
"""
|
||||
Save hyperopt results to file
|
||||
Store one line per epoch.
|
||||
While not a valid json object - this allows appending easily.
|
||||
:param epoch: result dictionary for this epoch.
|
||||
"""
|
||||
num_epochs = len(self.epochs)
|
||||
if num_epochs > self.num_epochs_saved:
|
||||
logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.")
|
||||
dump(self.epochs, self.results_file)
|
||||
self.num_epochs_saved = num_epochs
|
||||
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
|
||||
f"saved to '{self.results_file}'.")
|
||||
# Store hyperopt filename
|
||||
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
|
||||
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
|
||||
log=False)
|
||||
epoch[FTHYPT_FILEVERSION] = 2
|
||||
with self.results_file.open('a') as f:
|
||||
rapidjson.dump(epoch, f, default=hyperopt_serializer,
|
||||
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
|
||||
f.write("\n")
|
||||
|
||||
@staticmethod
|
||||
def _read_results(results_file: Path) -> List:
|
||||
"""
|
||||
Read hyperopt results from file
|
||||
"""
|
||||
logger.info("Reading epochs from '%s'", results_file)
|
||||
data = load(results_file)
|
||||
return data
|
||||
self.num_epochs_saved += 1
|
||||
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
|
||||
f"saved to '{self.results_file}'.")
|
||||
# Store hyperopt filename
|
||||
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
|
||||
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
|
||||
log=False)
|
||||
|
||||
def _get_params_details(self, params: Dict) -> Dict:
|
||||
"""
|
||||
@@ -184,118 +176,51 @@ class Hyperopt:
|
||||
"""
|
||||
result: Dict = {}
|
||||
|
||||
if self.has_space('buy'):
|
||||
result['buy'] = {p.name: params.get(p.name)
|
||||
for p in self.hyperopt_space('buy')}
|
||||
if self.has_space('sell'):
|
||||
result['sell'] = {p.name: params.get(p.name)
|
||||
for p in self.hyperopt_space('sell')}
|
||||
if self.has_space('roi'):
|
||||
result['roi'] = self.custom_hyperopt.generate_roi_table(params)
|
||||
if self.has_space('stoploss'):
|
||||
result['stoploss'] = {p.name: params.get(p.name)
|
||||
for p in self.hyperopt_space('stoploss')}
|
||||
if self.has_space('trailing'):
|
||||
if HyperoptTools.has_space(self.config, 'buy'):
|
||||
result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
|
||||
if HyperoptTools.has_space(self.config, 'sell'):
|
||||
result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
|
||||
if HyperoptTools.has_space(self.config, 'protection'):
|
||||
result['protection'] = {p.name: params.get(p.name) for p in self.protection_space}
|
||||
if HyperoptTools.has_space(self.config, 'roi'):
|
||||
result['roi'] = {str(k): v for k, v in
|
||||
self.custom_hyperopt.generate_roi_table(params).items()}
|
||||
if HyperoptTools.has_space(self.config, 'stoploss'):
|
||||
result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space}
|
||||
if HyperoptTools.has_space(self.config, 'trailing'):
|
||||
result['trailing'] = self.custom_hyperopt.generate_trailing_params(params)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def print_epoch_details(results, total_epochs: int, print_json: bool,
|
||||
no_header: bool = False, header_str: str = None) -> None:
|
||||
def _get_no_optimize_details(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Display details of the hyperopt result
|
||||
Get non-optimized parameters
|
||||
"""
|
||||
params = results.get('params_details', {})
|
||||
|
||||
# Default header string
|
||||
if header_str is None:
|
||||
header_str = "Best result"
|
||||
|
||||
if not no_header:
|
||||
explanation_str = Hyperopt._format_explanation_string(results, total_epochs)
|
||||
print(f"\n{header_str}:\n\n{explanation_str}\n")
|
||||
|
||||
if print_json:
|
||||
result_dict: Dict = {}
|
||||
for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
|
||||
Hyperopt._params_update_for_json(result_dict, params, s)
|
||||
print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
|
||||
|
||||
else:
|
||||
Hyperopt._params_pretty_print(params, 'buy', "Buy hyperspace params:")
|
||||
Hyperopt._params_pretty_print(params, 'sell', "Sell hyperspace params:")
|
||||
Hyperopt._params_pretty_print(params, 'roi', "ROI table:")
|
||||
Hyperopt._params_pretty_print(params, 'stoploss', "Stoploss:")
|
||||
Hyperopt._params_pretty_print(params, 'trailing', "Trailing stop:")
|
||||
|
||||
@staticmethod
|
||||
def _params_update_for_json(result_dict, params, space: str) -> None:
|
||||
if space in params:
|
||||
space_params = Hyperopt._space_params(params, space)
|
||||
if space in ['buy', 'sell']:
|
||||
result_dict.setdefault('params', {}).update(space_params)
|
||||
elif space == 'roi':
|
||||
# TODO: get rid of OrderedDict when support for python 3.6 will be
|
||||
# dropped (dicts keep the order as the language feature)
|
||||
|
||||
# Convert keys in min_roi dict to strings because
|
||||
# rapidjson cannot dump dicts with integer keys...
|
||||
# OrderedDict is used to keep the numeric order of the items
|
||||
# in the dict.
|
||||
result_dict['minimal_roi'] = OrderedDict(
|
||||
(str(k), v) for k, v in space_params.items()
|
||||
)
|
||||
else: # 'stoploss', 'trailing'
|
||||
result_dict.update(space_params)
|
||||
|
||||
@staticmethod
|
||||
def _params_pretty_print(params, space: str, header: str) -> None:
|
||||
if space in params:
|
||||
space_params = Hyperopt._space_params(params, space, 5)
|
||||
params_result = f"\n# {header}\n"
|
||||
if space == 'stoploss':
|
||||
params_result += f"stoploss = {space_params.get('stoploss')}"
|
||||
elif space == 'roi':
|
||||
# TODO: get rid of OrderedDict when support for python 3.6 will be
|
||||
# dropped (dicts keep the order as the language feature)
|
||||
minimal_roi_result = rapidjson.dumps(
|
||||
OrderedDict(
|
||||
(str(k), v) for k, v in space_params.items()
|
||||
),
|
||||
default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
|
||||
params_result += f"minimal_roi = {minimal_roi_result}"
|
||||
elif space == 'trailing':
|
||||
|
||||
for k, v in space_params.items():
|
||||
params_result += f'{k} = {v}\n'
|
||||
|
||||
else:
|
||||
params_result += f"{space}_params = {pformat(space_params, indent=4)}"
|
||||
params_result = params_result.replace("}", "\n}").replace("{", "{\n ")
|
||||
|
||||
params_result = params_result.replace("\n", "\n ")
|
||||
print(params_result)
|
||||
|
||||
@staticmethod
|
||||
def _space_params(params, space: str, r: int = None) -> Dict:
|
||||
d = params[space]
|
||||
# Round floats to `r` digits after the decimal point if requested
|
||||
return round_dict(d, r) if r else d
|
||||
|
||||
@staticmethod
|
||||
def is_best_loss(results, current_best_loss: float) -> bool:
|
||||
return results['loss'] < current_best_loss
|
||||
result: Dict[str, Any] = {}
|
||||
strategy = self.backtesting.strategy
|
||||
if not HyperoptTools.has_space(self.config, 'roi'):
|
||||
result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()}
|
||||
if not HyperoptTools.has_space(self.config, 'stoploss'):
|
||||
result['stoploss'] = {'stoploss': strategy.stoploss}
|
||||
if not HyperoptTools.has_space(self.config, 'trailing'):
|
||||
result['trailing'] = {
|
||||
'trailing_stop': strategy.trailing_stop,
|
||||
'trailing_stop_positive': strategy.trailing_stop_positive,
|
||||
'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset,
|
||||
'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached,
|
||||
}
|
||||
return result
|
||||
|
||||
def print_results(self, results) -> None:
|
||||
"""
|
||||
Log results if it is better than any previous evaluation
|
||||
TODO: this should be moved to HyperoptTools too
|
||||
"""
|
||||
is_best = results['is_best']
|
||||
|
||||
if self.print_all or is_best:
|
||||
print(
|
||||
self.get_result_table(
|
||||
HyperoptTools.get_result_table(
|
||||
self.config, results, self.total_epochs,
|
||||
self.print_all, self.print_colorized,
|
||||
self.hyperopt_table_header
|
||||
@@ -303,231 +228,76 @@ class Hyperopt:
|
||||
)
|
||||
self.hyperopt_table_header = 2
|
||||
|
||||
@staticmethod
|
||||
def _format_explanation_string(results, total_epochs) -> str:
|
||||
return (("*" if results['is_initial_point'] else " ") +
|
||||
f"{results['current_epoch']:5d}/{total_epochs}: " +
|
||||
f"{results['results_explanation']} " +
|
||||
f"Objective: {results['loss']:.5f}")
|
||||
|
||||
@staticmethod
|
||||
def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
|
||||
print_colorized: bool, remove_header: int) -> str:
|
||||
def init_spaces(self):
|
||||
"""
|
||||
Log result table
|
||||
Assign the dimensions in the hyperoptimization space.
|
||||
"""
|
||||
if not results:
|
||||
return ''
|
||||
if HyperoptTools.has_space(self.config, 'protection'):
|
||||
# Protections can only be optimized when using the Parameter interface
|
||||
logger.debug("Hyperopt has 'protection' space")
|
||||
# Enable Protections if protection space is selected.
|
||||
self.config['enable_protections'] = True
|
||||
self.protection_space = self.custom_hyperopt.protection_space()
|
||||
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
|
||||
trials = json_normalize(results, max_level=1)
|
||||
trials['Best'] = ''
|
||||
if 'results_metrics.winsdrawslosses' not in trials.columns:
|
||||
# Ensure compatibility with older versions of hyperopt results
|
||||
trials['results_metrics.winsdrawslosses'] = 'N/A'
|
||||
|
||||
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
|
||||
'results_metrics.winsdrawslosses',
|
||||
'results_metrics.avg_profit', 'results_metrics.total_profit',
|
||||
'results_metrics.profit', 'results_metrics.duration',
|
||||
'loss', 'is_initial_point', 'is_best']]
|
||||
trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
|
||||
'Total profit', 'Profit', 'Avg duration', 'Objective',
|
||||
'is_initial_point', 'is_best']
|
||||
trials['is_profit'] = False
|
||||
trials.loc[trials['is_initial_point'], 'Best'] = '* '
|
||||
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
||||
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
|
||||
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
||||
trials['Trades'] = trials['Trades'].astype(str)
|
||||
|
||||
trials['Epoch'] = trials['Epoch'].apply(
|
||||
lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
|
||||
)
|
||||
trials['Avg profit'] = trials['Avg profit'].apply(
|
||||
lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
|
||||
)
|
||||
trials['Avg duration'] = trials['Avg duration'].apply(
|
||||
lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
|
||||
)
|
||||
trials['Objective'] = trials['Objective'].apply(
|
||||
lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
|
||||
)
|
||||
|
||||
trials['Profit'] = trials.apply(
|
||||
lambda x: '{:,.8f} {} {}'.format(
|
||||
x['Total profit'], config['stake_currency'],
|
||||
'({:,.2f}%)'.format(x['Profit']).rjust(10, ' ')
|
||||
).rjust(25+len(config['stake_currency']))
|
||||
if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])),
|
||||
axis=1
|
||||
)
|
||||
trials = trials.drop(columns=['Total profit'])
|
||||
|
||||
if print_colorized:
|
||||
for i in range(len(trials)):
|
||||
if trials.loc[i]['is_profit']:
|
||||
for j in range(len(trials.loc[i])-3):
|
||||
trials.iat[i, j] = "{}{}{}".format(Fore.GREEN,
|
||||
str(trials.loc[i][j]), Fore.RESET)
|
||||
if trials.loc[i]['is_best'] and highlight_best:
|
||||
for j in range(len(trials.loc[i])-3):
|
||||
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
|
||||
str(trials.loc[i][j]), Style.RESET_ALL)
|
||||
|
||||
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
||||
if remove_header > 0:
|
||||
table = tabulate.tabulate(
|
||||
trials.to_dict(orient='list'), tablefmt='orgtbl',
|
||||
headers='keys', stralign="right"
|
||||
)
|
||||
|
||||
table = table.split("\n", remove_header)[remove_header]
|
||||
elif remove_header < 0:
|
||||
table = tabulate.tabulate(
|
||||
trials.to_dict(orient='list'), tablefmt='psql',
|
||||
headers='keys', stralign="right"
|
||||
)
|
||||
table = "\n".join(table.split("\n")[0:remove_header])
|
||||
else:
|
||||
table = tabulate.tabulate(
|
||||
trials.to_dict(orient='list'), tablefmt='psql',
|
||||
headers='keys', stralign="right"
|
||||
)
|
||||
return table
|
||||
|
||||
@staticmethod
|
||||
def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
|
||||
csv_file: str) -> None:
|
||||
"""
|
||||
Log result to csv-file
|
||||
"""
|
||||
if not results:
|
||||
return
|
||||
|
||||
# Verification for overwrite
|
||||
if Path(csv_file).is_file():
|
||||
logger.error(f"CSV file already exists: {csv_file}")
|
||||
return
|
||||
|
||||
try:
|
||||
io.open(csv_file, 'w+').close()
|
||||
except IOError:
|
||||
logger.error(f"Failed to create CSV file: {csv_file}")
|
||||
return
|
||||
|
||||
trials = json_normalize(results, max_level=1)
|
||||
trials['Best'] = ''
|
||||
trials['Stake currency'] = config['stake_currency']
|
||||
|
||||
base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
|
||||
'results_metrics.avg_profit', 'results_metrics.median_profit',
|
||||
'results_metrics.total_profit',
|
||||
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
|
||||
'loss', 'is_initial_point', 'is_best']
|
||||
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
|
||||
trials = trials[base_metrics + param_metrics]
|
||||
|
||||
base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Median profit', 'Total profit',
|
||||
'Stake currency', 'Profit', 'Avg duration', 'Objective',
|
||||
'is_initial_point', 'is_best']
|
||||
param_columns = list(results[0]['params_dict'].keys())
|
||||
trials.columns = base_columns + param_columns
|
||||
|
||||
trials['is_profit'] = False
|
||||
trials.loc[trials['is_initial_point'], 'Best'] = '*'
|
||||
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
||||
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
|
||||
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
||||
trials['Epoch'] = trials['Epoch'].astype(str)
|
||||
trials['Trades'] = trials['Trades'].astype(str)
|
||||
|
||||
trials['Total profit'] = trials['Total profit'].apply(
|
||||
lambda x: '{:,.8f}'.format(x) if x != 0.0 else ""
|
||||
)
|
||||
trials['Profit'] = trials['Profit'].apply(
|
||||
lambda x: '{:,.2f}'.format(x) if not isna(x) else ""
|
||||
)
|
||||
trials['Avg profit'] = trials['Avg profit'].apply(
|
||||
lambda x: '{:,.2f}%'.format(x) if not isna(x) else ""
|
||||
)
|
||||
trials['Avg duration'] = trials['Avg duration'].apply(
|
||||
lambda x: '{:,.1f} m'.format(x) if not isna(x) else ""
|
||||
)
|
||||
trials['Objective'] = trials['Objective'].apply(
|
||||
lambda x: '{:,.5f}'.format(x) if x != 100000 else ""
|
||||
)
|
||||
|
||||
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
||||
trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8')
|
||||
logger.info(f"CSV file created: {csv_file}")
|
||||
|
||||
def has_space(self, space: str) -> bool:
|
||||
"""
|
||||
Tell if the space value is contained in the configuration
|
||||
"""
|
||||
# The 'trailing' space is not included in the 'default' set of spaces
|
||||
if space == 'trailing':
|
||||
return any(s in self.config['spaces'] for s in [space, 'all'])
|
||||
else:
|
||||
return any(s in self.config['spaces'] for s in [space, 'all', 'default'])
|
||||
|
||||
def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]:
|
||||
"""
|
||||
Return the dimensions in the hyperoptimization space.
|
||||
:param space: Defines hyperspace to return dimensions for.
|
||||
If None, then the self.has_space() will be used to return dimensions
|
||||
for all hyperspaces used.
|
||||
"""
|
||||
spaces: List[Dimension] = []
|
||||
|
||||
if space == 'buy' or (space is None and self.has_space('buy')):
|
||||
if HyperoptTools.has_space(self.config, 'buy'):
|
||||
logger.debug("Hyperopt has 'buy' space")
|
||||
spaces += self.custom_hyperopt.indicator_space()
|
||||
self.buy_space = self.custom_hyperopt.buy_indicator_space()
|
||||
|
||||
if space == 'sell' or (space is None and self.has_space('sell')):
|
||||
if HyperoptTools.has_space(self.config, 'sell'):
|
||||
logger.debug("Hyperopt has 'sell' space")
|
||||
spaces += self.custom_hyperopt.sell_indicator_space()
|
||||
self.sell_space = self.custom_hyperopt.sell_indicator_space()
|
||||
|
||||
if space == 'roi' or (space is None and self.has_space('roi')):
|
||||
if HyperoptTools.has_space(self.config, 'roi'):
|
||||
logger.debug("Hyperopt has 'roi' space")
|
||||
spaces += self.custom_hyperopt.roi_space()
|
||||
self.roi_space = self.custom_hyperopt.roi_space()
|
||||
|
||||
if space == 'stoploss' or (space is None and self.has_space('stoploss')):
|
||||
if HyperoptTools.has_space(self.config, 'stoploss'):
|
||||
logger.debug("Hyperopt has 'stoploss' space")
|
||||
spaces += self.custom_hyperopt.stoploss_space()
|
||||
self.stoploss_space = self.custom_hyperopt.stoploss_space()
|
||||
|
||||
if space == 'trailing' or (space is None and self.has_space('trailing')):
|
||||
if HyperoptTools.has_space(self.config, 'trailing'):
|
||||
logger.debug("Hyperopt has 'trailing' space")
|
||||
spaces += self.custom_hyperopt.trailing_space()
|
||||
self.trailing_space = self.custom_hyperopt.trailing_space()
|
||||
|
||||
return spaces
|
||||
self.dimensions = (self.buy_space + self.sell_space + self.protection_space
|
||||
+ self.roi_space + self.stoploss_space + self.trailing_space)
|
||||
|
||||
def assign_params(self, params_dict: Dict, category: str) -> None:
|
||||
"""
|
||||
Assign hyperoptable parameters
|
||||
"""
|
||||
for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category):
|
||||
if attr.optimize:
|
||||
# noinspection PyProtectedMember
|
||||
attr.value = params_dict[attr_name]
|
||||
|
||||
def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
|
||||
"""
|
||||
Used Optimize function. Called once per epoch to optimize whatever is configured.
|
||||
Used Optimize function.
|
||||
Called once per epoch to optimize whatever is configured.
|
||||
Keep this function as optimized as possible!
|
||||
"""
|
||||
params_dict = self._get_params_dict(raw_params)
|
||||
params_details = self._get_params_details(params_dict)
|
||||
backtest_start_time = datetime.now(timezone.utc)
|
||||
params_dict = self._get_params_dict(self.dimensions, raw_params)
|
||||
|
||||
if self.has_space('roi'):
|
||||
# Apply parameters
|
||||
if HyperoptTools.has_space(self.config, 'buy'):
|
||||
self.assign_params(params_dict, 'buy')
|
||||
|
||||
if HyperoptTools.has_space(self.config, 'sell'):
|
||||
self.assign_params(params_dict, 'sell')
|
||||
|
||||
if HyperoptTools.has_space(self.config, 'protection'):
|
||||
self.assign_params(params_dict, 'protection')
|
||||
|
||||
if HyperoptTools.has_space(self.config, 'roi'):
|
||||
self.backtesting.strategy.minimal_roi = ( # type: ignore
|
||||
self.custom_hyperopt.generate_roi_table(params_dict))
|
||||
|
||||
if self.has_space('buy'):
|
||||
self.backtesting.strategy.advise_buy = ( # type: ignore
|
||||
self.custom_hyperopt.buy_strategy_generator(params_dict))
|
||||
|
||||
if self.has_space('sell'):
|
||||
self.backtesting.strategy.advise_sell = ( # type: ignore
|
||||
self.custom_hyperopt.sell_strategy_generator(params_dict))
|
||||
|
||||
if self.has_space('stoploss'):
|
||||
if HyperoptTools.has_space(self.config, 'stoploss'):
|
||||
self.backtesting.strategy.stoploss = params_dict['stoploss']
|
||||
|
||||
if self.has_space('trailing'):
|
||||
if HyperoptTools.has_space(self.config, 'trailing'):
|
||||
d = self.custom_hyperopt.generate_trailing_params(params_dict)
|
||||
self.backtesting.strategy.trailing_stop = d['trailing_stop']
|
||||
self.backtesting.strategy.trailing_stop_positive = d['trailing_stop_positive']
|
||||
@@ -536,30 +306,43 @@ class Hyperopt:
|
||||
self.backtesting.strategy.trailing_only_offset_is_reached = \
|
||||
d['trailing_only_offset_is_reached']
|
||||
|
||||
processed = load(self.data_pickle_file)
|
||||
|
||||
min_date, max_date = get_timerange(processed)
|
||||
|
||||
backtesting_results = self.backtesting.backtest(
|
||||
with self.data_pickle_file.open('rb') as f:
|
||||
processed = load(f, mmap_mode='r')
|
||||
bt_results = self.backtesting.backtest(
|
||||
processed=processed,
|
||||
start_date=min_date.datetime,
|
||||
end_date=max_date.datetime,
|
||||
start_date=self.min_date,
|
||||
end_date=self.max_date,
|
||||
max_open_trades=self.max_open_trades,
|
||||
position_stacking=self.position_stacking,
|
||||
enable_protections=self.config.get('enable_protections', False),
|
||||
|
||||
)
|
||||
return self._get_results_dict(backtesting_results, min_date, max_date,
|
||||
params_dict, params_details,
|
||||
backtest_end_time = datetime.now(timezone.utc)
|
||||
bt_results.update({
|
||||
'backtest_start_time': int(backtest_start_time.timestamp()),
|
||||
'backtest_end_time': int(backtest_end_time.timestamp()),
|
||||
})
|
||||
|
||||
return self._get_results_dict(bt_results, self.min_date, self.max_date,
|
||||
params_dict,
|
||||
processed=processed)
|
||||
|
||||
def _get_results_dict(self, backtesting_results, min_date, max_date,
|
||||
params_dict, params_details, processed: Dict[str, DataFrame]):
|
||||
results_metrics = self._calculate_results_metrics(backtesting_results)
|
||||
results_explanation = self._format_results_explanation_string(results_metrics)
|
||||
params_dict, processed: Dict[str, DataFrame]
|
||||
) -> Dict[str, Any]:
|
||||
params_details = self._get_params_details(params_dict)
|
||||
|
||||
trade_count = results_metrics['trade_count']
|
||||
total_profit = results_metrics['total_profit']
|
||||
strat_stats = generate_strategy_stats(
|
||||
processed, self.backtesting.strategy.get_strategy_name(),
|
||||
backtesting_results, min_date, max_date, market_change=0
|
||||
)
|
||||
results_explanation = HyperoptTools.format_results_explanation_string(
|
||||
strat_stats, self.config['stake_currency'])
|
||||
|
||||
not_optimized = self.backtesting.strategy.get_no_optimize_params()
|
||||
not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details())
|
||||
|
||||
trade_count = strat_stats['total_trades']
|
||||
total_profit = strat_stats['profit_total']
|
||||
|
||||
# If this evaluation contains too short amount of trades to be
|
||||
# interesting -- consider it as 'bad' (assigned max. loss value)
|
||||
@@ -567,55 +350,36 @@ class Hyperopt:
|
||||
# path. We do not want to optimize 'hodl' strategies.
|
||||
loss: float = MAX_LOSS
|
||||
if trade_count >= self.config['hyperopt_min_trades']:
|
||||
loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count,
|
||||
min_date=min_date.datetime, max_date=max_date.datetime,
|
||||
config=self.config, processed=processed)
|
||||
loss = self.calculate_loss(results=backtesting_results['results'],
|
||||
trade_count=trade_count,
|
||||
min_date=min_date, max_date=max_date,
|
||||
config=self.config, processed=processed,
|
||||
backtest_stats=strat_stats)
|
||||
return {
|
||||
'loss': loss,
|
||||
'params_dict': params_dict,
|
||||
'params_details': params_details,
|
||||
'results_metrics': results_metrics,
|
||||
'params_not_optimized': not_optimized,
|
||||
'results_metrics': strat_stats,
|
||||
'results_explanation': results_explanation,
|
||||
'total_profit': total_profit,
|
||||
}
|
||||
|
||||
def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
|
||||
wins = len(backtesting_results[backtesting_results['profit_ratio'] > 0])
|
||||
draws = len(backtesting_results[backtesting_results['profit_ratio'] == 0])
|
||||
losses = len(backtesting_results[backtesting_results['profit_ratio'] < 0])
|
||||
return {
|
||||
'trade_count': len(backtesting_results.index),
|
||||
'wins': wins,
|
||||
'draws': draws,
|
||||
'losses': losses,
|
||||
'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}",
|
||||
'avg_profit': backtesting_results['profit_ratio'].mean() * 100.0,
|
||||
'median_profit': backtesting_results['profit_ratio'].median() * 100.0,
|
||||
'total_profit': backtesting_results['profit_abs'].sum(),
|
||||
'profit': backtesting_results['profit_ratio'].sum() * 100.0,
|
||||
'duration': backtesting_results['trade_duration'].mean(),
|
||||
}
|
||||
|
||||
def _format_results_explanation_string(self, results_metrics: Dict) -> str:
|
||||
"""
|
||||
Return the formatted results explanation in a string
|
||||
"""
|
||||
stake_cur = self.config['stake_currency']
|
||||
return (f"{results_metrics['trade_count']:6d} trades. "
|
||||
f"{results_metrics['wins']}/{results_metrics['draws']}"
|
||||
f"/{results_metrics['losses']} Wins/Draws/Losses. "
|
||||
f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
|
||||
f"Median profit {results_metrics['median_profit']: 6.2f}%. "
|
||||
f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
|
||||
f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
|
||||
f"Avg duration {results_metrics['duration']:5.1f} min."
|
||||
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
|
||||
|
||||
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
|
||||
estimator = self.custom_hyperopt.generate_estimator()
|
||||
|
||||
acq_optimizer = "sampling"
|
||||
if isinstance(estimator, str):
|
||||
if estimator not in ("GP", "RF", "ET", "GBRT"):
|
||||
raise OperationalException(f"Estimator {estimator} not supported.")
|
||||
else:
|
||||
acq_optimizer = "auto"
|
||||
|
||||
logger.info(f"Using estimator {estimator}.")
|
||||
return Optimizer(
|
||||
dimensions,
|
||||
base_estimator="ET",
|
||||
acq_optimizer="auto",
|
||||
base_estimator=estimator,
|
||||
acq_optimizer=acq_optimizer,
|
||||
n_initial_points=INITIAL_POINTS,
|
||||
acq_optimizer_kwargs={'n_jobs': cpu_count},
|
||||
random_state=self.random_state,
|
||||
@@ -626,43 +390,33 @@ class Hyperopt:
|
||||
return parallel(delayed(
|
||||
wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked)
|
||||
|
||||
@staticmethod
|
||||
def load_previous_results(results_file: Path) -> List:
|
||||
"""
|
||||
Load data for epochs from the file if we have one
|
||||
"""
|
||||
epochs: List = []
|
||||
if results_file.is_file() and results_file.stat().st_size > 0:
|
||||
epochs = Hyperopt._read_results(results_file)
|
||||
# Detection of some old format, without 'is_best' field saved
|
||||
if epochs[0].get('is_best') is None:
|
||||
raise OperationalException(
|
||||
"The file with Hyperopt results is incompatible with this version "
|
||||
"of Freqtrade and cannot be loaded.")
|
||||
logger.info(f"Loaded {len(epochs)} previous evaluations from disk.")
|
||||
return epochs
|
||||
|
||||
def _set_random_state(self, random_state: Optional[int]) -> int:
|
||||
return random_state or random.randint(1, 2**16 - 1)
|
||||
|
||||
def prepare_hyperopt_data(self) -> None:
|
||||
data, timerange = self.backtesting.load_bt_data()
|
||||
logger.info("Dataload complete. Calculating indicators")
|
||||
|
||||
preprocessed = self.backtesting.strategy.advise_all_indicators(data)
|
||||
|
||||
# Trim startup period from analyzed dataframe to get correct dates for output.
|
||||
processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
|
||||
self.min_date, self.max_date = get_timerange(processed)
|
||||
|
||||
logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'({(self.max_date - self.min_date).days} days)..')
|
||||
# Store non-trimmed data - will be trimmed after signal generation.
|
||||
dump(preprocessed, self.data_pickle_file)
|
||||
|
||||
def start(self) -> None:
|
||||
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
|
||||
logger.info(f"Using optimizer random state: {self.random_state}")
|
||||
self.hyperopt_table_header = -1
|
||||
data, timerange = self.backtesting.load_bt_data()
|
||||
# Initialize spaces ...
|
||||
self.init_spaces()
|
||||
|
||||
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
|
||||
|
||||
# Trim startup period from analyzed dataframe
|
||||
for pair, df in preprocessed.items():
|
||||
preprocessed[pair] = trim_dataframe(df, timerange)
|
||||
min_date, max_date = get_timerange(preprocessed)
|
||||
|
||||
logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
|
||||
f'({(max_date - min_date).days} days)..')
|
||||
|
||||
dump(preprocessed, self.data_pickle_file)
|
||||
self.prepare_hyperopt_data()
|
||||
|
||||
# We don't need exchange instance anymore while running hyperopt
|
||||
self.backtesting.exchange.close()
|
||||
@@ -670,15 +424,12 @@ class Hyperopt:
|
||||
self.backtesting.exchange._api_async = None # type: ignore
|
||||
# self.backtesting.exchange = None # type: ignore
|
||||
self.backtesting.pairlists = None # type: ignore
|
||||
self.backtesting.strategy.dp = None # type: ignore
|
||||
IStrategy.dp = None # type: ignore
|
||||
|
||||
cpus = cpu_count()
|
||||
logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
|
||||
config_jobs = self.config.get('hyperopt_jobs', -1)
|
||||
logger.info(f'Number of parallel jobs set as: {config_jobs}')
|
||||
|
||||
self.dimensions: List[Dimension] = self.hyperopt_space()
|
||||
self.opt = self.get_optimizer(self.dimensions, config_jobs)
|
||||
|
||||
if self.print_colorized:
|
||||
@@ -711,9 +462,9 @@ class Hyperopt:
|
||||
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
|
||||
]
|
||||
with progressbar.ProgressBar(
|
||||
max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
|
||||
widgets=widgets
|
||||
) as pbar:
|
||||
max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
|
||||
widgets=widgets
|
||||
) as pbar:
|
||||
EVALS = ceil(self.total_epochs / jobs)
|
||||
for i in range(EVALS):
|
||||
# Correct the number of epochs to be processed for the last
|
||||
@@ -734,7 +485,7 @@ class Hyperopt:
|
||||
|
||||
logger.debug(f"Optimizer epoch evaluated: {val}")
|
||||
|
||||
is_best = self.is_best_loss(val, self.current_best_loss)
|
||||
is_best = HyperoptTools.is_best_loss(val, self.current_best_loss)
|
||||
# This value is assigned here and not in the optimization method
|
||||
# to keep proper order in the list of results. That's because
|
||||
# evaluations can take different time. Here they are aligned in the
|
||||
@@ -744,25 +495,26 @@ class Hyperopt:
|
||||
|
||||
if is_best:
|
||||
self.current_best_loss = val['loss']
|
||||
self.epochs.append(val)
|
||||
self.current_best_epoch = val
|
||||
|
||||
# Save results after each best epoch and every 100 epochs
|
||||
if is_best or current % 100 == 0:
|
||||
self._save_results()
|
||||
self._save_result(val)
|
||||
|
||||
pbar.update(current)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print('User interrupted..')
|
||||
|
||||
self._save_results()
|
||||
logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
|
||||
f"saved to '{self.results_file}'.")
|
||||
|
||||
if self.epochs:
|
||||
sorted_epochs = sorted(self.epochs, key=itemgetter('loss'))
|
||||
best_epoch = sorted_epochs[0]
|
||||
self.print_epoch_details(best_epoch, self.total_epochs, self.print_json)
|
||||
if self.current_best_epoch:
|
||||
HyperoptTools.try_export_params(
|
||||
self.config,
|
||||
self.backtesting.strategy.get_strategy_name(),
|
||||
self.current_best_epoch)
|
||||
|
||||
HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
|
||||
self.print_json)
|
||||
else:
|
||||
# This is printed when Ctrl+C is pressed quickly, before first epochs have
|
||||
# a chance to be evaluated.
|
||||
|
95
freqtrade/optimize/hyperopt_auto.py
Normal file
95
freqtrade/optimize/hyperopt_auto.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
HyperOptAuto class.
|
||||
This module implements a convenience auto-hyperopt class, which can be used together with strategies
|
||||
that implement IHyperStrategy interface.
|
||||
"""
|
||||
import logging
|
||||
from contextlib import suppress
|
||||
from typing import Callable, Dict, List
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
|
||||
|
||||
with suppress(ImportError):
|
||||
from skopt.space import Dimension
|
||||
|
||||
from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _format_exception_message(space: str, ignore_missing_space: bool) -> None:
|
||||
msg = (f"The '{space}' space is included into the hyperoptimization "
|
||||
f"but no parameter for this space was not found in your Strategy. "
|
||||
)
|
||||
if ignore_missing_space:
|
||||
logger.warning(msg + "This space will be ignored.")
|
||||
else:
|
||||
raise OperationalException(
|
||||
msg + f"Please make sure to have parameters for this space enabled for optimization "
|
||||
f"or remove the '{space}' space from hyperoptimization.")
|
||||
|
||||
|
||||
class HyperOptAuto(IHyperOpt):
|
||||
"""
|
||||
This class delegates functionality to Strategy(IHyperStrategy) and Strategy.HyperOpt classes.
|
||||
Most of the time Strategy.HyperOpt class would only implement indicator_space and
|
||||
sell_indicator_space methods, but other hyperopt methods can be overridden as well.
|
||||
"""
|
||||
|
||||
def _get_func(self, name) -> Callable:
|
||||
"""
|
||||
Return a function defined in Strategy.HyperOpt class, or one defined in super() class.
|
||||
:param name: function name.
|
||||
:return: a requested function.
|
||||
"""
|
||||
hyperopt_cls = getattr(self.strategy, 'HyperOpt', None)
|
||||
default_func = getattr(super(), name)
|
||||
if hyperopt_cls:
|
||||
return getattr(hyperopt_cls, name, default_func)
|
||||
else:
|
||||
return default_func
|
||||
|
||||
def _generate_indicator_space(self, category):
|
||||
for attr_name, attr in self.strategy.enumerate_parameters(category):
|
||||
if attr.optimize:
|
||||
yield attr.get_space(attr_name)
|
||||
|
||||
def _get_indicator_space(self, category) -> List:
|
||||
# TODO: is this necessary, or can we call "generate_space" directly?
|
||||
indicator_space = list(self._generate_indicator_space(category))
|
||||
if len(indicator_space) > 0:
|
||||
return indicator_space
|
||||
else:
|
||||
_format_exception_message(
|
||||
category,
|
||||
self.config.get("hyperopt_ignore_missing_space", False))
|
||||
return []
|
||||
|
||||
def buy_indicator_space(self) -> List['Dimension']:
|
||||
return self._get_indicator_space('buy')
|
||||
|
||||
def sell_indicator_space(self) -> List['Dimension']:
|
||||
return self._get_indicator_space('sell')
|
||||
|
||||
def protection_space(self) -> List['Dimension']:
|
||||
return self._get_indicator_space('protection')
|
||||
|
||||
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
|
||||
return self._get_func('generate_roi_table')(params)
|
||||
|
||||
def roi_space(self) -> List['Dimension']:
|
||||
return self._get_func('roi_space')()
|
||||
|
||||
def stoploss_space(self) -> List['Dimension']:
|
||||
return self._get_func('stoploss_space')()
|
||||
|
||||
def generate_trailing_params(self, params: Dict) -> Dict:
|
||||
return self._get_func('generate_trailing_params')(params)
|
||||
|
||||
def trailing_space(self) -> List['Dimension']:
|
||||
return self._get_func('trailing_space')()
|
||||
|
||||
def generate_estimator(self) -> EstimatorType:
|
||||
return self._get_func('generate_estimator')()
|
128
freqtrade/optimize/hyperopt_epoch_filters.py
Normal file
128
freqtrade/optimize/hyperopt_epoch_filters.py
Normal file
@@ -0,0 +1,128 @@
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) -> List:
|
||||
"""
|
||||
Filter our items from the list of hyperopt results
|
||||
"""
|
||||
if filteroptions['only_best']:
|
||||
epochs = [x for x in epochs if x['is_best']]
|
||||
if filteroptions['only_profitable']:
|
||||
epochs = [x for x in epochs
|
||||
if x['results_metrics'].get('profit_total', 0) > 0]
|
||||
|
||||
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
|
||||
|
||||
epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
|
||||
if log:
|
||||
logger.info(f"{len(epochs)} " +
|
||||
("best " if filteroptions['only_best'] else "") +
|
||||
("profitable " if filteroptions['only_profitable'] else "") +
|
||||
"epochs found.")
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
|
||||
"""
|
||||
Filter epochs with trade-counts > trades
|
||||
"""
|
||||
return [
|
||||
x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count
|
||||
]
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_trades'] > 0:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
|
||||
|
||||
if filteroptions['filter_max_trades'] > 0:
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades']
|
||||
]
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
def get_duration_value(x):
|
||||
# Duration in minutes ...
|
||||
if 'holding_avg_s' in x['results_metrics']:
|
||||
avg = x['results_metrics']['holding_avg_s']
|
||||
return avg // 60
|
||||
raise OperationalException(
|
||||
"Holding-average not available. Please omit the filter on average time, "
|
||||
"or rerun hyperopt with this version")
|
||||
|
||||
if filteroptions['filter_min_avg_time'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if get_duration_value(x) > filteroptions['filter_min_avg_time']
|
||||
]
|
||||
if filteroptions['filter_max_avg_time'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if get_duration_value(x) < filteroptions['filter_max_avg_time']
|
||||
]
|
||||
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_avg_profit'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics'].get('profit_mean', 0) * 100
|
||||
> filteroptions['filter_min_avg_profit']
|
||||
]
|
||||
if filteroptions['filter_max_avg_profit'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics'].get('profit_mean', 0) * 100
|
||||
< filteroptions['filter_max_avg_profit']
|
||||
]
|
||||
if filteroptions['filter_min_total_profit'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics'].get('profit_total_abs', 0)
|
||||
> filteroptions['filter_min_total_profit']
|
||||
]
|
||||
if filteroptions['filter_max_total_profit'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
epochs = [
|
||||
x for x in epochs
|
||||
if x['results_metrics'].get('profit_total_abs', 0)
|
||||
< filteroptions['filter_max_total_profit']
|
||||
]
|
||||
return epochs
|
||||
|
||||
|
||||
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
|
||||
|
||||
if filteroptions['filter_min_objective'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
|
||||
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
|
||||
if filteroptions['filter_max_objective'] is not None:
|
||||
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
|
||||
|
||||
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
|
||||
|
||||
return epochs
|
@@ -5,24 +5,20 @@ This module defines the interface to apply for hyperopt
|
||||
import logging
|
||||
import math
|
||||
from abc import ABC
|
||||
from typing import Any, Callable, Dict, List
|
||||
from typing import Dict, List, Union
|
||||
|
||||
from skopt.space import Categorical, Dimension, Integer, Real
|
||||
from sklearn.base import RegressorMixin
|
||||
from skopt.space import Categorical, Dimension, Integer
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import timeframe_to_minutes
|
||||
from freqtrade.misc import round_dict
|
||||
from freqtrade.optimize.space import SKDecimal
|
||||
from freqtrade.strategy import IStrategy
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _format_exception_message(method: str, space: str) -> str:
|
||||
return (f"The '{space}' space is included into the hyperoptimization "
|
||||
f"but {method}() method is not found in your "
|
||||
f"custom Hyperopt class. You should either implement this "
|
||||
f"method or remove the '{space}' space from hyperoptimization.")
|
||||
EstimatorType = Union[RegressorMixin, str]
|
||||
|
||||
|
||||
class IHyperOpt(ABC):
|
||||
@@ -31,7 +27,7 @@ class IHyperOpt(ABC):
|
||||
Defines the mandatory structure must follow any custom hyperopt
|
||||
|
||||
Class attributes you can use:
|
||||
ticker_interval -> int: value of the ticker interval to use for the strategy
|
||||
timeframe -> int: value of the timeframe to use for the strategy
|
||||
"""
|
||||
ticker_interval: str # DEPRECATED
|
||||
timeframe: str
|
||||
@@ -44,36 +40,15 @@ class IHyperOpt(ABC):
|
||||
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
|
||||
IHyperOpt.timeframe = str(config['timeframe'])
|
||||
|
||||
@staticmethod
|
||||
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
|
||||
def generate_estimator(self) -> EstimatorType:
|
||||
"""
|
||||
Create a buy strategy generator.
|
||||
Return base_estimator.
|
||||
Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
|
||||
inheriting from RegressorMixin (from sklearn).
|
||||
"""
|
||||
raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy'))
|
||||
return 'ET'
|
||||
|
||||
@staticmethod
|
||||
def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
|
||||
"""
|
||||
Create a sell strategy generator.
|
||||
"""
|
||||
raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
|
||||
|
||||
@staticmethod
|
||||
def indicator_space() -> List[Dimension]:
|
||||
"""
|
||||
Create an indicator space.
|
||||
"""
|
||||
raise OperationalException(_format_exception_message('indicator_space', 'buy'))
|
||||
|
||||
@staticmethod
|
||||
def sell_indicator_space() -> List[Dimension]:
|
||||
"""
|
||||
Create a sell indicator space.
|
||||
"""
|
||||
raise OperationalException(_format_exception_message('sell_indicator_space', 'sell'))
|
||||
|
||||
@staticmethod
|
||||
def generate_roi_table(params: Dict) -> Dict[int, float]:
|
||||
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
|
||||
"""
|
||||
Create a ROI table.
|
||||
|
||||
@@ -88,8 +63,7 @@ class IHyperOpt(ABC):
|
||||
|
||||
return roi_table
|
||||
|
||||
@staticmethod
|
||||
def roi_space() -> List[Dimension]:
|
||||
def roi_space(self) -> List[Dimension]:
|
||||
"""
|
||||
Create a ROI space.
|
||||
|
||||
@@ -97,7 +71,7 @@ class IHyperOpt(ABC):
|
||||
|
||||
This method implements adaptive roi hyperspace with varied
|
||||
ranges for parameters which automatically adapts to the
|
||||
ticker interval used.
|
||||
timeframe used.
|
||||
|
||||
It's used by Freqtrade by default, if no custom roi_space method is defined.
|
||||
"""
|
||||
@@ -109,7 +83,7 @@ class IHyperOpt(ABC):
|
||||
roi_t_alpha = 1.0
|
||||
roi_p_alpha = 1.0
|
||||
|
||||
timeframe_min = timeframe_to_minutes(IHyperOpt.ticker_interval)
|
||||
timeframe_min = timeframe_to_minutes(self.timeframe)
|
||||
|
||||
# We define here limits for the ROI space parameters automagically adapted to the
|
||||
# timeframe used by the bot:
|
||||
@@ -119,7 +93,7 @@ class IHyperOpt(ABC):
|
||||
# * 'roi_p' (limits for the ROI value steps) components are scaled logarithmically.
|
||||
#
|
||||
# The scaling is designed so that it maps exactly to the legacy Freqtrade roi_space()
|
||||
# method for the 5m ticker interval.
|
||||
# method for the 5m timeframe.
|
||||
roi_t_scale = timeframe_min / 5
|
||||
roi_p_scale = math.log1p(timeframe_min) / math.log1p(5)
|
||||
roi_limits = {
|
||||
@@ -145,7 +119,7 @@ class IHyperOpt(ABC):
|
||||
'roi_p2': roi_limits['roi_p2_min'],
|
||||
'roi_p3': roi_limits['roi_p3_min'],
|
||||
}
|
||||
logger.info(f"Min roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}")
|
||||
logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 3)}")
|
||||
p = {
|
||||
'roi_t1': roi_limits['roi_t1_max'],
|
||||
'roi_t2': roi_limits['roi_t2_max'],
|
||||
@@ -154,19 +128,21 @@ class IHyperOpt(ABC):
|
||||
'roi_p2': roi_limits['roi_p2_max'],
|
||||
'roi_p3': roi_limits['roi_p3_max'],
|
||||
}
|
||||
logger.info(f"Max roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}")
|
||||
logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 3)}")
|
||||
|
||||
return [
|
||||
Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'),
|
||||
Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'),
|
||||
Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'),
|
||||
Real(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], name='roi_p1'),
|
||||
Real(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], name='roi_p2'),
|
||||
Real(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], name='roi_p3'),
|
||||
SKDecimal(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], decimals=3,
|
||||
name='roi_p1'),
|
||||
SKDecimal(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], decimals=3,
|
||||
name='roi_p2'),
|
||||
SKDecimal(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], decimals=3,
|
||||
name='roi_p3'),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def stoploss_space() -> List[Dimension]:
|
||||
def stoploss_space(self) -> List[Dimension]:
|
||||
"""
|
||||
Create a stoploss space.
|
||||
|
||||
@@ -174,11 +150,10 @@ class IHyperOpt(ABC):
|
||||
You may override it in your custom Hyperopt class.
|
||||
"""
|
||||
return [
|
||||
Real(-0.35, -0.02, name='stoploss'),
|
||||
SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def generate_trailing_params(params: Dict) -> Dict:
|
||||
def generate_trailing_params(self, params: Dict) -> Dict:
|
||||
"""
|
||||
Create dict with trailing stop parameters.
|
||||
"""
|
||||
@@ -190,8 +165,7 @@ class IHyperOpt(ABC):
|
||||
'trailing_only_offset_is_reached': params['trailing_only_offset_is_reached'],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def trailing_space() -> List[Dimension]:
|
||||
def trailing_space(self) -> List[Dimension]:
|
||||
"""
|
||||
Create a trailing stoploss space.
|
||||
|
||||
@@ -206,14 +180,14 @@ class IHyperOpt(ABC):
|
||||
# other 'trailing' hyperspace parameters.
|
||||
Categorical([True], name='trailing_stop'),
|
||||
|
||||
Real(0.01, 0.35, name='trailing_stop_positive'),
|
||||
SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
|
||||
|
||||
# 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
|
||||
# so this intermediate parameter is used as the value of the difference between
|
||||
# them. The value of the 'trailing_stop_positive_offset' is constructed in the
|
||||
# generate_trailing_params() method.
|
||||
# This is similar to the hyperspace dimensions used for constructing the ROI tables.
|
||||
Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'),
|
||||
SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
|
||||
|
||||
Categorical([True, False], name='trailing_only_offset_is_reached'),
|
||||
]
|
||||
|
@@ -5,7 +5,7 @@ This module defines the interface for the loss-function for hyperopt
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Dict
|
||||
from typing import Any, Dict
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
@@ -22,6 +22,7 @@ class IHyperOptLoss(ABC):
|
||||
def hyperopt_loss_function(results: DataFrame, trade_count: int,
|
||||
min_date: datetime, max_date: datetime,
|
||||
config: Dict, processed: Dict[str, DataFrame],
|
||||
backtest_stats: Dict[str, Any],
|
||||
*args, **kwargs) -> float:
|
||||
"""
|
||||
Objective function, returns smaller number for better results
|
||||
|
41
freqtrade/optimize/hyperopt_loss_max_drawdown.py
Normal file
41
freqtrade/optimize/hyperopt_loss_max_drawdown.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
MaxDrawDownHyperOptLoss
|
||||
|
||||
This module defines the alternative HyperOptLoss class which can be used for
|
||||
Hyperoptimization.
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.data.btanalysis import calculate_max_drawdown
|
||||
from freqtrade.optimize.hyperopt import IHyperOptLoss
|
||||
|
||||
|
||||
class MaxDrawDownHyperOptLoss(IHyperOptLoss):
|
||||
|
||||
"""
|
||||
Defines the loss function for hyperopt.
|
||||
|
||||
This implementation optimizes for max draw down and profit
|
||||
Less max drawdown more profit -> Lower return value
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def hyperopt_loss_function(results: DataFrame, trade_count: int,
|
||||
min_date: datetime, max_date: datetime,
|
||||
*args, **kwargs) -> float:
|
||||
|
||||
"""
|
||||
Objective function.
|
||||
|
||||
Uses profit ratio weighted max_drawdown when drawdown is available.
|
||||
Otherwise directly optimizes profit ratio.
|
||||
"""
|
||||
total_profit = results['profit_abs'].sum()
|
||||
try:
|
||||
max_drawdown = calculate_max_drawdown(results, value_col='profit_abs')
|
||||
except ValueError:
|
||||
# No losing trade, therefore no drawdown.
|
||||
return -total_profit
|
||||
return -total_profit / max_drawdown[0]
|
@@ -9,23 +9,11 @@ from pandas import DataFrame
|
||||
from freqtrade.optimize.hyperopt import IHyperOptLoss
|
||||
|
||||
|
||||
# This is assumed to be expected avg profit * expected trade count.
|
||||
# For example, for 0.35% avg per trade (or 0.0035 as ratio) and 1100 trades,
|
||||
# expected max profit = 3.85
|
||||
#
|
||||
# Note, this is ratio. 3.85 stated above means 385Σ%, 3.0 means 300Σ%.
|
||||
#
|
||||
# In this implementation it's only used in calculation of the resulting value
|
||||
# of the objective function as a normalization coefficient and does not
|
||||
# represent any limit for profits as in the Freqtrade legacy default loss function.
|
||||
EXPECTED_MAX_PROFIT = 3.0
|
||||
|
||||
|
||||
class OnlyProfitHyperOptLoss(IHyperOptLoss):
|
||||
"""
|
||||
Defines the loss function for hyperopt.
|
||||
|
||||
This implementation takes only profit into account.
|
||||
This implementation takes only absolute profit into account, not looking at any other indicator.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@@ -34,5 +22,5 @@ class OnlyProfitHyperOptLoss(IHyperOptLoss):
|
||||
"""
|
||||
Objective function, returns smaller number for better results.
|
||||
"""
|
||||
total_profit = results['profit_ratio'].sum()
|
||||
return 1 - total_profit / EXPECTED_MAX_PROFIT
|
||||
total_profit = results['profit_abs'].sum()
|
||||
return -1 * total_profit
|
||||
|
502
freqtrade/optimize/hyperopt_tools.py
Executable file
502
freqtrade/optimize/hyperopt_tools.py
Executable file
@@ -0,0 +1,502 @@
|
||||
|
||||
import io
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import rapidjson
|
||||
import tabulate
|
||||
from colorama import Fore, Style
|
||||
from pandas import isna, json_normalize
|
||||
|
||||
from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
|
||||
from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NON_OPT_PARAM_APPENDIX = " # value loaded from strategy"
|
||||
|
||||
|
||||
def hyperopt_serializer(x):
|
||||
if isinstance(x, np.integer):
|
||||
return int(x)
|
||||
if isinstance(x, np.bool_):
|
||||
return bool(x)
|
||||
|
||||
return str(x)
|
||||
|
||||
|
||||
class HyperoptTools():
|
||||
|
||||
@staticmethod
|
||||
def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]:
|
||||
"""
|
||||
Get Strategy-location (filename) from strategy_name
|
||||
"""
|
||||
from freqtrade.resolvers.strategy_resolver import StrategyResolver
|
||||
directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
|
||||
strategy_objs = StrategyResolver.search_all_objects(directory, False)
|
||||
strategies = [s for s in strategy_objs if s['name'] == strategy_name]
|
||||
if strategies:
|
||||
strategy = strategies[0]
|
||||
|
||||
return Path(strategy['location'])
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def export_params(params, strategy_name: str, filename: Path):
|
||||
"""
|
||||
Generate files
|
||||
"""
|
||||
final_params = deepcopy(params['params_not_optimized'])
|
||||
final_params = deep_merge_dicts(params['params_details'], final_params)
|
||||
final_params = {
|
||||
'strategy_name': strategy_name,
|
||||
'params': final_params,
|
||||
'ft_stratparam_v': 1,
|
||||
'export_time': datetime.now(timezone.utc),
|
||||
}
|
||||
logger.info(f"Dumping parameters to {filename}")
|
||||
rapidjson.dump(final_params, filename.open('w'), indent=2,
|
||||
default=hyperopt_serializer,
|
||||
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict):
|
||||
if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False):
|
||||
# Export parameters ...
|
||||
fn = HyperoptTools.get_strategy_filename(config, strategy_name)
|
||||
if fn:
|
||||
HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json'))
|
||||
else:
|
||||
logger.warning("Strategy not found, not exporting parameter file.")
|
||||
|
||||
@staticmethod
|
||||
def has_space(config: Dict[str, Any], space: str) -> bool:
|
||||
"""
|
||||
Tell if the space value is contained in the configuration
|
||||
"""
|
||||
# 'trailing' and 'protection spaces are not included in the 'default' set of spaces
|
||||
if space in ('trailing', 'protection'):
|
||||
return any(s in config['spaces'] for s in [space, 'all'])
|
||||
else:
|
||||
return any(s in config['spaces'] for s in [space, 'all', 'default'])
|
||||
|
||||
@staticmethod
|
||||
def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]:
|
||||
"""
|
||||
Stream hyperopt results from file
|
||||
"""
|
||||
import rapidjson
|
||||
logger.info(f"Reading epochs from '{results_file}'")
|
||||
with results_file.open('r') as f:
|
||||
data = []
|
||||
for line in f:
|
||||
data += [rapidjson.loads(line)]
|
||||
if len(data) >= batch_size:
|
||||
yield data
|
||||
data = []
|
||||
yield data
|
||||
|
||||
@staticmethod
|
||||
def _test_hyperopt_results_exist(results_file) -> bool:
|
||||
if results_file.is_file() and results_file.stat().st_size > 0:
|
||||
if results_file.suffix == '.pickle':
|
||||
raise OperationalException(
|
||||
"Legacy hyperopt results are no longer supported."
|
||||
"Please rerun hyperopt or use an older version to load this file."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
# No file found.
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]:
|
||||
filteroptions = {
|
||||
'only_best': config.get('hyperopt_list_best', False),
|
||||
'only_profitable': config.get('hyperopt_list_profitable', False),
|
||||
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
|
||||
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
|
||||
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
|
||||
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
|
||||
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
|
||||
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
|
||||
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
|
||||
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
|
||||
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
|
||||
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
|
||||
}
|
||||
if not HyperoptTools._test_hyperopt_results_exist(results_file):
|
||||
# No file found.
|
||||
return [], 0
|
||||
|
||||
epochs = []
|
||||
total_epochs = 0
|
||||
for epochs_tmp in HyperoptTools._read_results(results_file):
|
||||
if total_epochs == 0 and epochs_tmp[0].get('is_best') is None:
|
||||
raise OperationalException(
|
||||
"The file with HyperoptTools results is incompatible with this version "
|
||||
"of Freqtrade and cannot be loaded.")
|
||||
total_epochs += len(epochs_tmp)
|
||||
epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False)
|
||||
|
||||
logger.info(f"Loaded {total_epochs} previous evaluations from disk.")
|
||||
|
||||
# Final filter run ...
|
||||
epochs = hyperopt_filter_epochs(epochs, filteroptions, log=True)
|
||||
|
||||
return epochs, total_epochs
|
||||
|
||||
@staticmethod
|
||||
def show_epoch_details(results, total_epochs: int, print_json: bool,
|
||||
no_header: bool = False, header_str: str = None) -> None:
|
||||
"""
|
||||
Display details of the hyperopt result
|
||||
"""
|
||||
params = results.get('params_details', {})
|
||||
non_optimized = results.get('params_not_optimized', {})
|
||||
|
||||
# Default header string
|
||||
if header_str is None:
|
||||
header_str = "Best result"
|
||||
|
||||
if not no_header:
|
||||
explanation_str = HyperoptTools._format_explanation_string(results, total_epochs)
|
||||
print(f"\n{header_str}:\n\n{explanation_str}\n")
|
||||
|
||||
if print_json:
|
||||
result_dict: Dict = {}
|
||||
for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing']:
|
||||
HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s)
|
||||
print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
|
||||
|
||||
else:
|
||||
HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:",
|
||||
non_optimized)
|
||||
HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
|
||||
non_optimized)
|
||||
HyperoptTools._params_pretty_print(params, 'protection',
|
||||
"Protection hyperspace params:", non_optimized)
|
||||
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized)
|
||||
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized)
|
||||
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized)
|
||||
|
||||
@staticmethod
|
||||
def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None:
|
||||
if (space in params) or (space in non_optimized):
|
||||
space_params = HyperoptTools._space_params(params, space)
|
||||
space_non_optimized = HyperoptTools._space_params(non_optimized, space)
|
||||
all_space_params = space_params
|
||||
|
||||
# Merge non optimized params if there are any
|
||||
if len(space_non_optimized) > 0:
|
||||
all_space_params = {**space_params, **space_non_optimized}
|
||||
|
||||
if space in ['buy', 'sell']:
|
||||
result_dict.setdefault('params', {}).update(all_space_params)
|
||||
elif space == 'roi':
|
||||
# Convert keys in min_roi dict to strings because
|
||||
# rapidjson cannot dump dicts with integer keys...
|
||||
result_dict['minimal_roi'] = {str(k): v for k, v in all_space_params.items()}
|
||||
else: # 'stoploss', 'trailing'
|
||||
result_dict.update(all_space_params)
|
||||
|
||||
@staticmethod
|
||||
def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
|
||||
if space in params or space in non_optimized:
|
||||
space_params = HyperoptTools._space_params(params, space, 5)
|
||||
no_params = HyperoptTools._space_params(non_optimized, space, 5)
|
||||
appendix = ''
|
||||
if not space_params and not no_params:
|
||||
# No parameters - don't print
|
||||
return
|
||||
if not space_params:
|
||||
# Not optimized parameters - append string
|
||||
appendix = NON_OPT_PARAM_APPENDIX
|
||||
|
||||
result = f"\n# {header}\n"
|
||||
if space == "stoploss":
|
||||
stoploss = safe_value_fallback2(space_params, no_params, space, space)
|
||||
result += (f"stoploss = {stoploss}{appendix}")
|
||||
|
||||
elif space == "roi":
|
||||
result = result[:-1] + f'{appendix}\n'
|
||||
minimal_roi_result = rapidjson.dumps({
|
||||
str(k): v for k, v in (space_params or no_params).items()
|
||||
}, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
|
||||
result += f"minimal_roi = {minimal_roi_result}"
|
||||
elif space == "trailing":
|
||||
for k, v in (space_params or no_params).items():
|
||||
result += f"{k} = {v}{appendix}\n"
|
||||
|
||||
else:
|
||||
# Buy / sell parameters
|
||||
|
||||
result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}"
|
||||
|
||||
result = result.replace("\n", "\n ")
|
||||
print(result)
|
||||
|
||||
@staticmethod
|
||||
def _space_params(params, space: str, r: int = None) -> Dict:
|
||||
d = params.get(space)
|
||||
if d:
|
||||
# Round floats to `r` digits after the decimal point if requested
|
||||
return round_dict(d, r) if r else d
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _pprint_dict(params, non_optimized, indent: int = 4):
|
||||
"""
|
||||
Pretty-print hyperopt results (based on 2 dicts - with add. comment)
|
||||
"""
|
||||
p = params.copy()
|
||||
p.update(non_optimized)
|
||||
result = '{\n'
|
||||
|
||||
for k, param in p.items():
|
||||
result += " " * indent + f'"{k}": '
|
||||
result += f'"{param}",' if isinstance(param, str) else f'{param},'
|
||||
if k in non_optimized:
|
||||
result += NON_OPT_PARAM_APPENDIX
|
||||
result += "\n"
|
||||
result += '}'
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def is_best_loss(results, current_best_loss: float) -> bool:
|
||||
return bool(results['loss'] < current_best_loss)
|
||||
|
||||
@staticmethod
|
||||
def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str:
|
||||
"""
|
||||
Return the formatted results explanation in a string
|
||||
"""
|
||||
return (f"{results_metrics['total_trades']:6d} trades. "
|
||||
f"{results_metrics['wins']}/{results_metrics['draws']}"
|
||||
f"/{results_metrics['losses']} Wins/Draws/Losses. "
|
||||
f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. "
|
||||
f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. "
|
||||
f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} "
|
||||
f"({results_metrics['profit_total'] * 100: 7.2f}%). "
|
||||
f"Avg duration {results_metrics['holding_avg']} min."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _format_explanation_string(results, total_epochs) -> str:
|
||||
return (("*" if results['is_initial_point'] else " ") +
|
||||
f"{results['current_epoch']:5d}/{total_epochs}: " +
|
||||
f"{results['results_explanation']} " +
|
||||
f"Objective: {results['loss']:.5f}")
|
||||
|
||||
@staticmethod
|
||||
def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool,
|
||||
has_drawdown: bool) -> pd.DataFrame:
|
||||
trials['Best'] = ''
|
||||
|
||||
if 'results_metrics.winsdrawslosses' not in trials.columns:
|
||||
# Ensure compatibility with older versions of hyperopt results
|
||||
trials['results_metrics.winsdrawslosses'] = 'N/A'
|
||||
|
||||
if not has_drawdown:
|
||||
# Ensure compatibility with older versions of hyperopt results
|
||||
trials['results_metrics.max_drawdown_abs'] = None
|
||||
trials['results_metrics.max_drawdown'] = None
|
||||
|
||||
if not legacy_mode:
|
||||
# New mode, using backtest result for metrics
|
||||
trials['results_metrics.winsdrawslosses'] = trials.apply(
|
||||
lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
|
||||
f"{x['results_metrics.losses']:>4}", axis=1)
|
||||
trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
|
||||
'results_metrics.winsdrawslosses',
|
||||
'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
|
||||
'results_metrics.profit_total', 'results_metrics.holding_avg',
|
||||
'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs',
|
||||
'loss', 'is_initial_point', 'is_best']]
|
||||
|
||||
else:
|
||||
# Legacy mode
|
||||
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
|
||||
'results_metrics.winsdrawslosses', 'results_metrics.avg_profit',
|
||||
'results_metrics.total_profit', 'results_metrics.profit',
|
||||
'results_metrics.duration', 'results_metrics.max_drawdown',
|
||||
'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point',
|
||||
'is_best']]
|
||||
|
||||
trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
|
||||
'Total profit', 'Profit', 'Avg duration', 'Max Drawdown',
|
||||
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best']
|
||||
|
||||
return trials
|
||||
|
||||
@staticmethod
|
||||
def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool,
|
||||
print_colorized: bool, remove_header: int) -> str:
|
||||
"""
|
||||
Log result table
|
||||
"""
|
||||
if not results:
|
||||
return ''
|
||||
|
||||
tabulate.PRESERVE_WHITESPACE = True
|
||||
trials = json_normalize(results, max_level=1)
|
||||
|
||||
legacy_mode = 'results_metrics.total_trades' not in trials
|
||||
has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns
|
||||
|
||||
trials = HyperoptTools.prepare_trials_columns(trials, legacy_mode, has_drawdown)
|
||||
|
||||
trials['is_profit'] = False
|
||||
trials.loc[trials['is_initial_point'], 'Best'] = '* '
|
||||
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
||||
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
|
||||
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
||||
trials['Trades'] = trials['Trades'].astype(str)
|
||||
perc_multi = 1 if legacy_mode else 100
|
||||
trials['Epoch'] = trials['Epoch'].apply(
|
||||
lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
|
||||
)
|
||||
trials['Avg profit'] = trials['Avg profit'].apply(
|
||||
lambda x: f'{x * perc_multi:,.2f}%'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
|
||||
)
|
||||
trials['Avg duration'] = trials['Avg duration'].apply(
|
||||
lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}"
|
||||
if not isna(x) else "--".rjust(7, ' ')
|
||||
)
|
||||
trials['Objective'] = trials['Objective'].apply(
|
||||
lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
|
||||
)
|
||||
|
||||
stake_currency = config['stake_currency']
|
||||
|
||||
if has_drawdown:
|
||||
trials['Max Drawdown'] = trials.apply(
|
||||
lambda x: '{} {}'.format(
|
||||
round_coin_value(x['max_drawdown_abs'], stake_currency),
|
||||
'({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ')
|
||||
).rjust(25 + len(stake_currency))
|
||||
if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)),
|
||||
axis=1
|
||||
)
|
||||
else:
|
||||
trials = trials.drop(columns=['Max Drawdown'])
|
||||
|
||||
trials = trials.drop(columns=['max_drawdown_abs'])
|
||||
|
||||
trials['Profit'] = trials.apply(
|
||||
lambda x: '{} {}'.format(
|
||||
round_coin_value(x['Total profit'], stake_currency),
|
||||
'({:,.2f}%)'.format(x['Profit'] * perc_multi).rjust(10, ' ')
|
||||
).rjust(25+len(stake_currency))
|
||||
if x['Total profit'] != 0.0 else '--'.rjust(25+len(stake_currency)),
|
||||
axis=1
|
||||
)
|
||||
trials = trials.drop(columns=['Total profit'])
|
||||
|
||||
if print_colorized:
|
||||
for i in range(len(trials)):
|
||||
if trials.loc[i]['is_profit']:
|
||||
for j in range(len(trials.loc[i])-3):
|
||||
trials.iat[i, j] = "{}{}{}".format(Fore.GREEN,
|
||||
str(trials.loc[i][j]), Fore.RESET)
|
||||
if trials.loc[i]['is_best'] and highlight_best:
|
||||
for j in range(len(trials.loc[i])-3):
|
||||
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
|
||||
str(trials.loc[i][j]), Style.RESET_ALL)
|
||||
|
||||
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
||||
if remove_header > 0:
|
||||
table = tabulate.tabulate(
|
||||
trials.to_dict(orient='list'), tablefmt='orgtbl',
|
||||
headers='keys', stralign="right"
|
||||
)
|
||||
|
||||
table = table.split("\n", remove_header)[remove_header]
|
||||
elif remove_header < 0:
|
||||
table = tabulate.tabulate(
|
||||
trials.to_dict(orient='list'), tablefmt='psql',
|
||||
headers='keys', stralign="right"
|
||||
)
|
||||
table = "\n".join(table.split("\n")[0:remove_header])
|
||||
else:
|
||||
table = tabulate.tabulate(
|
||||
trials.to_dict(orient='list'), tablefmt='psql',
|
||||
headers='keys', stralign="right"
|
||||
)
|
||||
return table
|
||||
|
||||
@staticmethod
|
||||
def export_csv_file(config: dict, results: list, csv_file: str) -> None:
|
||||
"""
|
||||
Log result to csv-file
|
||||
"""
|
||||
if not results:
|
||||
return
|
||||
|
||||
# Verification for overwrite
|
||||
if Path(csv_file).is_file():
|
||||
logger.error(f"CSV file already exists: {csv_file}")
|
||||
return
|
||||
|
||||
try:
|
||||
io.open(csv_file, 'w+').close()
|
||||
except IOError:
|
||||
logger.error(f"Failed to create CSV file: {csv_file}")
|
||||
return
|
||||
|
||||
trials = json_normalize(results, max_level=1)
|
||||
trials['Best'] = ''
|
||||
trials['Stake currency'] = config['stake_currency']
|
||||
|
||||
base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
|
||||
'results_metrics.profit_mean', 'results_metrics.profit_median',
|
||||
'results_metrics.profit_total',
|
||||
'Stake currency',
|
||||
'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
|
||||
'loss', 'is_initial_point', 'is_best']
|
||||
perc_multi = 100
|
||||
|
||||
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
|
||||
trials = trials[base_metrics + param_metrics]
|
||||
|
||||
base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Median profit', 'Total profit',
|
||||
'Stake currency', 'Profit', 'Avg duration', 'Objective',
|
||||
'is_initial_point', 'is_best']
|
||||
param_columns = list(results[0]['params_dict'].keys())
|
||||
trials.columns = base_columns + param_columns
|
||||
|
||||
trials['is_profit'] = False
|
||||
trials.loc[trials['is_initial_point'], 'Best'] = '*'
|
||||
trials.loc[trials['is_best'], 'Best'] = 'Best'
|
||||
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
|
||||
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
|
||||
trials['Epoch'] = trials['Epoch'].astype(str)
|
||||
trials['Trades'] = trials['Trades'].astype(str)
|
||||
trials['Median profit'] = trials['Median profit'] * perc_multi
|
||||
|
||||
trials['Total profit'] = trials['Total profit'].apply(
|
||||
lambda x: f'{x:,.8f}' if x != 0.0 else ""
|
||||
)
|
||||
trials['Profit'] = trials['Profit'].apply(
|
||||
lambda x: f'{x:,.2f}' if not isna(x) else ""
|
||||
)
|
||||
trials['Avg profit'] = trials['Avg profit'].apply(
|
||||
lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
|
||||
)
|
||||
trials['Objective'] = trials['Objective'].apply(
|
||||
lambda x: f'{x:,.5f}' if x != 100000 else ""
|
||||
)
|
||||
|
||||
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])
|
||||
trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8')
|
||||
logger.info(f"CSV file created: {csv_file}")
|
@@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Union
|
||||
|
||||
from arrow import Arrow
|
||||
from numpy import int64
|
||||
from pandas import DataFrame
|
||||
from tabulate import tabulate
|
||||
@@ -22,7 +21,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
|
||||
Stores backtest results
|
||||
:param recordfilename: Path object, which can either be a filename or a directory.
|
||||
Filenames will be appended with a timestamp right before the suffix
|
||||
while for diectories, <directory>/backtest-result-<datetime>.json will be used as filename
|
||||
while for directories, <directory>/backtest-result-<datetime>.json will be used as filename
|
||||
:param stats: Dataframe containing the backtesting statistics
|
||||
"""
|
||||
if recordfilename.is_dir():
|
||||
@@ -32,7 +31,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
|
||||
filename = Path.joinpath(
|
||||
recordfilename.parent,
|
||||
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
|
||||
).with_suffix(recordfilename.suffix)
|
||||
).with_suffix(recordfilename.suffix)
|
||||
file_dump_json(filename, stats)
|
||||
|
||||
latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
|
||||
@@ -44,7 +43,7 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
|
||||
Generate floatformat (goes in line with _generate_result_line())
|
||||
"""
|
||||
return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f',
|
||||
'.2f', 'd', 'd', 'd', 'd']
|
||||
'.2f', 'd', 's', 's']
|
||||
|
||||
|
||||
def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
|
||||
@@ -53,7 +52,17 @@ def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
|
||||
"""
|
||||
return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %',
|
||||
f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration',
|
||||
'Wins', 'Draws', 'Losses']
|
||||
'Win Draw Loss Win%']
|
||||
|
||||
|
||||
def _generate_wins_draws_losses(wins, draws, losses):
|
||||
if wins > 0 and losses == 0:
|
||||
wl_ratio = '100'
|
||||
elif wins == 0:
|
||||
wl_ratio = '0'
|
||||
else:
|
||||
wl_ratio = f'{100.0 / (wins + draws + losses) * wins:.1f}' if losses > 0 else '100'
|
||||
return f'{wins:>4} {draws:>4} {losses:>4} {wl_ratio:>4}'
|
||||
|
||||
|
||||
def _generate_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict:
|
||||
@@ -110,6 +119,9 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_b
|
||||
|
||||
tabular_data.append(_generate_result_line(result, starting_balance, pair))
|
||||
|
||||
# Sort by total profit %:
|
||||
tabular_data = sorted(tabular_data, key=lambda k: k['profit_total_abs'], reverse=True)
|
||||
|
||||
# Append Total
|
||||
tabular_data.append(_generate_result_line(results, starting_balance, 'TOTAL'))
|
||||
return tabular_data
|
||||
@@ -150,7 +162,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
|
||||
return tabular_data
|
||||
|
||||
|
||||
def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
|
||||
def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
|
||||
"""
|
||||
Generate summary per strategy
|
||||
:param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
|
||||
@@ -162,6 +174,17 @@ def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
|
||||
tabular_data.append(_generate_result_line(
|
||||
results['results'], results['config']['dry_run_wallet'], strategy)
|
||||
)
|
||||
try:
|
||||
max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
|
||||
value_col='profit_ratio')
|
||||
max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'],
|
||||
value_col='profit_abs')
|
||||
except ValueError:
|
||||
max_drawdown_per = 0
|
||||
max_drawdown_abs = 0
|
||||
tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2)
|
||||
tabular_data[-1]['max_drawdown_abs'] = \
|
||||
round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False)
|
||||
return tabular_data
|
||||
|
||||
|
||||
@@ -213,7 +236,44 @@ def generate_days_breakdown_stats(results: DataFrame, starting_balance: int) ->
|
||||
return days_stats
|
||||
|
||||
|
||||
def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
|
||||
""" Generate overall trade statistics """
|
||||
if len(results) == 0:
|
||||
return {
|
||||
'wins': 0,
|
||||
'losses': 0,
|
||||
'draws': 0,
|
||||
'holding_avg': timedelta(),
|
||||
'winner_holding_avg': timedelta(),
|
||||
'loser_holding_avg': timedelta(),
|
||||
}
|
||||
|
||||
winning_trades = results.loc[results['profit_ratio'] > 0]
|
||||
draw_trades = results.loc[results['profit_ratio'] == 0]
|
||||
losing_trades = results.loc[results['profit_ratio'] < 0]
|
||||
|
||||
holding_avg = (timedelta(minutes=round(results['trade_duration'].mean()))
|
||||
if not results.empty else timedelta())
|
||||
winner_holding_avg = (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
|
||||
if not winning_trades.empty else timedelta())
|
||||
loser_holding_avg = (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
|
||||
if not losing_trades.empty else timedelta())
|
||||
|
||||
return {
|
||||
'wins': len(winning_trades),
|
||||
'losses': len(losing_trades),
|
||||
'draws': len(draw_trades),
|
||||
'holding_avg': holding_avg,
|
||||
'holding_avg_s': holding_avg.total_seconds(),
|
||||
'winner_holding_avg': winner_holding_avg,
|
||||
'winner_holding_avg_s': winner_holding_avg.total_seconds(),
|
||||
'loser_holding_avg': loser_holding_avg,
|
||||
'loser_holding_avg_s': loser_holding_avg.total_seconds(),
|
||||
}
|
||||
|
||||
|
||||
def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
|
||||
""" Generate daily statistics """
|
||||
if len(results) == 0:
|
||||
return {
|
||||
'backtest_best_day': 0,
|
||||
@@ -223,8 +283,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
|
||||
'winning_days': 0,
|
||||
'draw_days': 0,
|
||||
'losing_days': 0,
|
||||
'winner_holding_avg': timedelta(),
|
||||
'loser_holding_avg': timedelta(),
|
||||
'daily_profit_list': [],
|
||||
}
|
||||
daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
|
||||
daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
|
||||
@@ -235,9 +294,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
|
||||
winning_days = sum(daily_profit > 0)
|
||||
draw_days = sum(daily_profit == 0)
|
||||
losing_days = sum(daily_profit < 0)
|
||||
|
||||
winning_trades = results.loc[results['profit_ratio'] > 0]
|
||||
losing_trades = results.loc[results['profit_ratio'] < 0]
|
||||
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
|
||||
|
||||
return {
|
||||
'backtest_best_day': best_rel,
|
||||
@@ -247,16 +304,159 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
|
||||
'winning_days': winning_days,
|
||||
'draw_days': draw_days,
|
||||
'losing_days': losing_days,
|
||||
'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
|
||||
if not winning_trades.empty else timedelta()),
|
||||
'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
|
||||
if not losing_trades.empty else timedelta()),
|
||||
'daily_profit': daily_profit_list,
|
||||
}
|
||||
|
||||
|
||||
def generate_strategy_stats(btdata: Dict[str, DataFrame],
|
||||
strategy: str,
|
||||
content: Dict[str, Any],
|
||||
min_date: datetime, max_date: datetime,
|
||||
market_change: float
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
:param btdata: Backtest data
|
||||
:param strategy: Strategy name
|
||||
:param content: Backtest result data in the format:
|
||||
{'results: results, 'config: config}}.
|
||||
:param min_date: Backtest start date
|
||||
:param max_date: Backtest end date
|
||||
:param market_change: float indicating the market change
|
||||
:return: Dictionary containing results per strategy and a strategy summary.
|
||||
"""
|
||||
results: Dict[str, DataFrame] = content['results']
|
||||
if not isinstance(results, DataFrame):
|
||||
return {}
|
||||
config = content['config']
|
||||
max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
|
||||
starting_balance = config['dry_run_wallet']
|
||||
stake_currency = config['stake_currency']
|
||||
|
||||
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
|
||||
starting_balance=starting_balance,
|
||||
results=results, skip_nan=False)
|
||||
sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
|
||||
results=results)
|
||||
left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
|
||||
starting_balance=starting_balance,
|
||||
results=results.loc[results['is_open']],
|
||||
skip_nan=True)
|
||||
days_breakdown_stats = generate_days_breakdown_stats(
|
||||
results=results, starting_balance=starting_balance)
|
||||
daily_stats = generate_daily_stats(results)
|
||||
trade_stats = generate_trading_stats(results)
|
||||
best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
|
||||
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
|
||||
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
|
||||
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
|
||||
if not results.empty:
|
||||
results['open_timestamp'] = results['open_date'].view(int64) // 1e6
|
||||
results['close_timestamp'] = results['close_date'].view(int64) // 1e6
|
||||
|
||||
backtest_days = (max_date - min_date).days
|
||||
strat_stats = {
|
||||
'trades': results.to_dict(orient='records'),
|
||||
'locks': [lock.to_json() for lock in content['locks']],
|
||||
'best_pair': best_pair,
|
||||
'worst_pair': worst_pair,
|
||||
'results_per_pair': pair_results,
|
||||
'sell_reason_summary': sell_reason_stats,
|
||||
'left_open_trades': left_open_results,
|
||||
'days_breakdown_stats': days_breakdown_stats,
|
||||
|
||||
'total_trades': len(results),
|
||||
'total_volume': float(results['stake_amount'].sum()),
|
||||
'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
|
||||
'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
|
||||
'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
|
||||
'profit_total': results['profit_abs'].sum() / starting_balance,
|
||||
'profit_total_abs': results['profit_abs'].sum(),
|
||||
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
|
||||
'backtest_start_ts': int(min_date.timestamp() * 1000),
|
||||
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
|
||||
'backtest_end_ts': int(max_date.timestamp() * 1000),
|
||||
'backtest_days': backtest_days,
|
||||
|
||||
'backtest_run_start_ts': content['backtest_start_time'],
|
||||
'backtest_run_end_ts': content['backtest_end_time'],
|
||||
|
||||
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
|
||||
'market_change': market_change,
|
||||
'pairlist': list(btdata.keys()),
|
||||
'stake_amount': config['stake_amount'],
|
||||
'stake_currency': config['stake_currency'],
|
||||
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
|
||||
'starting_balance': starting_balance,
|
||||
'dry_run_wallet': starting_balance,
|
||||
'final_balance': content['final_balance'],
|
||||
'rejected_signals': content['rejected_signals'],
|
||||
'max_open_trades': max_open_trades,
|
||||
'max_open_trades_setting': (config['max_open_trades']
|
||||
if config['max_open_trades'] != float('inf') else -1),
|
||||
'timeframe': config['timeframe'],
|
||||
'timeframe_detail': config.get('timeframe_detail', ''),
|
||||
'timerange': config.get('timerange', ''),
|
||||
'enable_protections': config.get('enable_protections', False),
|
||||
'strategy_name': strategy,
|
||||
# Parameters relevant for backtesting
|
||||
'stoploss': config['stoploss'],
|
||||
'trailing_stop': config.get('trailing_stop', False),
|
||||
'trailing_stop_positive': config.get('trailing_stop_positive'),
|
||||
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
|
||||
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
|
||||
'use_custom_stoploss': config.get('use_custom_stoploss', False),
|
||||
'minimal_roi': config['minimal_roi'],
|
||||
'use_sell_signal': config['use_sell_signal'],
|
||||
'sell_profit_only': config['sell_profit_only'],
|
||||
'sell_profit_offset': config['sell_profit_offset'],
|
||||
'ignore_roi_if_buy_signal': config['ignore_roi_if_buy_signal'],
|
||||
**daily_stats,
|
||||
**trade_stats
|
||||
}
|
||||
|
||||
try:
|
||||
max_drawdown, _, _, _, _ = calculate_max_drawdown(
|
||||
results, value_col='profit_ratio')
|
||||
drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
|
||||
results, value_col='profit_abs')
|
||||
strat_stats.update({
|
||||
'max_drawdown': max_drawdown,
|
||||
'max_drawdown_abs': drawdown_abs,
|
||||
'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
|
||||
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
|
||||
'drawdown_end': drawdown_end.strftime(DATETIME_PRINT_FORMAT),
|
||||
'drawdown_end_ts': drawdown_end.timestamp() * 1000,
|
||||
|
||||
'max_drawdown_low': low_val,
|
||||
'max_drawdown_high': high_val,
|
||||
})
|
||||
|
||||
csum_min, csum_max = calculate_csum(results, starting_balance)
|
||||
strat_stats.update({
|
||||
'csum_min': csum_min,
|
||||
'csum_max': csum_max
|
||||
})
|
||||
|
||||
except ValueError:
|
||||
strat_stats.update({
|
||||
'max_drawdown': 0.0,
|
||||
'max_drawdown_abs': 0.0,
|
||||
'max_drawdown_low': 0.0,
|
||||
'max_drawdown_high': 0.0,
|
||||
'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
|
||||
'drawdown_start_ts': 0,
|
||||
'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
|
||||
'drawdown_end_ts': 0,
|
||||
'csum_min': 0,
|
||||
'csum_max': 0
|
||||
})
|
||||
|
||||
return strat_stats
|
||||
|
||||
|
||||
def generate_backtest_stats(btdata: Dict[str, DataFrame],
|
||||
all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
|
||||
min_date: Arrow, max_date: Arrow
|
||||
min_date: datetime, max_date: datetime
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
:param btdata: Backtest data
|
||||
@@ -264,135 +464,17 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
|
||||
{ Strategy: {'results: results, 'config: config}}.
|
||||
:param min_date: Backtest start date
|
||||
:param max_date: Backtest end date
|
||||
:return:
|
||||
Dictionary containing results per strategy and a stratgy summary.
|
||||
:return: Dictionary containing results per strategy and a strategy summary.
|
||||
"""
|
||||
result: Dict[str, Any] = {'strategy': {}}
|
||||
market_change = calculate_market_change(btdata, 'close')
|
||||
|
||||
for strategy, content in all_results.items():
|
||||
results: Dict[str, DataFrame] = content['results']
|
||||
if not isinstance(results, DataFrame):
|
||||
continue
|
||||
config = content['config']
|
||||
max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
|
||||
starting_balance = config['dry_run_wallet']
|
||||
stake_currency = config['stake_currency']
|
||||
|
||||
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
|
||||
starting_balance=starting_balance,
|
||||
results=results, skip_nan=False)
|
||||
sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
|
||||
results=results)
|
||||
left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
|
||||
starting_balance=starting_balance,
|
||||
results=results.loc[results['is_open']],
|
||||
skip_nan=True)
|
||||
days_breakdown_stats = generate_days_breakdown_stats(results=results,
|
||||
starting_balance=starting_balance)
|
||||
daily_stats = generate_daily_stats(results)
|
||||
best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
|
||||
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
|
||||
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
|
||||
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
|
||||
results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
|
||||
results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
|
||||
|
||||
backtest_days = (max_date - min_date).days
|
||||
strat_stats = {
|
||||
'trades': results.to_dict(orient='records'),
|
||||
'locks': [lock.to_json() for lock in content['locks']],
|
||||
'best_pair': best_pair,
|
||||
'worst_pair': worst_pair,
|
||||
'results_per_pair': pair_results,
|
||||
'sell_reason_summary': sell_reason_stats,
|
||||
'left_open_trades': left_open_results,
|
||||
'days_breakdown_stats': days_breakdown_stats,
|
||||
'total_trades': len(results),
|
||||
'total_volume': float(results['stake_amount'].sum()),
|
||||
'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
|
||||
'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
|
||||
'profit_total': results['profit_abs'].sum() / starting_balance,
|
||||
'profit_total_abs': results['profit_abs'].sum(),
|
||||
'backtest_start': min_date.datetime,
|
||||
'backtest_start_ts': min_date.int_timestamp * 1000,
|
||||
'backtest_end': max_date.datetime,
|
||||
'backtest_end_ts': max_date.int_timestamp * 1000,
|
||||
'backtest_days': backtest_days,
|
||||
|
||||
'backtest_run_start_ts': content['backtest_start_time'],
|
||||
'backtest_run_end_ts': content['backtest_end_time'],
|
||||
|
||||
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
|
||||
'market_change': market_change,
|
||||
'pairlist': list(btdata.keys()),
|
||||
'stake_amount': config['stake_amount'],
|
||||
'stake_currency': config['stake_currency'],
|
||||
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
|
||||
'starting_balance': starting_balance,
|
||||
'dry_run_wallet': starting_balance,
|
||||
'final_balance': content['final_balance'],
|
||||
'max_open_trades': max_open_trades,
|
||||
'max_open_trades_setting': (config['max_open_trades']
|
||||
if config['max_open_trades'] != float('inf') else -1),
|
||||
'timeframe': config['timeframe'],
|
||||
'timerange': config.get('timerange', ''),
|
||||
'enable_protections': config.get('enable_protections', False),
|
||||
'strategy_name': strategy,
|
||||
# Parameters relevant for backtesting
|
||||
'stoploss': config['stoploss'],
|
||||
'trailing_stop': config.get('trailing_stop', False),
|
||||
'trailing_stop_positive': config.get('trailing_stop_positive'),
|
||||
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
|
||||
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
|
||||
'use_custom_stoploss': config.get('use_custom_stoploss', False),
|
||||
'minimal_roi': config['minimal_roi'],
|
||||
'use_sell_signal': config['ask_strategy']['use_sell_signal'],
|
||||
'sell_profit_only': config['ask_strategy']['sell_profit_only'],
|
||||
'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
|
||||
'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
|
||||
**daily_stats,
|
||||
}
|
||||
strat_stats = generate_strategy_stats(btdata, strategy, content,
|
||||
min_date, max_date, market_change=market_change)
|
||||
result['strategy'][strategy] = strat_stats
|
||||
|
||||
try:
|
||||
max_drawdown, _, _, _, _ = calculate_max_drawdown(
|
||||
results, value_col='profit_ratio')
|
||||
drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
|
||||
results, value_col='profit_abs')
|
||||
strat_stats.update({
|
||||
'max_drawdown': max_drawdown,
|
||||
'max_drawdown_abs': drawdown_abs,
|
||||
'drawdown_start': drawdown_start,
|
||||
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
|
||||
'drawdown_end': drawdown_end,
|
||||
'drawdown_end_ts': drawdown_end.timestamp() * 1000,
|
||||
|
||||
'max_drawdown_low': low_val,
|
||||
'max_drawdown_high': high_val,
|
||||
})
|
||||
|
||||
csum_min, csum_max = calculate_csum(results, starting_balance)
|
||||
strat_stats.update({
|
||||
'csum_min': csum_min,
|
||||
'csum_max': csum_max
|
||||
})
|
||||
|
||||
except ValueError:
|
||||
strat_stats.update({
|
||||
'max_drawdown': 0.0,
|
||||
'max_drawdown_abs': 0.0,
|
||||
'max_drawdown_low': 0.0,
|
||||
'max_drawdown_high': 0.0,
|
||||
'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
|
||||
'drawdown_start_ts': 0,
|
||||
'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
|
||||
'drawdown_end_ts': 0,
|
||||
'csum_min': 0,
|
||||
'csum_max': 0
|
||||
})
|
||||
|
||||
strategy_results = generate_strategy_metrics(all_results=all_results)
|
||||
strategy_results = generate_strategy_comparison(all_results=all_results)
|
||||
|
||||
result['strategy_comparison'] = strategy_results
|
||||
|
||||
@@ -415,7 +497,8 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
|
||||
floatfmt = _get_line_floatfmt(stake_currency)
|
||||
output = [[
|
||||
t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
|
||||
t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
|
||||
t['profit_total_pct'], t['duration_avg'],
|
||||
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses'])
|
||||
] for t in pair_results]
|
||||
# Ignore type as floatfmt does allow tuples but mypy does not know that
|
||||
return tabulate(output, headers=headers,
|
||||
@@ -432,9 +515,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
|
||||
headers = [
|
||||
'Sell Reason',
|
||||
'Sells',
|
||||
'Wins',
|
||||
'Draws',
|
||||
'Losses',
|
||||
'Win Draws Loss Win%',
|
||||
'Avg Profit %',
|
||||
'Cum Profit %',
|
||||
f'Tot Profit {stake_currency}',
|
||||
@@ -442,7 +523,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
|
||||
]
|
||||
|
||||
output = [[
|
||||
t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
|
||||
t['sell_reason'], t['trades'],
|
||||
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
|
||||
t['profit_mean_pct'], t['profit_sum_pct'],
|
||||
round_coin_value(t['profit_total_abs'], stake_currency, False),
|
||||
t['profit_total_pct'],
|
||||
@@ -450,7 +532,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
|
||||
return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
|
||||
|
||||
|
||||
def text_table_days_breakdown(days_breakdown_stats: List[Dict[str, Any]], stake_currency: str) -> str:
|
||||
def text_table_days_breakdown(days_breakdown_stats: List[Dict[str, Any]],
|
||||
stake_currency: str) -> str:
|
||||
"""
|
||||
Generate small table with Backtest results by days
|
||||
:param days_breakdown_stats: Days breakdown metrics
|
||||
@@ -475,18 +558,28 @@ def text_table_days_breakdown(days_breakdown_stats: List[Dict[str, Any]], stake_
|
||||
def text_table_strategy(strategy_results, stake_currency: str) -> str:
|
||||
"""
|
||||
Generate summary table per strategy
|
||||
:param strategy_results: Dict of <Strategyname: DataFrame> containing results for all strategies
|
||||
:param stake_currency: stake-currency - used to correctly name headers
|
||||
:param max_open_trades: Maximum allowed open trades used for backtest
|
||||
:param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
|
||||
:return: pretty printed table with tabulate as string
|
||||
"""
|
||||
floatfmt = _get_line_floatfmt(stake_currency)
|
||||
headers = _get_line_header('Strategy', stake_currency)
|
||||
# _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless
|
||||
# therefore we slip this column in only for strategy summary here.
|
||||
headers.append('Drawdown')
|
||||
|
||||
# Align drawdown string on the center two space separator.
|
||||
drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
|
||||
dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results])
|
||||
dd_pad_per = max([len(dd) for dd in drawdown])
|
||||
drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%'
|
||||
for t, dd in zip(strategy_results, drawdown)]
|
||||
|
||||
output = [[
|
||||
t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
|
||||
t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
|
||||
] for t in strategy_results]
|
||||
t['profit_total_pct'], t['duration_avg'],
|
||||
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown]
|
||||
for t, drawdown in zip(strategy_results, drawdown)]
|
||||
# Ignore type as floatfmt does allow tuples but mypy does not know that
|
||||
return tabulate(output, headers=headers,
|
||||
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
|
||||
@@ -496,12 +589,17 @@ def text_table_add_metrics(strat_results: Dict) -> str:
|
||||
if len(strat_results['trades']) > 0:
|
||||
best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio'])
|
||||
worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio'])
|
||||
|
||||
# Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
|
||||
# command stores these results and newer version of freqtrade must be able to handle old
|
||||
# results with missing new fields.
|
||||
metrics = [
|
||||
('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
|
||||
('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
|
||||
('Backtesting from', strat_results['backtest_start']),
|
||||
('Backtesting to', strat_results['backtest_end']),
|
||||
('Max open trades', strat_results['max_open_trades']),
|
||||
('', ''), # Empty line to improve readability
|
||||
('Total trades', strat_results['total_trades']),
|
||||
('Total/Daily Avg Trades',
|
||||
f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"),
|
||||
('Starting balance', round_coin_value(strat_results['starting_balance'],
|
||||
strat_results['stake_currency'])),
|
||||
('Final balance', round_coin_value(strat_results['final_balance'],
|
||||
@@ -516,7 +614,6 @@ def text_table_add_metrics(strat_results: Dict) -> str:
|
||||
strat_results['stake_currency'])),
|
||||
('Total trade volume', round_coin_value(strat_results['total_volume'],
|
||||
strat_results['stake_currency'])),
|
||||
|
||||
('', ''), # Empty line to improve readability
|
||||
('Best Pair', f"{strat_results['best_pair']['key']} "
|
||||
f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"),
|
||||
@@ -531,9 +628,10 @@ def text_table_add_metrics(strat_results: Dict) -> str:
|
||||
('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'],
|
||||
strat_results['stake_currency'])),
|
||||
('Days win/draw/lose', f"{strat_results['winning_days']} / "
|
||||
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
|
||||
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
|
||||
('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
|
||||
('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
|
||||
('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
|
||||
('', ''), # Empty line to improve readability
|
||||
|
||||
('Min balance', round_coin_value(strat_results['csum_min'],
|
||||
@@ -548,8 +646,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
|
||||
strat_results['stake_currency'])),
|
||||
('Drawdown low', round_coin_value(strat_results['max_drawdown_low'],
|
||||
strat_results['stake_currency'])),
|
||||
('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
|
||||
('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
|
||||
('Drawdown Start', strat_results['drawdown_start']),
|
||||
('Drawdown End', strat_results['drawdown_end']),
|
||||
('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
|
||||
]
|
||||
|
||||
@@ -559,7 +657,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
|
||||
strat_results['stake_currency'])
|
||||
stake_amount = round_coin_value(
|
||||
strat_results['stake_amount'], strat_results['stake_currency']
|
||||
) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
|
||||
) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
|
||||
|
||||
message = ("No trades made. "
|
||||
f"Your starting balance was {start_balance}, "
|
||||
@@ -568,49 +666,58 @@ def text_table_add_metrics(strat_results: Dict) -> str:
|
||||
return message
|
||||
|
||||
|
||||
def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str,
|
||||
show_days=False):
|
||||
"""
|
||||
Print results for one strategy
|
||||
"""
|
||||
# Print results
|
||||
print(f"Result for strategy {strategy}")
|
||||
table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
|
||||
if isinstance(table, str):
|
||||
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
|
||||
stake_currency=stake_currency)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
if show_days:
|
||||
table = text_table_days_breakdown(days_breakdown_stats=results['days_breakdown_stats'],
|
||||
stake_currency=stake_currency)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' DAYS BREAKDOWN '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = text_table_add_metrics(results)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
print()
|
||||
|
||||
|
||||
def show_backtest_results(config: Dict, backtest_stats: Dict):
|
||||
stake_currency = config['stake_currency']
|
||||
|
||||
for strategy, results in backtest_stats['strategy'].items():
|
||||
|
||||
# Print results
|
||||
print(f"Result for strategy {strategy}")
|
||||
table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
|
||||
if isinstance(table, str):
|
||||
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
|
||||
stake_currency=stake_currency)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
if config.get('show_days', False):
|
||||
table = text_table_days_breakdown(days_breakdown_stats=results['days_breakdown_stats'],
|
||||
stake_currency=stake_currency)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' DAYS BREAKDOWN '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
table = text_table_add_metrics(results)
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
|
||||
if isinstance(table, str) and len(table) > 0:
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
print()
|
||||
show_backtest_result(strategy, results, stake_currency, config.get('show_days', False))
|
||||
|
||||
if len(backtest_stats['strategy']) > 1:
|
||||
# Print Strategy summary table
|
||||
|
||||
table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency)
|
||||
print(f"{results['backtest_start']} -> {results['backtest_end']} |"
|
||||
f" Max open trades : {results['max_open_trades']}")
|
||||
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
|
||||
print(table)
|
||||
print('=' * len(table.splitlines()[0]))
|
||||
|
4
freqtrade/optimize/space/__init__.py
Normal file
4
freqtrade/optimize/space/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# flake8: noqa: F401
|
||||
from skopt.space import Categorical, Dimension, Integer, Real
|
||||
|
||||
from .decimalspace import SKDecimal
|
33
freqtrade/optimize/space/decimalspace.py
Normal file
33
freqtrade/optimize/space/decimalspace.py
Normal file
@@ -0,0 +1,33 @@
|
||||
import numpy as np
|
||||
from skopt.space import Integer
|
||||
|
||||
|
||||
class SKDecimal(Integer):
|
||||
|
||||
def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None,
|
||||
name=None, dtype=np.int64):
|
||||
self.decimals = decimals
|
||||
_low = int(low * pow(10, self.decimals))
|
||||
_high = int(high * pow(10, self.decimals))
|
||||
# trunc to precision to avoid points out of space
|
||||
self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals)
|
||||
self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals)
|
||||
|
||||
super().__init__(_low, _high, prior, base, transform, name, dtype)
|
||||
|
||||
def __repr__(self):
|
||||
return "Decimal(low={}, high={}, decimals={}, prior='{}', transform='{}')".format(
|
||||
self.low_orig, self.high_orig, self.decimals, self.prior, self.transform_)
|
||||
|
||||
def __contains__(self, point):
|
||||
if isinstance(point, list):
|
||||
point = np.array(point)
|
||||
return self.low_orig <= point <= self.high_orig
|
||||
|
||||
def transform(self, Xt):
|
||||
aa = [int(x * pow(10, self.decimals)) for x in Xt]
|
||||
return super().transform(aa)
|
||||
|
||||
def inverse_transform(self, Xt):
|
||||
res = super().inverse_transform(Xt)
|
||||
return [round(x * pow(0.1, self.decimals), self.decimals) for x in res]
|
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -47,6 +47,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
|
||||
min_rate = get_column_def(cols, 'min_rate', 'null')
|
||||
sell_reason = get_column_def(cols, 'sell_reason', 'null')
|
||||
strategy = get_column_def(cols, 'strategy', 'null')
|
||||
buy_tag = get_column_def(cols, 'buy_tag', 'null')
|
||||
# If ticker-interval existed use that, else null.
|
||||
if has_column(cols, 'ticker_interval'):
|
||||
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
|
||||
@@ -62,33 +63,29 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
|
||||
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
|
||||
|
||||
# Schema migration necessary
|
||||
engine.execute(f"alter table trades rename to {table_back_name}")
|
||||
# drop indexes on backup table
|
||||
for index in inspector.get_indexes(table_back_name):
|
||||
engine.execute(f"drop index {index['name']}")
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text(f"alter table trades rename to {table_back_name}"))
|
||||
with engine.begin() as connection:
|
||||
# drop indexes on backup table in new session
|
||||
for index in inspector.get_indexes(table_back_name):
|
||||
connection.execute(text(f"drop index {index['name']}"))
|
||||
# let SQLAlchemy create the schema as required
|
||||
decl_base.metadata.create_all(engine)
|
||||
|
||||
# Copy data back - following the correct schema
|
||||
engine.execute(f"""insert into trades
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text(f"""insert into trades
|
||||
(id, exchange, pair, is_open,
|
||||
fee_open, fee_open_cost, fee_open_currency,
|
||||
fee_close, fee_close_cost, fee_open_currency, open_rate,
|
||||
fee_close, fee_close_cost, fee_close_currency, open_rate,
|
||||
open_rate_requested, close_rate, close_rate_requested, close_profit,
|
||||
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
|
||||
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
|
||||
stoploss_order_id, stoploss_last_update,
|
||||
max_rate, min_rate, sell_reason, sell_order_status, strategy,
|
||||
max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag,
|
||||
timeframe, open_trade_value, close_profit_abs
|
||||
)
|
||||
select id, lower(exchange),
|
||||
case
|
||||
when instr(pair, '_') != 0 then
|
||||
substr(pair, instr(pair, '_') + 1) || '/' ||
|
||||
substr(pair, 1, instr(pair, '_') - 1)
|
||||
else pair
|
||||
end
|
||||
pair,
|
||||
select id, lower(exchange), pair,
|
||||
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
|
||||
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
|
||||
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
|
||||
@@ -101,14 +98,15 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
|
||||
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
|
||||
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
|
||||
{sell_order_status} sell_order_status,
|
||||
{strategy} strategy, {timeframe} timeframe,
|
||||
{strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe,
|
||||
{open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs
|
||||
from {table_back_name}
|
||||
""")
|
||||
"""))
|
||||
|
||||
|
||||
def migrate_open_orders_to_trades(engine):
|
||||
engine.execute("""
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text("""
|
||||
insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
|
||||
select id ft_trade_id, pair ft_pair, open_order_id,
|
||||
case when close_rate_requested is null then 'buy'
|
||||
@@ -120,7 +118,32 @@ def migrate_open_orders_to_trades(engine):
|
||||
'stoploss' ft_order_side, 1 ft_is_open
|
||||
from trades
|
||||
where stoploss_order_id is not null
|
||||
""")
|
||||
"""))
|
||||
|
||||
|
||||
def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List):
|
||||
# Schema migration necessary
|
||||
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text(f"alter table orders rename to {table_back_name}"))
|
||||
|
||||
with engine.begin() as connection:
|
||||
# drop indexes on backup table in new session
|
||||
for index in inspector.get_indexes(table_back_name):
|
||||
connection.execute(text(f"drop index {index['name']}"))
|
||||
|
||||
# let SQLAlchemy create the schema as required
|
||||
decl_base.metadata.create_all(engine)
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text(f"""
|
||||
insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
|
||||
status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
|
||||
order_date, order_filled_date, order_update_date)
|
||||
select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
|
||||
status, symbol, order_type, side, price, amount, filled, null average, remaining, cost,
|
||||
order_date, order_filled_date, order_update_date
|
||||
from {table_back_name}
|
||||
"""))
|
||||
|
||||
|
||||
def check_migrate(engine, decl_base, previous_tables) -> None:
|
||||
@@ -134,7 +157,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
|
||||
table_back_name = get_backup_name(tabs, 'trades_bak')
|
||||
|
||||
# Check for latest column
|
||||
if not has_column(cols, 'open_trade_value'):
|
||||
if not has_column(cols, 'buy_tag'):
|
||||
logger.info(f'Running database migration for trades - backup: {table_back_name}')
|
||||
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
|
||||
# Reread columns - the above recreated the table!
|
||||
@@ -145,6 +168,11 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
|
||||
logger.info('Moving open orders to Orders table.')
|
||||
migrate_open_orders_to_trades(engine)
|
||||
else:
|
||||
pass
|
||||
# Empty for now - as there is only one iteration of the orders table so far.
|
||||
# table_back_name = get_backup_name(tabs, 'orders_bak')
|
||||
cols_order = inspector.get_columns('orders')
|
||||
|
||||
if not has_column(cols_order, 'average'):
|
||||
tabs = get_table_names_for_table(inspector, 'orders')
|
||||
# Empty for now - as there is only one iteration of the orders table so far.
|
||||
table_back_name = get_backup_name(tabs, 'orders_bak')
|
||||
|
||||
migrate_orders_table(decl_base, inspector, engine, table_back_name, cols)
|
||||
|
@@ -2,22 +2,19 @@
|
||||
This module contains the class to persist trades into SQLite
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import arrow
|
||||
from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
|
||||
create_engine, desc, func, inspect)
|
||||
from sqlalchemy.exc import NoSuchModuleError
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import Query, relationship
|
||||
from sqlalchemy.orm.scoping import scoped_session
|
||||
from sqlalchemy.orm.session import sessionmaker
|
||||
from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlalchemy.sql.schema import UniqueConstraint
|
||||
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT
|
||||
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
|
||||
from freqtrade.enums import SellType
|
||||
from freqtrade.exceptions import DependencyException, OperationalException
|
||||
from freqtrade.misc import safe_value_fallback
|
||||
from freqtrade.persistence.migrations import check_migrate
|
||||
@@ -42,16 +39,18 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
|
||||
"""
|
||||
kwargs = {}
|
||||
|
||||
# Take care of thread ownership if in-memory db
|
||||
if db_url == 'sqlite://':
|
||||
kwargs.update({
|
||||
'connect_args': {'check_same_thread': False},
|
||||
'poolclass': StaticPool,
|
||||
'echo': False,
|
||||
})
|
||||
# Take care of thread ownership
|
||||
if db_url.startswith('sqlite://'):
|
||||
kwargs.update({
|
||||
'connect_args': {'check_same_thread': False},
|
||||
})
|
||||
|
||||
try:
|
||||
engine = create_engine(db_url, **kwargs)
|
||||
engine = create_engine(db_url, future=True, **kwargs)
|
||||
except NoSuchModuleError:
|
||||
raise OperationalException(f"Given value for db_url: '{db_url}' "
|
||||
f"is no valid database URL! (See {_SQL_DOCS_URL})")
|
||||
@@ -59,13 +58,10 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
|
||||
# https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
|
||||
# Scoped sessions proxy requests to the appropriate thread-local session.
|
||||
# We should use the scoped_session object - not a seperately initialized version
|
||||
Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
|
||||
Trade.query = Trade.session.query_property()
|
||||
# Copy session attributes to order object too
|
||||
Order.session = Trade.session
|
||||
Order.query = Order.session.query_property()
|
||||
PairLock.session = Trade.session
|
||||
PairLock.query = PairLock.session.query_property()
|
||||
Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True))
|
||||
Trade.query = Trade._session.query_property()
|
||||
Order.query = Trade._session.query_property()
|
||||
PairLock.query = Trade._session.query_property()
|
||||
|
||||
previous_tables = inspect(engine).get_table_names()
|
||||
_DECL_BASE.metadata.create_all(engine)
|
||||
@@ -81,7 +77,7 @@ def cleanup_db() -> None:
|
||||
Flushes all pending operations to disk.
|
||||
:return: None
|
||||
"""
|
||||
Trade.session.flush()
|
||||
Trade.commit()
|
||||
|
||||
|
||||
def clean_dry_run_db() -> None:
|
||||
@@ -93,6 +89,7 @@ def clean_dry_run_db() -> None:
|
||||
# Check we are updating only a dry_run order not a prod one
|
||||
if 'dry_run' in trade.open_order_id:
|
||||
trade.open_order_id = None
|
||||
Trade.commit()
|
||||
|
||||
|
||||
class Order(_DECL_BASE):
|
||||
@@ -116,16 +113,17 @@ class Order(_DECL_BASE):
|
||||
|
||||
trade = relationship("Trade", back_populates="orders")
|
||||
|
||||
ft_order_side = Column(String, nullable=False)
|
||||
ft_pair = Column(String, nullable=False)
|
||||
ft_order_side = Column(String(25), nullable=False)
|
||||
ft_pair = Column(String(25), nullable=False)
|
||||
ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
|
||||
|
||||
order_id = Column(String, nullable=False, index=True)
|
||||
status = Column(String, nullable=True)
|
||||
symbol = Column(String, nullable=True)
|
||||
order_type = Column(String, nullable=True)
|
||||
side = Column(String, nullable=True)
|
||||
order_id = Column(String(255), nullable=False, index=True)
|
||||
status = Column(String(255), nullable=True)
|
||||
symbol = Column(String(25), nullable=True)
|
||||
order_type = Column(String(50), nullable=True)
|
||||
side = Column(String(25), nullable=True)
|
||||
price = Column(Float, nullable=True)
|
||||
average = Column(Float, nullable=True)
|
||||
amount = Column(Float, nullable=True)
|
||||
filled = Column(Float, nullable=True)
|
||||
remaining = Column(Float, nullable=True)
|
||||
@@ -154,17 +152,18 @@ class Order(_DECL_BASE):
|
||||
self.price = order.get('price', self.price)
|
||||
self.amount = order.get('amount', self.amount)
|
||||
self.filled = order.get('filled', self.filled)
|
||||
self.average = order.get('average', self.average)
|
||||
self.remaining = order.get('remaining', self.remaining)
|
||||
self.cost = order.get('cost', self.cost)
|
||||
if 'timestamp' in order and order['timestamp'] is not None:
|
||||
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
|
||||
|
||||
self.ft_is_open = True
|
||||
if self.status in ('closed', 'canceled', 'cancelled'):
|
||||
if self.status in NON_OPEN_EXCHANGE_STATES:
|
||||
self.ft_is_open = False
|
||||
if order.get('filled', 0) > 0:
|
||||
self.order_filled_date = arrow.utcnow().datetime
|
||||
self.order_update_date = arrow.utcnow().datetime
|
||||
if (order.get('filled', 0.0) or 0.0) > 0:
|
||||
self.order_filled_date = datetime.now(timezone.utc)
|
||||
self.order_update_date = datetime.now(timezone.utc)
|
||||
|
||||
@staticmethod
|
||||
def update_orders(orders: List['Order'], order: Dict[str, Any]):
|
||||
@@ -179,6 +178,7 @@ class Order(_DECL_BASE):
|
||||
if filtered_orders:
|
||||
oobj = filtered_orders[0]
|
||||
oobj.update_from_ccxt_object(order)
|
||||
Order.query.session.commit()
|
||||
else:
|
||||
logger.warning(f"Did not find order for {order}.")
|
||||
|
||||
@@ -257,6 +257,7 @@ class LocalTrade():
|
||||
sell_reason: str = ''
|
||||
sell_order_status: str = ''
|
||||
strategy: str = ''
|
||||
buy_tag: Optional[str] = None
|
||||
timeframe: Optional[int] = None
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
@@ -288,6 +289,7 @@ class LocalTrade():
|
||||
'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
|
||||
'stake_amount': round(self.stake_amount, 8),
|
||||
'strategy': self.strategy,
|
||||
'buy_tag': self.buy_tag,
|
||||
'timeframe': self.timeframe,
|
||||
|
||||
'fee_open': self.fee_open,
|
||||
@@ -297,15 +299,12 @@ class LocalTrade():
|
||||
'fee_close_cost': self.fee_close_cost,
|
||||
'fee_close_currency': self.fee_close_currency,
|
||||
|
||||
'open_date_hum': arrow.get(self.open_date).humanize(),
|
||||
'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
|
||||
'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
|
||||
'open_rate': self.open_rate,
|
||||
'open_rate_requested': self.open_rate_requested,
|
||||
'open_trade_value': round(self.open_trade_value, 8),
|
||||
|
||||
'close_date_hum': (arrow.get(self.close_date).humanize()
|
||||
if self.close_date else None),
|
||||
'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
|
||||
if self.close_date else None),
|
||||
'close_timestamp': int(self.close_date.replace(
|
||||
@@ -355,12 +354,12 @@ class LocalTrade():
|
||||
LocalTrade.trades_open = []
|
||||
LocalTrade.total_profit = 0
|
||||
|
||||
def adjust_min_max_rates(self, current_price: float) -> None:
|
||||
def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
|
||||
"""
|
||||
Adjust the max_rate and min_rate.
|
||||
"""
|
||||
self.max_rate = max(current_price, self.max_rate or self.open_rate)
|
||||
self.min_rate = min(current_price, self.min_rate or self.open_rate)
|
||||
self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
|
||||
|
||||
def _set_new_stoploss(self, new_loss: float, stoploss: float):
|
||||
"""Assign new stop value"""
|
||||
@@ -434,12 +433,13 @@ class LocalTrade():
|
||||
elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'):
|
||||
self.stoploss_order_id = None
|
||||
self.close_rate_requested = self.stop_loss
|
||||
self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value
|
||||
if self.is_open:
|
||||
logger.info(f'{order_type.upper()} is hit for {self}.')
|
||||
self.close(safe_value_fallback(order, 'average', 'price'))
|
||||
else:
|
||||
raise ValueError(f'Unknown order type: {order_type}')
|
||||
cleanup_db()
|
||||
Trade.commit()
|
||||
|
||||
def close(self, rate: float, *, show_msg: bool = True) -> None:
|
||||
"""
|
||||
@@ -554,6 +554,8 @@ class LocalTrade():
|
||||
rate=(rate or self.close_rate),
|
||||
fee=(fee or self.fee_close)
|
||||
)
|
||||
if self.open_trade_value == 0.0:
|
||||
return 0.0
|
||||
profit_ratio = (close_trade_value / self.open_trade_value) - 1
|
||||
return float(f"{profit_ratio:.8f}")
|
||||
|
||||
@@ -572,23 +574,6 @@ class LocalTrade():
|
||||
else:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_trades(trade_filter=None) -> Query:
|
||||
"""
|
||||
Helper function to query Trades using filters.
|
||||
:param trade_filter: Optional filter to apply to trades
|
||||
Can be either a Filter object, or a List of filters
|
||||
e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
|
||||
e.g. `(trade_filter=Trade.id == trade_id)`
|
||||
:return: unsorted query object
|
||||
"""
|
||||
if trade_filter is not None:
|
||||
if not isinstance(trade_filter, list):
|
||||
trade_filter = [trade_filter]
|
||||
return Trade.query.filter(*trade_filter)
|
||||
else:
|
||||
return Trade.query
|
||||
|
||||
@staticmethod
|
||||
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
|
||||
open_date: datetime = None, close_date: datetime = None,
|
||||
@@ -611,7 +596,7 @@ class LocalTrade():
|
||||
|
||||
else:
|
||||
# Not used during backtesting, but might be used by a strategy
|
||||
sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open]
|
||||
sel_trades = list(LocalTrade.trades + LocalTrade.trades_open)
|
||||
|
||||
if pair:
|
||||
sel_trades = [trade for trade in sel_trades if trade.pair == pair]
|
||||
@@ -641,83 +626,7 @@ class LocalTrade():
|
||||
"""
|
||||
Query trades from persistence layer
|
||||
"""
|
||||
return Trade.get_trades(Trade.is_open.is_(True)).all()
|
||||
|
||||
@staticmethod
|
||||
def get_open_order_trades():
|
||||
"""
|
||||
Returns all open trades
|
||||
"""
|
||||
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
|
||||
|
||||
@staticmethod
|
||||
def get_open_trades_without_assigned_fees():
|
||||
"""
|
||||
Returns all open trades which don't have open fees set correctly
|
||||
"""
|
||||
return Trade.get_trades([Trade.fee_open_currency.is_(None),
|
||||
Trade.orders.any(),
|
||||
Trade.is_open.is_(True),
|
||||
]).all()
|
||||
|
||||
@staticmethod
|
||||
def get_sold_trades_without_assigned_fees():
|
||||
"""
|
||||
Returns all closed trades which don't have fees set correctly
|
||||
"""
|
||||
return Trade.get_trades([Trade.fee_close_currency.is_(None),
|
||||
Trade.orders.any(),
|
||||
Trade.is_open.is_(False),
|
||||
]).all()
|
||||
|
||||
@staticmethod
|
||||
def total_open_trades_stakes() -> float:
|
||||
"""
|
||||
Calculates total invested amount in open trades
|
||||
in stake currency
|
||||
"""
|
||||
if Trade.use_db:
|
||||
total_open_stake_amount = Trade.session.query(
|
||||
func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
|
||||
else:
|
||||
total_open_stake_amount = sum(
|
||||
t.stake_amount for t in Trade.get_trades_proxy(is_open=True))
|
||||
return total_open_stake_amount or 0
|
||||
|
||||
@staticmethod
|
||||
def get_overall_performance() -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Returns List of dicts containing all Trades, including profit and trade count
|
||||
"""
|
||||
pair_rates = Trade.session.query(
|
||||
Trade.pair,
|
||||
func.sum(Trade.close_profit).label('profit_sum'),
|
||||
func.count(Trade.pair).label('count')
|
||||
).filter(Trade.is_open.is_(False))\
|
||||
.group_by(Trade.pair) \
|
||||
.order_by(desc('profit_sum')) \
|
||||
.all()
|
||||
return [
|
||||
{
|
||||
'pair': pair,
|
||||
'profit': rate,
|
||||
'count': count
|
||||
}
|
||||
for pair, rate, count in pair_rates
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_best_pair():
|
||||
"""
|
||||
Get best pair with closed trade.
|
||||
:returns: Tuple containing (pair, profit_sum)
|
||||
"""
|
||||
best_pair = Trade.session.query(
|
||||
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
|
||||
).filter(Trade.is_open.is_(False)) \
|
||||
.group_by(Trade.pair) \
|
||||
.order_by(desc('profit_sum')).first()
|
||||
return best_pair
|
||||
return Trade.get_trades_proxy(is_open=True)
|
||||
|
||||
@staticmethod
|
||||
def stoploss_reinitialization(desired_stoploss):
|
||||
@@ -729,7 +638,7 @@ class LocalTrade():
|
||||
|
||||
# skip case if trailing-stop changed the stoploss already.
|
||||
if (trade.stop_loss == trade.initial_stop_loss
|
||||
and trade.initial_stop_loss_pct != desired_stoploss):
|
||||
and trade.initial_stop_loss_pct != desired_stoploss):
|
||||
# Stoploss value got changed
|
||||
|
||||
logger.info(f"Stoploss for {trade} needs adjustment...")
|
||||
@@ -754,15 +663,15 @@ class Trade(_DECL_BASE, LocalTrade):
|
||||
|
||||
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
|
||||
|
||||
exchange = Column(String, nullable=False)
|
||||
pair = Column(String, nullable=False, index=True)
|
||||
exchange = Column(String(25), nullable=False)
|
||||
pair = Column(String(25), nullable=False, index=True)
|
||||
is_open = Column(Boolean, nullable=False, default=True, index=True)
|
||||
fee_open = Column(Float, nullable=False, default=0.0)
|
||||
fee_open_cost = Column(Float, nullable=True)
|
||||
fee_open_currency = Column(String, nullable=True)
|
||||
fee_open_currency = Column(String(25), nullable=True)
|
||||
fee_close = Column(Float, nullable=False, default=0.0)
|
||||
fee_close_cost = Column(Float, nullable=True)
|
||||
fee_close_currency = Column(String, nullable=True)
|
||||
fee_close_currency = Column(String(25), nullable=True)
|
||||
open_rate = Column(Float)
|
||||
open_rate_requested = Column(Float)
|
||||
# open_trade_value - calculated via _calc_open_trade_value
|
||||
@@ -776,7 +685,7 @@ class Trade(_DECL_BASE, LocalTrade):
|
||||
amount_requested = Column(Float)
|
||||
open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||
close_date = Column(DateTime)
|
||||
open_order_id = Column(String)
|
||||
open_order_id = Column(String(255))
|
||||
# absolute value of the stop loss
|
||||
stop_loss = Column(Float, nullable=True, default=0.0)
|
||||
# percentage value of the stop loss
|
||||
@@ -786,16 +695,17 @@ class Trade(_DECL_BASE, LocalTrade):
|
||||
# percentage value of the initial stop loss
|
||||
initial_stop_loss_pct = Column(Float, nullable=True)
|
||||
# stoploss order id which is on exchange
|
||||
stoploss_order_id = Column(String, nullable=True, index=True)
|
||||
stoploss_order_id = Column(String(255), nullable=True, index=True)
|
||||
# last update time of the stoploss order on exchange
|
||||
stoploss_last_update = Column(DateTime, nullable=True)
|
||||
# absolute value of the highest reached price
|
||||
max_rate = Column(Float, nullable=True, default=0.0)
|
||||
# Lowest price reached
|
||||
min_rate = Column(Float, nullable=True)
|
||||
sell_reason = Column(String, nullable=True)
|
||||
sell_order_status = Column(String, nullable=True)
|
||||
strategy = Column(String, nullable=True)
|
||||
sell_reason = Column(String(100), nullable=True)
|
||||
sell_order_status = Column(String(100), nullable=True)
|
||||
strategy = Column(String(100), nullable=True)
|
||||
buy_tag = Column(String(100), nullable=True)
|
||||
timeframe = Column(Integer, nullable=True)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
@@ -805,17 +715,21 @@ class Trade(_DECL_BASE, LocalTrade):
|
||||
def delete(self) -> None:
|
||||
|
||||
for order in self.orders:
|
||||
Order.session.delete(order)
|
||||
Order.query.session.delete(order)
|
||||
|
||||
Trade.session.delete(self)
|
||||
Trade.session.flush()
|
||||
Trade.query.session.delete(self)
|
||||
Trade.commit()
|
||||
|
||||
@staticmethod
|
||||
def commit():
|
||||
Trade.query.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
|
||||
open_date: datetime = None, close_date: datetime = None,
|
||||
) -> List['LocalTrade']:
|
||||
"""
|
||||
Helper function to query Trades.
|
||||
Helper function to query Trades.j
|
||||
Returns a List of trades, filtered on the parameters given.
|
||||
In live mode, converts the filter to a database query and returns all rows
|
||||
In Backtest mode, uses filters on Trade.trades to get the result.
|
||||
@@ -840,6 +754,126 @@ class Trade(_DECL_BASE, LocalTrade):
|
||||
close_date=close_date
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_trades(trade_filter=None) -> Query:
|
||||
"""
|
||||
Helper function to query Trades using filters.
|
||||
NOTE: Not supported in Backtesting.
|
||||
:param trade_filter: Optional filter to apply to trades
|
||||
Can be either a Filter object, or a List of filters
|
||||
e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
|
||||
e.g. `(trade_filter=Trade.id == trade_id)`
|
||||
:return: unsorted query object
|
||||
"""
|
||||
if not Trade.use_db:
|
||||
raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.')
|
||||
if trade_filter is not None:
|
||||
if not isinstance(trade_filter, list):
|
||||
trade_filter = [trade_filter]
|
||||
return Trade.query.filter(*trade_filter)
|
||||
else:
|
||||
return Trade.query
|
||||
|
||||
@staticmethod
|
||||
def get_open_order_trades():
|
||||
"""
|
||||
Returns all open trades
|
||||
NOTE: Not supported in Backtesting.
|
||||
"""
|
||||
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
|
||||
|
||||
@staticmethod
|
||||
def get_open_trades_without_assigned_fees():
|
||||
"""
|
||||
Returns all open trades which don't have open fees set correctly
|
||||
NOTE: Not supported in Backtesting.
|
||||
"""
|
||||
return Trade.get_trades([Trade.fee_open_currency.is_(None),
|
||||
Trade.orders.any(),
|
||||
Trade.is_open.is_(True),
|
||||
]).all()
|
||||
|
||||
@staticmethod
|
||||
def get_sold_trades_without_assigned_fees():
|
||||
"""
|
||||
Returns all closed trades which don't have fees set correctly
|
||||
NOTE: Not supported in Backtesting.
|
||||
"""
|
||||
return Trade.get_trades([Trade.fee_close_currency.is_(None),
|
||||
Trade.orders.any(),
|
||||
Trade.is_open.is_(False),
|
||||
]).all()
|
||||
|
||||
@staticmethod
|
||||
def get_total_closed_profit() -> float:
|
||||
"""
|
||||
Retrieves total realized profit
|
||||
"""
|
||||
if Trade.use_db:
|
||||
total_profit = Trade.query.with_entities(
|
||||
func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar()
|
||||
else:
|
||||
total_profit = sum(
|
||||
t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False))
|
||||
return total_profit or 0
|
||||
|
||||
@staticmethod
|
||||
def total_open_trades_stakes() -> float:
|
||||
"""
|
||||
Calculates total invested amount in open trades
|
||||
in stake currency
|
||||
"""
|
||||
if Trade.use_db:
|
||||
total_open_stake_amount = Trade.query.with_entities(
|
||||
func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
|
||||
else:
|
||||
total_open_stake_amount = sum(
|
||||
t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True))
|
||||
return total_open_stake_amount or 0
|
||||
|
||||
@staticmethod
|
||||
def get_overall_performance(minutes=None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Returns List of dicts containing all Trades, including profit and trade count
|
||||
NOTE: Not supported in Backtesting.
|
||||
"""
|
||||
filters = [Trade.is_open.is_(False)]
|
||||
if minutes:
|
||||
start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes)
|
||||
filters.append(Trade.close_date >= start_date)
|
||||
pair_rates = Trade.query.with_entities(
|
||||
Trade.pair,
|
||||
func.sum(Trade.close_profit).label('profit_sum'),
|
||||
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
|
||||
func.count(Trade.pair).label('count')
|
||||
).filter(*filters)\
|
||||
.group_by(Trade.pair) \
|
||||
.order_by(desc('profit_sum_abs')) \
|
||||
.all()
|
||||
return [
|
||||
{
|
||||
'pair': pair,
|
||||
'profit': profit,
|
||||
'profit_abs': profit_abs,
|
||||
'count': count
|
||||
}
|
||||
for pair, profit, profit_abs, count in pair_rates
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)):
|
||||
"""
|
||||
Get best pair with closed trade.
|
||||
NOTE: Not supported in Backtesting.
|
||||
:returns: Tuple containing (pair, profit_sum)
|
||||
"""
|
||||
best_pair = Trade.query.with_entities(
|
||||
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
|
||||
).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date)) \
|
||||
.group_by(Trade.pair) \
|
||||
.order_by(desc('profit_sum')).first()
|
||||
return best_pair
|
||||
|
||||
|
||||
class PairLock(_DECL_BASE):
|
||||
"""
|
||||
@@ -849,8 +883,8 @@ class PairLock(_DECL_BASE):
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
||||
pair = Column(String, nullable=False, index=True)
|
||||
reason = Column(String, nullable=True)
|
||||
pair = Column(String(25), nullable=False, index=True)
|
||||
reason = Column(String(255), nullable=True)
|
||||
# Time the pair was locked (start time)
|
||||
lock_time = Column(DateTime, nullable=False)
|
||||
# Time until the pair is locked (end time)
|
||||
|
@@ -30,7 +30,8 @@ class PairLocks():
|
||||
PairLocks.locks = []
|
||||
|
||||
@staticmethod
|
||||
def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None:
|
||||
def lock_pair(pair: str, until: datetime, reason: str = None, *,
|
||||
now: datetime = None) -> PairLock:
|
||||
"""
|
||||
Create PairLock from now to "until".
|
||||
Uses database by default, unless PairLocks.use_db is set to False,
|
||||
@@ -48,10 +49,11 @@ class PairLocks():
|
||||
active=True
|
||||
)
|
||||
if PairLocks.use_db:
|
||||
PairLock.session.add(lock)
|
||||
PairLock.session.flush()
|
||||
PairLock.query.session.add(lock)
|
||||
PairLock.query.session.commit()
|
||||
else:
|
||||
PairLocks.locks.append(lock)
|
||||
return lock
|
||||
|
||||
@staticmethod
|
||||
def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:
|
||||
@@ -99,7 +101,7 @@ class PairLocks():
|
||||
for lock in locks:
|
||||
lock.active = False
|
||||
if PairLocks.use_db:
|
||||
PairLock.session.flush()
|
||||
PairLock.query.session.commit()
|
||||
|
||||
@staticmethod
|
||||
def is_global_lock(now: Optional[datetime] = None) -> bool:
|
||||
|
@@ -47,7 +47,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
|
||||
data = load_data(
|
||||
datadir=config.get('datadir'),
|
||||
pairs=pairs,
|
||||
timeframe=config.get('timeframe', '5m'),
|
||||
timeframe=config['timeframe'],
|
||||
timerange=timerange,
|
||||
startup_candles=startup_candles,
|
||||
data_format=config.get('dataformat_ohlcv', 'json'),
|
||||
@@ -56,7 +56,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
|
||||
if startup_candles and data:
|
||||
min_date, max_date = get_timerange(data)
|
||||
logger.info(f"Loading data from {min_date} to {max_date}")
|
||||
timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')),
|
||||
timerange.adjust_start_if_necessary(timeframe_to_seconds(config['timeframe']),
|
||||
startup_candles, min_date)
|
||||
|
||||
no_trades = False
|
||||
@@ -77,7 +77,8 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
|
||||
)
|
||||
except ValueError as e:
|
||||
raise OperationalException(e) from e
|
||||
trades = trim_dataframe(trades, timerange, 'open_date')
|
||||
if not trades.empty:
|
||||
trades = trim_dataframe(trades, timerange, 'open_date')
|
||||
|
||||
return {"ohlcv": data,
|
||||
"trades": trades,
|
||||
@@ -95,20 +96,34 @@ def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) ->
|
||||
Dict key must correspond to dataframe column.
|
||||
:param data: candlestick DataFrame
|
||||
"""
|
||||
plot_kinds = {
|
||||
'scatter': go.Scatter,
|
||||
'bar': go.Bar,
|
||||
}
|
||||
for indicator, conf in indicators.items():
|
||||
logger.debug(f"indicator {indicator} with config {conf}")
|
||||
if indicator in data:
|
||||
kwargs = {'x': data['date'],
|
||||
'y': data[indicator].values,
|
||||
'mode': 'lines',
|
||||
'name': indicator
|
||||
}
|
||||
if 'color' in conf:
|
||||
kwargs.update({'line': {'color': conf['color']}})
|
||||
scatter = go.Scatter(
|
||||
**kwargs
|
||||
)
|
||||
fig.add_trace(scatter, row, 1)
|
||||
|
||||
plot_type = conf.get('type', 'scatter')
|
||||
color = conf.get('color')
|
||||
if plot_type == 'bar':
|
||||
kwargs.update({'marker_color': color or 'DarkSlateGrey',
|
||||
'marker_line_color': color or 'DarkSlateGrey'})
|
||||
else:
|
||||
if color:
|
||||
kwargs.update({'line': {'color': color}})
|
||||
kwargs['mode'] = 'lines'
|
||||
if plot_type != 'scatter':
|
||||
logger.warning(f'Indicator {indicator} has unknown plot trace kind {plot_type}'
|
||||
f', assuming "scatter".')
|
||||
|
||||
kwargs.update(conf.get('plotly', {}))
|
||||
trace = plot_kinds[plot_type](**kwargs)
|
||||
fig.add_trace(trace, row, 1)
|
||||
else:
|
||||
logger.info(
|
||||
'Indicator "%s" ignored. Reason: This indicator is not found '
|
||||
@@ -273,8 +288,8 @@ def plot_area(fig, row: int, data: pd.DataFrame, indicator_a: str,
|
||||
:param fig: Plot figure to append to
|
||||
:param row: row number for this plot
|
||||
:param data: candlestick DataFrame
|
||||
:param indicator_a: indicator name as populated in stragetie
|
||||
:param indicator_b: indicator name as populated in stragetie
|
||||
:param indicator_a: indicator name as populated in strategy
|
||||
:param indicator_b: indicator name as populated in strategy
|
||||
:param label: label for the filled area
|
||||
:param fill_color: color to be used for the filled area
|
||||
:return: fig with added filled_traces plot
|
||||
@@ -319,8 +334,8 @@ def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots:
|
||||
)
|
||||
elif indicator_b not in data:
|
||||
logger.info(
|
||||
'fill_to: "%s" ignored. Reason: This indicator is not '
|
||||
'in your strategy.', indicator_b
|
||||
'fill_to: "%s" ignored. Reason: This indicator is not '
|
||||
'in your strategy.', indicator_b
|
||||
)
|
||||
return fig
|
||||
|
||||
@@ -358,6 +373,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
|
||||
for i, name in enumerate(plot_config['subplots']):
|
||||
fig['layout'][f'yaxis{3 + i}'].update(title=name)
|
||||
fig['layout']['xaxis']['rangeslider'].update(visible=False)
|
||||
fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
|
||||
|
||||
# Common information
|
||||
candles = go.Candlestick(
|
||||
@@ -437,11 +453,12 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
|
||||
data=data)
|
||||
# fill area between indicators ( 'fill_to': 'other_indicator')
|
||||
fig = add_areas(fig, row, data, sub_config)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
|
||||
trades: pd.DataFrame, timeframe: str) -> go.Figure:
|
||||
trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure:
|
||||
# Combine close-values for all pairs, rename columns to "pair"
|
||||
df_comb = combine_dataframes_with_mean(data, "close")
|
||||
|
||||
@@ -466,9 +483,10 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
|
||||
subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"])
|
||||
fig['layout'].update(title="Freqtrade Profit plot")
|
||||
fig['layout']['yaxis1'].update(title='Price')
|
||||
fig['layout']['yaxis2'].update(title='Profit')
|
||||
fig['layout']['yaxis3'].update(title='Profit')
|
||||
fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
|
||||
fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
|
||||
fig['layout']['xaxis']['rangeslider'].update(visible=False)
|
||||
fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
|
||||
|
||||
fig.add_trace(avgclose, 1, 1)
|
||||
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
|
||||
@@ -482,7 +500,6 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
|
||||
fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
@@ -521,7 +538,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
|
||||
- Initializes plot-script
|
||||
- Get candle (OHLCV) data
|
||||
- Generate Dafaframes populated with indicators and signals based on configured strategy
|
||||
- Load trades excecuted during the selected period
|
||||
- Load trades executed during the selected period
|
||||
- Generate Plotly plot objects
|
||||
- Generate plot files
|
||||
:return: None
|
||||
@@ -540,8 +557,11 @@ def load_and_plot_trades(config: Dict[str, Any]):
|
||||
|
||||
df_analyzed = strategy.analyze_ticker(data, {'pair': pair})
|
||||
df_analyzed = trim_dataframe(df_analyzed, timerange)
|
||||
trades_pair = trades.loc[trades['pair'] == pair]
|
||||
trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
|
||||
if not trades.empty:
|
||||
trades_pair = trades.loc[trades['pair'] == pair]
|
||||
trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
|
||||
else:
|
||||
trades_pair = trades
|
||||
|
||||
fig = generate_candlestick_graph(
|
||||
pair=pair,
|
||||
@@ -565,6 +585,9 @@ def plot_profit(config: Dict[str, Any]) -> None:
|
||||
But should be somewhat proportional, and therefor useful
|
||||
in helping out to find a good algorithm.
|
||||
"""
|
||||
if 'timeframe' not in config:
|
||||
raise OperationalException('Timeframe must be set in either config or via --timeframe.')
|
||||
|
||||
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config)
|
||||
plot_elements = init_plotscript(config, list(exchange.markets))
|
||||
trades = plot_elements['trades']
|
||||
@@ -581,6 +604,8 @@ def plot_profit(config: Dict[str, Any]) -> None:
|
||||
# Create an average close price of all the pairs that were involved.
|
||||
# this could be useful to gauge the overall market trend
|
||||
fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
|
||||
trades, config.get('timeframe', '5m'))
|
||||
trades, config['timeframe'],
|
||||
config.get('stake_currency', ''))
|
||||
store_plot_file(fig, filename='freqtrade-profit-plot.html',
|
||||
directory=config['user_data_dir'] / 'plot', auto_open=True)
|
||||
directory=config['user_data_dir'] / 'plot',
|
||||
auto_open=config.get('plot_auto_open', False))
|
||||
|
@@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
|
||||
import arrow
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.configuration import PeriodicCache
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import plural
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
@@ -18,15 +19,17 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class AgeFilter(IPairList):
|
||||
|
||||
# Checked symbols cache (dictionary of ticker symbol => timestamp)
|
||||
_symbolsChecked: Dict[str, int] = {}
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
# Checked symbols cache (dictionary of ticker symbol => timestamp)
|
||||
self._symbolsChecked: Dict[str, int] = {}
|
||||
self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
|
||||
|
||||
self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
|
||||
self._max_days_listed = pairlistconfig.get('max_days_listed', None)
|
||||
|
||||
if self._min_days_listed < 1:
|
||||
raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
|
||||
@@ -34,6 +37,12 @@ class AgeFilter(IPairList):
|
||||
raise OperationalException("AgeFilter requires min_days_listed to not exceed "
|
||||
"exchange max request size "
|
||||
f"({exchange.ohlcv_candle_limit('1d')})")
|
||||
if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
|
||||
raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
|
||||
if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
|
||||
raise OperationalException("AgeFilter requires max_days_listed to not exceed "
|
||||
"exchange max request size "
|
||||
f"({exchange.ohlcv_candle_limit('1d')})")
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
@@ -48,8 +57,13 @@ class AgeFilter(IPairList):
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
"""
|
||||
return (f"{self.name} - Filtering pairs with age less than "
|
||||
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
|
||||
return (
|
||||
f"{self.name} - Filtering pairs with age less than "
|
||||
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
|
||||
) + ((
|
||||
" or more than "
|
||||
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
|
||||
) if self._max_days_listed else '')
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
@@ -57,13 +71,19 @@ class AgeFilter(IPairList):
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: new allowlist
|
||||
"""
|
||||
needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
|
||||
needed_pairs = [
|
||||
(p, '1d') for p in pairlist
|
||||
if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
|
||||
if not needed_pairs:
|
||||
return pairlist
|
||||
# Remove pairs that have been removed before
|
||||
return [p for p in pairlist if p not in self._symbolsCheckFailed]
|
||||
|
||||
since_days = -(
|
||||
self._max_days_listed if self._max_days_listed else self._min_days_listed
|
||||
) - 1
|
||||
since_ms = int(arrow.utcnow()
|
||||
.floor('day')
|
||||
.shift(days=-self._min_days_listed - 1)
|
||||
.shift(days=since_days)
|
||||
.float_timestamp) * 1000
|
||||
candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
|
||||
if self._enabled:
|
||||
@@ -71,14 +91,14 @@ class AgeFilter(IPairList):
|
||||
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
|
||||
if not self._validate_pair_loc(p, daily_candles):
|
||||
pairlist.remove(p)
|
||||
logger.info(f"Validated {len(pairlist)} pairs.")
|
||||
self.log_once(f"Validated {len(pairlist)} pairs.", logger.info)
|
||||
return pairlist
|
||||
|
||||
def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
|
||||
"""
|
||||
Validate age for the ticker
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.load_markets()
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
# Check symbol in cache
|
||||
@@ -86,14 +106,23 @@ class AgeFilter(IPairList):
|
||||
return True
|
||||
|
||||
if daily_candles is not None:
|
||||
if len(daily_candles) > self._min_days_listed:
|
||||
if (
|
||||
len(daily_candles) >= self._min_days_listed
|
||||
and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed)
|
||||
):
|
||||
# We have fetched at least the minimum required number of daily candles
|
||||
# Add to cache, store the time we last checked this symbol
|
||||
self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
|
||||
self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
|
||||
return True
|
||||
else:
|
||||
self.log_once(f"Removed {pair} from whitelist, because age "
|
||||
f"{len(daily_candles)} is less than {self._min_days_listed} "
|
||||
f"{plural(self._min_days_listed, 'day')}", logger.info)
|
||||
self.log_once((
|
||||
f"Removed {pair} from whitelist, because age "
|
||||
f"{len(daily_candles)} is less than {self._min_days_listed} "
|
||||
f"{plural(self._min_days_listed, 'day')}"
|
||||
) + ((
|
||||
" or more than "
|
||||
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
|
||||
) if self._max_days_listed else ''), logger.info)
|
||||
self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
|
||||
return False
|
||||
return False
|
||||
|
@@ -7,7 +7,7 @@ from copy import deepcopy
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import market_is_active
|
||||
from freqtrade.exchange import Exchange, market_is_active
|
||||
from freqtrade.mixins import LoggingMixin
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class IPairList(LoggingMixin, ABC):
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
def __init__(self, exchange: Exchange, pairlistmanager,
|
||||
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
"""
|
||||
@@ -28,7 +28,7 @@ class IPairList(LoggingMixin, ABC):
|
||||
"""
|
||||
self._enabled = True
|
||||
|
||||
self._exchange = exchange
|
||||
self._exchange: Exchange = exchange
|
||||
self._pairlistmanager = pairlistmanager
|
||||
self._config = config
|
||||
self._pairlistconfig = pairlistconfig
|
||||
@@ -68,12 +68,12 @@ class IPairList(LoggingMixin, ABC):
|
||||
filter_pairlist() method.
|
||||
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.load_markets()
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
def gen_pairlist(self, tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Generate the pairlist.
|
||||
|
||||
@@ -84,8 +84,7 @@ class IPairList(LoggingMixin, ABC):
|
||||
it will raise the exception if a Pairlist Handler is used at the first
|
||||
position in the chain.
|
||||
|
||||
:param cached_pairlist: Previously generated pairlist (cached)
|
||||
:param tickers: Tickers (from exchange.get_tickers()).
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: List of pairs
|
||||
"""
|
||||
raise OperationalException("This Pairlist Handler should not be used "
|
||||
@@ -145,24 +144,26 @@ class IPairList(LoggingMixin, ABC):
|
||||
markets = self._exchange.markets
|
||||
if not markets:
|
||||
raise OperationalException(
|
||||
'Markets not loaded. Make sure that exchange is initialized correctly.')
|
||||
'Markets not loaded. Make sure that exchange is initialized correctly.')
|
||||
|
||||
sanitized_whitelist: List[str] = []
|
||||
for pair in pairlist:
|
||||
# pair is not in the generated dynamic market or has the wrong stake currency
|
||||
if pair not in markets:
|
||||
logger.warning(f"Pair {pair} is not compatible with exchange "
|
||||
f"{self._exchange.name}. Removing it from whitelist..")
|
||||
self.log_once(f"Pair {pair} is not compatible with exchange "
|
||||
f"{self._exchange.name}. Removing it from whitelist..",
|
||||
logger.warning)
|
||||
continue
|
||||
|
||||
if not self._exchange.market_is_tradable(markets[pair]):
|
||||
logger.warning(f"Pair {pair} is not tradable with Freqtrade."
|
||||
"Removing it from whitelist..")
|
||||
self.log_once(f"Pair {pair} is not tradable with Freqtrade."
|
||||
"Removing it from whitelist..", logger.warning)
|
||||
continue
|
||||
|
||||
if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
|
||||
logger.warning(f"Pair {pair} is not compatible with your stake currency "
|
||||
f"{self._config['stake_currency']}. Removing it from whitelist..")
|
||||
self.log_once(f"Pair {pair} is not compatible with your stake currency "
|
||||
f"{self._config['stake_currency']}. Removing it from whitelist..",
|
||||
logger.warning)
|
||||
continue
|
||||
|
||||
# Check if market is active
|
||||
|
54
freqtrade/plugins/pairlist/OffsetFilter.py
Normal file
54
freqtrade/plugins/pairlist/OffsetFilter.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
Offset pair list filter
|
||||
"""
|
||||
import logging
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OffsetFilter(IPairList):
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
self._offset = pairlistconfig.get('offset', 0)
|
||||
|
||||
if self._offset < 0:
|
||||
raise OperationalException("OffsetFilter requires offset to be >= 0")
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
Boolean property defining if tickers are necessary.
|
||||
If no Pairlist requires tickers, an empty Dict is passed
|
||||
as tickers argument to filter_pairlist
|
||||
"""
|
||||
return False
|
||||
|
||||
def short_desc(self) -> str:
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
"""
|
||||
return f"{self.name} - Offseting pairs by {self._offset}."
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Filters and sorts pairlist and returns the whitelist again.
|
||||
Called on each bot iteration - please use internal caching if necessary
|
||||
:param pairlist: pairlist to filter or sort
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: new whitelist
|
||||
"""
|
||||
if self._offset > len(pairlist):
|
||||
self.log_once(f"Offset of {self._offset} is larger than " +
|
||||
f"pair count of {len(pairlist)}", logger.warning)
|
||||
pairs = pairlist[self._offset:]
|
||||
self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
|
||||
return pairs
|
@@ -20,11 +20,14 @@ class PerformanceFilter(IPairList):
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
self._minutes = pairlistconfig.get('minutes', 0)
|
||||
self._min_profit = pairlistconfig.get('min_profit', None)
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
Boolean property defining if tickers are necessary.
|
||||
If no Pairlist requries tickers, an empty List is passed
|
||||
If no Pairlist requires tickers, an empty List is passed
|
||||
as tickers argument to filter_pairlist
|
||||
"""
|
||||
return False
|
||||
@@ -44,7 +47,12 @@ class PerformanceFilter(IPairList):
|
||||
:return: new allowlist
|
||||
"""
|
||||
# Get the trading performance for pairs from database
|
||||
performance = pd.DataFrame(Trade.get_overall_performance())
|
||||
try:
|
||||
performance = pd.DataFrame(Trade.get_overall_performance(self._minutes))
|
||||
except AttributeError:
|
||||
# Performancefilter does not work in backtesting.
|
||||
self.log_once("PerformanceFilter is not available in this mode.", logger.warning)
|
||||
return pairlist
|
||||
|
||||
# Skip performance-based sorting if no performance data is available
|
||||
if len(performance) == 0:
|
||||
@@ -61,6 +69,14 @@ class PerformanceFilter(IPairList):
|
||||
sorted_df = list_df.merge(performance, on='pair', how='left')\
|
||||
.fillna(0).sort_values(by=['count', 'pair'], ascending=True)\
|
||||
.sort_values(by=['profit'], ascending=False)
|
||||
if self._min_profit is not None:
|
||||
removed = sorted_df[sorted_df['profit'] < self._min_profit]
|
||||
for _, row in removed.iterrows():
|
||||
self.log_once(
|
||||
f"Removing pair {row['pair']} since {row['profit']} is "
|
||||
f"below {self._min_profit}", logger.info)
|
||||
sorted_df = sorted_df[sorted_df['profit'] >= self._min_profit]
|
||||
|
||||
pairlist = sorted_df['pair'].tolist()
|
||||
|
||||
return pairlist
|
||||
|
@@ -48,7 +48,7 @@ class PrecisionFilter(IPairList):
|
||||
Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
|
||||
low value pairs.
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.load_markets()
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
stop_price = ticker['ask'] * self._stoploss
|
||||
|
@@ -27,9 +27,13 @@ class PriceFilter(IPairList):
|
||||
self._max_price = pairlistconfig.get('max_price', 0)
|
||||
if self._max_price < 0:
|
||||
raise OperationalException("PriceFilter requires max_price to be >= 0")
|
||||
self._max_value = pairlistconfig.get('max_value', 0)
|
||||
if self._max_value < 0:
|
||||
raise OperationalException("PriceFilter requires max_value to be >= 0")
|
||||
self._enabled = ((self._low_price_ratio > 0) or
|
||||
(self._min_price > 0) or
|
||||
(self._max_price > 0))
|
||||
(self._max_price > 0) or
|
||||
(self._max_value > 0))
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
@@ -51,6 +55,8 @@ class PriceFilter(IPairList):
|
||||
active_price_filters.append(f"below {self._min_price:.8f}")
|
||||
if self._max_price != 0:
|
||||
active_price_filters.append(f"above {self._max_price:.8f}")
|
||||
if self._max_value != 0:
|
||||
active_price_filters.append(f"Value above {self._max_value:.8f}")
|
||||
|
||||
if len(active_price_filters):
|
||||
return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}."
|
||||
@@ -61,10 +67,10 @@ class PriceFilter(IPairList):
|
||||
"""
|
||||
Check if if one price-step (pip) is > than a certain barrier.
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.load_markets()
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
if ticker['last'] is None or ticker['last'] == 0:
|
||||
if ticker.get('last', None) is None or ticker.get('last') == 0:
|
||||
self.log_once(f"Removed {pair} from whitelist, because "
|
||||
"ticker['last'] is empty (Usually no trade in the last 24h).",
|
||||
logger.info)
|
||||
@@ -79,6 +85,32 @@ class PriceFilter(IPairList):
|
||||
f"because 1 unit is {changeperc * 100:.3f}%", logger.info)
|
||||
return False
|
||||
|
||||
# Perform low_amount check
|
||||
if self._max_value != 0:
|
||||
price = ticker['last']
|
||||
market = self._exchange.markets[pair]
|
||||
limits = market['limits']
|
||||
if ('amount' in limits and 'min' in limits['amount']
|
||||
and limits['amount']['min'] is not None):
|
||||
min_amount = limits['amount']['min']
|
||||
min_precision = market['precision']['amount']
|
||||
|
||||
min_value = min_amount * price
|
||||
if self._exchange.precisionMode == 4:
|
||||
# tick size
|
||||
next_value = (min_amount + min_precision) * price
|
||||
else:
|
||||
# Decimal places
|
||||
min_precision = pow(0.1, min_precision)
|
||||
next_value = (min_amount + min_precision) * price
|
||||
diff = next_value - min_value
|
||||
|
||||
if diff > self._max_value:
|
||||
self.log_once(f"Removed {pair} from whitelist, "
|
||||
f"because min value change of {diff} > {self._max_value}.",
|
||||
logger.info)
|
||||
return False
|
||||
|
||||
# Perform min_price check.
|
||||
if self._min_price != 0:
|
||||
if ticker['last'] < self._min_price:
|
||||
@@ -89,7 +121,7 @@ class PriceFilter(IPairList):
|
||||
# Perform max_price check.
|
||||
if self._max_price != 0:
|
||||
if ticker['last'] > self._max_price:
|
||||
self.log_once(f"Removed {ticker['symbol']} from whitelist, "
|
||||
self.log_once(f"Removed {pair} from whitelist, "
|
||||
f"because last price > {self._max_price:.8f}", logger.info)
|
||||
return False
|
||||
|
||||
|
@@ -40,7 +40,7 @@ class SpreadFilter(IPairList):
|
||||
"""
|
||||
Validate spread for the ticker
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.load_markets()
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
if 'bid' in ticker and 'ask' in ticker and ticker['ask']:
|
||||
|
@@ -42,11 +42,10 @@ class StaticPairList(IPairList):
|
||||
"""
|
||||
return f"{self.name}"
|
||||
|
||||
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
def gen_pairlist(self, tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Generate the pairlist
|
||||
:param cached_pairlist: Previously generated pairlist (cached)
|
||||
:param tickers: Tickers (from exchange.get_tickers()).
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: List of pairs
|
||||
"""
|
||||
if self._allow_inactive:
|
||||
|
121
freqtrade/plugins/pairlist/VolatilityFilter.py
Normal file
121
freqtrade/plugins/pairlist/VolatilityFilter.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""
|
||||
Volatility pairlist filter
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import arrow
|
||||
import numpy as np
|
||||
from cachetools.ttl import TTLCache
|
||||
from pandas import DataFrame
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.misc import plural
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VolatilityFilter(IPairList):
|
||||
"""
|
||||
Filters pairs by volatility
|
||||
"""
|
||||
|
||||
def __init__(self, exchange, pairlistmanager,
|
||||
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
|
||||
pairlist_pos: int) -> None:
|
||||
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
|
||||
|
||||
self._days = pairlistconfig.get('lookback_days', 10)
|
||||
self._min_volatility = pairlistconfig.get('min_volatility', 0)
|
||||
self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
|
||||
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
|
||||
|
||||
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
|
||||
|
||||
if self._days < 1:
|
||||
raise OperationalException("VolatilityFilter requires lookback_days to be >= 1")
|
||||
if self._days > exchange.ohlcv_candle_limit('1d'):
|
||||
raise OperationalException("VolatilityFilter requires lookback_days to not "
|
||||
"exceed exchange max request size "
|
||||
f"({exchange.ohlcv_candle_limit('1d')})")
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
Boolean property defining if tickers are necessary.
|
||||
If no Pairlist requires tickers, an empty List is passed
|
||||
as tickers argument to filter_pairlist
|
||||
"""
|
||||
return False
|
||||
|
||||
def short_desc(self) -> str:
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
"""
|
||||
return (f"{self.name} - Filtering pairs with volatility range "
|
||||
f"{self._min_volatility}-{self._max_volatility} "
|
||||
f" the last {self._days} {plural(self._days, 'day')}.")
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Validate trading range
|
||||
:param pairlist: pairlist to filter or sort
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: new allowlist
|
||||
"""
|
||||
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
|
||||
|
||||
since_ms = (arrow.utcnow()
|
||||
.floor('day')
|
||||
.shift(days=-self._days - 1)
|
||||
.int_timestamp) * 1000
|
||||
# Get all candles
|
||||
candles = {}
|
||||
if needed_pairs:
|
||||
candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
|
||||
cache=False)
|
||||
|
||||
if self._enabled:
|
||||
for p in deepcopy(pairlist):
|
||||
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
|
||||
if not self._validate_pair_loc(p, daily_candles):
|
||||
pairlist.remove(p)
|
||||
return pairlist
|
||||
|
||||
def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
|
||||
"""
|
||||
Validate trading range
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
# Check symbol in cache
|
||||
cached_res = self._pair_cache.get(pair, None)
|
||||
if cached_res is not None:
|
||||
return cached_res
|
||||
|
||||
result = False
|
||||
if daily_candles is not None and not daily_candles.empty:
|
||||
returns = (np.log(daily_candles.close / daily_candles.close.shift(-1)))
|
||||
returns.fillna(0, inplace=True)
|
||||
|
||||
volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days)
|
||||
volatility_avg = volatility_series.mean()
|
||||
|
||||
if self._min_volatility <= volatility_avg <= self._max_volatility:
|
||||
result = True
|
||||
else:
|
||||
self.log_once(f"Removed {pair} from whitelist, because volatility "
|
||||
f"over {self._days} {plural(self._days, 'day')} "
|
||||
f"is: {volatility_avg:.3f} "
|
||||
f"which is not in the configured range of "
|
||||
f"{self._min_volatility}-{self._max_volatility}.",
|
||||
logger.info)
|
||||
result = False
|
||||
self._pair_cache[pair] = result
|
||||
|
||||
return result
|
@@ -4,10 +4,15 @@ Volume PairList provider
|
||||
Provides dynamic pair list based on trade volumes
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import arrow
|
||||
from cachetools.ttl import TTLCache
|
||||
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.exchange import timeframe_to_minutes
|
||||
from freqtrade.misc import format_ms_time
|
||||
from freqtrade.plugins.pairlist.IPairList import IPairList
|
||||
|
||||
|
||||
@@ -33,7 +38,37 @@ class VolumePairList(IPairList):
|
||||
self._number_pairs = self._pairlistconfig['number_assets']
|
||||
self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
|
||||
self._min_value = self._pairlistconfig.get('min_value', 0)
|
||||
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
|
||||
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
|
||||
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
|
||||
self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
|
||||
self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
|
||||
self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
|
||||
|
||||
if (self._lookback_days > 0) & (self._lookback_period > 0):
|
||||
raise OperationalException(
|
||||
'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
|
||||
'config. Please set lookback_days only or lookback_period and lookback_timeframe '
|
||||
'and restart the bot.'
|
||||
)
|
||||
|
||||
# overwrite lookback timeframe and days when lookback_days is set
|
||||
if self._lookback_days > 0:
|
||||
self._lookback_timeframe = '1d'
|
||||
self._lookback_period = self._lookback_days
|
||||
|
||||
# get timeframe in minutes and seconds
|
||||
self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
|
||||
self._tf_in_sec = self._tf_in_min * 60
|
||||
|
||||
# wether to use range lookback or not
|
||||
self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
|
||||
|
||||
if self._use_range & (self._refresh_period < self._tf_in_sec):
|
||||
raise OperationalException(
|
||||
f'Refresh period of {self._refresh_period} seconds is smaller than one '
|
||||
f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
|
||||
f'to at least {self._tf_in_sec} and restart the bot.'
|
||||
)
|
||||
|
||||
if not self._exchange.exchange_has('fetchTickers'):
|
||||
raise OperationalException(
|
||||
@@ -45,6 +80,13 @@ class VolumePairList(IPairList):
|
||||
raise OperationalException(
|
||||
f'key {self._sort_key} not in {SORT_VALUES}')
|
||||
|
||||
if self._lookback_period < 0:
|
||||
raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
|
||||
if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
|
||||
raise OperationalException("VolumeFilter requires lookback_period to not "
|
||||
"exceed exchange max request size "
|
||||
f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
|
||||
|
||||
@property
|
||||
def needstickers(self) -> bool:
|
||||
"""
|
||||
@@ -63,28 +105,29 @@ class VolumePairList(IPairList):
|
||||
"""
|
||||
return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
|
||||
|
||||
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
def gen_pairlist(self, tickers: Dict) -> List[str]:
|
||||
"""
|
||||
Generate the pairlist
|
||||
:param cached_pairlist: Previously generated pairlist (cached)
|
||||
:param tickers: Tickers (from exchange.get_tickers()).
|
||||
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
|
||||
:return: List of pairs
|
||||
"""
|
||||
# Generate dynamic whitelist
|
||||
# Must always run if this pairlist is not the first in the list.
|
||||
if self._last_refresh + self.refresh_period < datetime.now().timestamp():
|
||||
self._last_refresh = int(datetime.now().timestamp())
|
||||
|
||||
pairlist = self._pair_cache.get('pairlist')
|
||||
if pairlist:
|
||||
# Item found - no refresh necessary
|
||||
return pairlist.copy()
|
||||
else:
|
||||
# Use fresh pairlist
|
||||
# Check if pair quote currency equals to the stake currency.
|
||||
filtered_tickers = [
|
||||
v for k, v in tickers.items()
|
||||
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
|
||||
and v[self._sort_key] is not None)]
|
||||
v for k, v in tickers.items()
|
||||
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
|
||||
and (self._use_range or v[self._sort_key] is not None))]
|
||||
pairlist = [s['symbol'] for s in filtered_tickers]
|
||||
else:
|
||||
# Use the cached pairlist if it's not time yet to refresh
|
||||
pairlist = cached_pairlist
|
||||
|
||||
pairlist = self.filter_pairlist(pairlist, tickers)
|
||||
self._pair_cache['pairlist'] = pairlist.copy()
|
||||
|
||||
return pairlist
|
||||
|
||||
@@ -99,15 +142,69 @@ class VolumePairList(IPairList):
|
||||
# Use the incoming pairlist.
|
||||
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
|
||||
|
||||
# get lookback period in ms, for exchange ohlcv fetch
|
||||
if self._use_range:
|
||||
since_ms = int(arrow.utcnow()
|
||||
.floor('minute')
|
||||
.shift(minutes=-(self._lookback_period * self._tf_in_min)
|
||||
- self._tf_in_min)
|
||||
.int_timestamp) * 1000
|
||||
|
||||
to_ms = int(arrow.utcnow()
|
||||
.floor('minute')
|
||||
.shift(minutes=-self._tf_in_min)
|
||||
.int_timestamp) * 1000
|
||||
|
||||
# todo: utc date output for starting date
|
||||
self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
|
||||
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
|
||||
f"till {format_ms_time(to_ms)}", logger.info)
|
||||
needed_pairs = [
|
||||
(p, self._lookback_timeframe) for p in
|
||||
[
|
||||
s['symbol'] for s in filtered_tickers
|
||||
] if p not in self._pair_cache
|
||||
]
|
||||
|
||||
# Get all candles
|
||||
candles = {}
|
||||
if needed_pairs:
|
||||
candles = self._exchange.refresh_latest_ohlcv(
|
||||
needed_pairs, since_ms=since_ms, cache=False
|
||||
)
|
||||
for i, p in enumerate(filtered_tickers):
|
||||
pair_candles = candles[
|
||||
(p['symbol'], self._lookback_timeframe)
|
||||
] if (p['symbol'], self._lookback_timeframe) in candles else None
|
||||
# in case of candle data calculate typical price and quoteVolume for candle
|
||||
if pair_candles is not None and not pair_candles.empty:
|
||||
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
|
||||
+ pair_candles['close']) / 3
|
||||
pair_candles['quoteVolume'] = (
|
||||
pair_candles['volume'] * pair_candles['typical_price']
|
||||
)
|
||||
|
||||
# ensure that a rolling sum over the lookback_period is built
|
||||
# if pair_candles contains more candles than lookback_period
|
||||
quoteVolume = (pair_candles['quoteVolume']
|
||||
.rolling(self._lookback_period)
|
||||
.sum()
|
||||
.iloc[-1])
|
||||
|
||||
# replace quoteVolume with range quoteVolume sum calculated above
|
||||
filtered_tickers[i]['quoteVolume'] = quoteVolume
|
||||
else:
|
||||
filtered_tickers[i]['quoteVolume'] = 0
|
||||
|
||||
if self._min_value > 0:
|
||||
filtered_tickers = [
|
||||
v for v in filtered_tickers if v[self._sort_key] > self._min_value]
|
||||
v for v in filtered_tickers if v[self._sort_key] > self._min_value]
|
||||
|
||||
sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
|
||||
|
||||
# Validate whitelist to only have active market pairs
|
||||
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
|
||||
pairs = self.verify_blacklist(pairs, logger.info)
|
||||
pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
|
||||
# Limit pairlist to the requested number of pairs
|
||||
pairs = pairs[:self._number_pairs]
|
||||
|
||||
|
@@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
|
||||
if keep_invalid:
|
||||
for pair_wc in wildcardpl:
|
||||
try:
|
||||
comp = re.compile(pair_wc)
|
||||
comp = re.compile(pair_wc, re.IGNORECASE)
|
||||
result_partial = [
|
||||
pair for pair in available_pairs if re.fullmatch(comp, pair)
|
||||
]
|
||||
@@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
|
||||
else:
|
||||
for pair_wc in wildcardpl:
|
||||
try:
|
||||
comp = re.compile(pair_wc)
|
||||
comp = re.compile(pair_wc, re.IGNORECASE)
|
||||
result += [
|
||||
pair for pair in available_pairs if re.fullmatch(comp, pair)
|
||||
]
|
||||
|
@@ -26,6 +26,7 @@ class RangeStabilityFilter(IPairList):
|
||||
|
||||
self._days = pairlistconfig.get('lookback_days', 10)
|
||||
self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
|
||||
self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None)
|
||||
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
|
||||
|
||||
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
|
||||
@@ -50,8 +51,12 @@ class RangeStabilityFilter(IPairList):
|
||||
"""
|
||||
Short whitelist method description - used for startup-messages
|
||||
"""
|
||||
max_rate_desc = ""
|
||||
if self._max_rate_of_change:
|
||||
max_rate_desc = (f" and above {self._max_rate_of_change}")
|
||||
return (f"{self.name} - Filtering pairs with rate of change below "
|
||||
f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
|
||||
f"{self._min_rate_of_change}{max_rate_desc} over the "
|
||||
f"last {plural(self._days, 'day')}.")
|
||||
|
||||
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
|
||||
"""
|
||||
@@ -62,10 +67,10 @@ class RangeStabilityFilter(IPairList):
|
||||
"""
|
||||
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
|
||||
|
||||
since_ms = int(arrow.utcnow()
|
||||
.floor('day')
|
||||
.shift(days=-self._days - 1)
|
||||
.float_timestamp) * 1000
|
||||
since_ms = (arrow.utcnow()
|
||||
.floor('day')
|
||||
.shift(days=-self._days - 1)
|
||||
.int_timestamp) * 1000
|
||||
# Get all candles
|
||||
candles = {}
|
||||
if needed_pairs:
|
||||
@@ -83,12 +88,13 @@ class RangeStabilityFilter(IPairList):
|
||||
"""
|
||||
Validate trading range
|
||||
:param pair: Pair that's currently validated
|
||||
:param ticker: ticker dict as returned from ccxt.load_markets()
|
||||
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
|
||||
:return: True if the pair can stay, false if it should be removed
|
||||
"""
|
||||
# Check symbol in cache
|
||||
if pair in self._pair_cache:
|
||||
return self._pair_cache[pair]
|
||||
cached_res = self._pair_cache.get(pair, None)
|
||||
if cached_res is not None:
|
||||
return cached_res
|
||||
|
||||
result = False
|
||||
if daily_candles is not None and not daily_candles.empty:
|
||||
@@ -103,6 +109,17 @@ class RangeStabilityFilter(IPairList):
|
||||
f"which is below the threshold of {self._min_rate_of_change}.",
|
||||
logger.info)
|
||||
result = False
|
||||
if self._max_rate_of_change:
|
||||
if pct_change <= self._max_rate_of_change:
|
||||
result = True
|
||||
else:
|
||||
self.log_once(
|
||||
f"Removed {pair} from whitelist, because rate of change "
|
||||
f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
|
||||
f"which is above the threshold of {self._max_rate_of_change}.",
|
||||
logger.info)
|
||||
result = False
|
||||
self._pair_cache[pair] = result
|
||||
|
||||
else:
|
||||
self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
|
||||
return result
|
||||
|
@@ -3,7 +3,7 @@ PairList manager class
|
||||
"""
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, List
|
||||
from typing import Dict, List
|
||||
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
@@ -28,13 +28,13 @@ class PairListManager():
|
||||
self._tickers_needed = False
|
||||
for pairlist_handler_config in self._config.get('pairlists', None):
|
||||
pairlist_handler = PairListResolver.load_pairlist(
|
||||
pairlist_handler_config['method'],
|
||||
exchange=exchange,
|
||||
pairlistmanager=self,
|
||||
config=config,
|
||||
pairlistconfig=pairlist_handler_config,
|
||||
pairlist_pos=len(self._pairlist_handlers)
|
||||
)
|
||||
pairlist_handler_config['method'],
|
||||
exchange=exchange,
|
||||
pairlistmanager=self,
|
||||
config=config,
|
||||
pairlistconfig=pairlist_handler_config,
|
||||
pairlist_pos=len(self._pairlist_handlers)
|
||||
)
|
||||
self._tickers_needed |= pairlist_handler.needstickers
|
||||
self._pairlist_handlers.append(pairlist_handler)
|
||||
|
||||
@@ -79,14 +79,12 @@ class PairListManager():
|
||||
if self._tickers_needed:
|
||||
tickers = self._get_cached_tickers()
|
||||
|
||||
# Adjust whitelist if filters are using tickers
|
||||
pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
|
||||
|
||||
# Generate the pairlist with first Pairlist Handler in the chain
|
||||
pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
|
||||
pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
|
||||
|
||||
# Process all Pairlist Handlers in the chain
|
||||
for pairlist_handler in self._pairlist_handlers:
|
||||
# except for the first one, which is the generator.
|
||||
for pairlist_handler in self._pairlist_handlers[1:]:
|
||||
pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)
|
||||
|
||||
# Validation against blacklist happens after the chain of Pairlist Handlers
|
||||
@@ -95,19 +93,6 @@ class PairListManager():
|
||||
|
||||
self._whitelist = pairlist
|
||||
|
||||
def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
|
||||
"""
|
||||
Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
|
||||
pairs that do not have ticker available
|
||||
"""
|
||||
if self._tickers_needed:
|
||||
# Copy list since we're modifying this list
|
||||
for p in deepcopy(pairlist):
|
||||
if p not in tickers:
|
||||
pairlist.remove(p)
|
||||
|
||||
return pairlist
|
||||
|
||||
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
|
||||
"""
|
||||
Verify and remove items from pairlist - returning a filtered pairlist.
|
||||
|
@@ -6,6 +6,7 @@ from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from freqtrade.persistence import PairLocks
|
||||
from freqtrade.persistence.models import PairLock
|
||||
from freqtrade.plugins.protections import IProtection
|
||||
from freqtrade.resolvers import ProtectionResolver
|
||||
|
||||
@@ -15,11 +16,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class ProtectionManager():
|
||||
|
||||
def __init__(self, config: dict) -> None:
|
||||
def __init__(self, config: Dict, protections: List) -> None:
|
||||
self._config = config
|
||||
|
||||
self._protection_handlers: List[IProtection] = []
|
||||
for protection_handler_config in self._config.get('protections', []):
|
||||
for protection_handler_config in protections:
|
||||
protection_handler = ProtectionResolver.load_protection(
|
||||
protection_handler_config['method'],
|
||||
config=config,
|
||||
@@ -43,30 +44,28 @@ class ProtectionManager():
|
||||
"""
|
||||
return [{p.name: p.short_desc()} for p in self._protection_handlers]
|
||||
|
||||
def global_stop(self, now: Optional[datetime] = None) -> bool:
|
||||
def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]:
|
||||
if not now:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = False
|
||||
result = None
|
||||
for protection_handler in self._protection_handlers:
|
||||
if protection_handler.has_global_stop:
|
||||
result, until, reason = protection_handler.global_stop(now)
|
||||
lock, until, reason = protection_handler.global_stop(now)
|
||||
|
||||
# Early stopping - first positive result blocks further trades
|
||||
if result and until:
|
||||
if lock and until:
|
||||
if not PairLocks.is_global_lock(until):
|
||||
PairLocks.lock_pair('*', until, reason, now=now)
|
||||
result = True
|
||||
result = PairLocks.lock_pair('*', until, reason, now=now)
|
||||
return result
|
||||
|
||||
def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool:
|
||||
def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]:
|
||||
if not now:
|
||||
now = datetime.now(timezone.utc)
|
||||
result = False
|
||||
result = None
|
||||
for protection_handler in self._protection_handlers:
|
||||
if protection_handler.has_local_stop:
|
||||
result, until, reason = protection_handler.stop_per_pair(pair, now)
|
||||
if result and until:
|
||||
lock, until, reason = protection_handler.stop_per_pair(pair, now)
|
||||
if lock and until:
|
||||
if not PairLocks.is_pair_locked(pair, until):
|
||||
PairLocks.lock_pair(pair, until, reason, now=now)
|
||||
result = True
|
||||
result = PairLocks.lock_pair(pair, until, reason, now=now)
|
||||
return result
|
||||
|
@@ -1,7 +1,6 @@
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.persistence import Trade
|
||||
from freqtrade.plugins.protections import IProtection, ProtectionReturn
|
||||
@@ -15,9 +14,6 @@ class CooldownPeriod(IProtection):
|
||||
has_global_stop: bool = False
|
||||
has_local_stop: bool = True
|
||||
|
||||
def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
|
||||
super().__init__(config, protection_config)
|
||||
|
||||
def _reason(self) -> str:
|
||||
"""
|
||||
LockReason to use
|
||||
|
@@ -25,19 +25,22 @@ class IProtection(LoggingMixin, ABC):
|
||||
def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
|
||||
self._config = config
|
||||
self._protection_config = protection_config
|
||||
self._stop_duration_candles: Optional[int] = None
|
||||
self._lookback_period_candles: Optional[int] = None
|
||||
|
||||
tf_in_min = timeframe_to_minutes(config['timeframe'])
|
||||
if 'stop_duration_candles' in protection_config:
|
||||
self._stop_duration_candles = protection_config.get('stop_duration_candles', 1)
|
||||
self._stop_duration_candles = int(protection_config.get('stop_duration_candles', 1))
|
||||
self._stop_duration = (tf_in_min * self._stop_duration_candles)
|
||||
else:
|
||||
self._stop_duration_candles = None
|
||||
self._stop_duration = protection_config.get('stop_duration', 60)
|
||||
if 'lookback_period_candles' in protection_config:
|
||||
self._lookback_period_candles = protection_config.get('lookback_period_candles', 1)
|
||||
self._lookback_period_candles = int(protection_config.get('lookback_period_candles', 1))
|
||||
self._lookback_period = tf_in_min * self._lookback_period_candles
|
||||
else:
|
||||
self._lookback_period_candles = None
|
||||
self._lookback_period = protection_config.get('lookback_period', 60)
|
||||
self._lookback_period = int(protection_config.get('lookback_period', 60))
|
||||
|
||||
LoggingMixin.__init__(self, logger)
|
||||
|
||||
|
@@ -61,7 +61,7 @@ class MaxDrawdown(IProtection):
|
||||
|
||||
if drawdown > self._max_allowed_drawdown:
|
||||
self.log_once(
|
||||
f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}"
|
||||
f"Trading stopped due to Max Drawdown {drawdown:.2f} > {self._max_allowed_drawdown}"
|
||||
f" within {self.lookback_period_str}.", logger.info)
|
||||
until = self.calculate_lock_end(trades, self._stop_duration)
|
||||
|
||||
|
@@ -3,9 +3,9 @@ import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict
|
||||
|
||||
from freqtrade.enums import SellType
|
||||
from freqtrade.persistence import Trade
|
||||
from freqtrade.plugins.protections import IProtection, ProtectionReturn
|
||||
from freqtrade.strategy.interface import SellType
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -54,9 +54,9 @@ class StoplossGuard(IProtection):
|
||||
|
||||
trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until)
|
||||
trades = [trade for trade in trades1 if (str(trade.sell_reason) in (
|
||||
SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
|
||||
SellType.STOPLOSS_ON_EXCHANGE.value)
|
||||
and trade.close_profit and trade.close_profit < 0)]
|
||||
SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
|
||||
SellType.STOPLOSS_ON_EXCHANGE.value)
|
||||
and trade.close_profit and trade.close_profit < 0)]
|
||||
|
||||
if len(trades) < self._trade_limit:
|
||||
return False, None, None
|
||||
|
@@ -8,6 +8,3 @@ from freqtrade.resolvers.exchange_resolver import ExchangeResolver
|
||||
from freqtrade.resolvers.pairlist_resolver import PairListResolver
|
||||
from freqtrade.resolvers.protection_resolver import ProtectionResolver
|
||||
from freqtrade.resolvers.strategy_resolver import StrategyResolver
|
||||
|
||||
|
||||
|
||||
|
@@ -21,6 +21,7 @@ class ExchangeResolver(IResolver):
|
||||
def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange:
|
||||
"""
|
||||
Load the custom class from config parameter
|
||||
:param exchange_name: name of the Exchange to load
|
||||
:param config: configuration dictionary
|
||||
"""
|
||||
# Map exchange name to avoid duplicate classes for identical exchanges
|
||||
|
@@ -9,7 +9,6 @@ from typing import Dict
|
||||
|
||||
from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS
|
||||
from freqtrade.exceptions import OperationalException
|
||||
from freqtrade.optimize.hyperopt_interface import IHyperOpt
|
||||
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss
|
||||
from freqtrade.resolvers import IResolver
|
||||
|
||||
@@ -17,43 +16,6 @@ from freqtrade.resolvers import IResolver
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HyperOptResolver(IResolver):
|
||||
"""
|
||||
This class contains all the logic to load custom hyperopt class
|
||||
"""
|
||||
object_type = IHyperOpt
|
||||
object_type_str = "Hyperopt"
|
||||
user_subdir = USERPATH_HYPEROPTS
|
||||
initial_search_path = None
|
||||
|
||||
@staticmethod
|
||||
def load_hyperopt(config: Dict) -> IHyperOpt:
|
||||
"""
|
||||
Load the custom hyperopt class from config parameter
|
||||
:param config: configuration dictionary
|
||||
"""
|
||||
if not config.get('hyperopt'):
|
||||
raise OperationalException("No Hyperopt set. Please use `--hyperopt` to specify "
|
||||
"the Hyperopt class to use.")
|
||||
|
||||
hyperopt_name = config['hyperopt']
|
||||
|
||||
hyperopt = HyperOptResolver.load_object(hyperopt_name, config,
|
||||
kwargs={'config': config},
|
||||
extra_dir=config.get('hyperopt_path'))
|
||||
|
||||
if not hasattr(hyperopt, 'populate_indicators'):
|
||||
logger.info("Hyperopt class does not provide populate_indicators() method. "
|
||||
"Using populate_indicators from the strategy.")
|
||||
if not hasattr(hyperopt, 'populate_buy_trend'):
|
||||
logger.info("Hyperopt class does not provide populate_buy_trend() method. "
|
||||
"Using populate_buy_trend from the strategy.")
|
||||
if not hasattr(hyperopt, 'populate_sell_trend'):
|
||||
logger.info("Hyperopt class does not provide populate_sell_trend() method. "
|
||||
"Using populate_sell_trend from the strategy.")
|
||||
return hyperopt
|
||||
|
||||
|
||||
class HyperOptLossResolver(IResolver):
|
||||
"""
|
||||
This class contains all the logic to load custom hyperopt loss class
|
||||
|
@@ -58,10 +58,13 @@ class IResolver:
|
||||
# Generate spec based on absolute path
|
||||
# Pass object_name as first argument to have logging print a reasonable name.
|
||||
spec = importlib.util.spec_from_file_location(object_name or "", str(module_path))
|
||||
if not spec:
|
||||
return iter([None])
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
try:
|
||||
spec.loader.exec_module(module) # type: ignore # importlib does not use typehints
|
||||
except (ModuleNotFoundError, SyntaxError, ImportError) as err:
|
||||
except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
|
||||
# Catch errors in case a specific module is not installed
|
||||
logger.warning(f"Could not import {module_path} due to '{err}'")
|
||||
if enum_failed:
|
||||
@@ -91,6 +94,9 @@ class IResolver:
|
||||
if not str(entry).endswith('.py'):
|
||||
logger.debug('Ignoring %s', entry)
|
||||
continue
|
||||
if entry.is_symlink() and not entry.is_file():
|
||||
logger.debug('Ignoring broken symlink %s', entry)
|
||||
continue
|
||||
module_path = entry.resolve()
|
||||
|
||||
obj = next(cls._get_valid_object(module_path, object_name), None)
|
||||
@@ -129,7 +135,7 @@ class IResolver:
|
||||
extra_dir: Optional[str] = None) -> Any:
|
||||
"""
|
||||
Search and loads the specified object as configured in hte child class.
|
||||
:param objectname: name of the module to import
|
||||
:param object_name: name of the module to import
|
||||
:param config: configuration dictionary
|
||||
:param extra_dir: additional directory to search for the given pairlist
|
||||
:raises: OperationalException if the class is invalid or does not exist.
|
||||
@@ -157,7 +163,7 @@ class IResolver:
|
||||
:param directory: Path to search
|
||||
:param enum_failed: If True, will return None for modules which fail.
|
||||
Otherwise, failing modules are skipped.
|
||||
:return: List of dicts containing 'name', 'class' and 'location' entires
|
||||
:return: List of dicts containing 'name', 'class' and 'location' entries
|
||||
"""
|
||||
logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'")
|
||||
objects = []
|
||||
|
@@ -6,7 +6,6 @@ This module load custom strategies
|
||||
import logging
|
||||
import tempfile
|
||||
from base64 import urlsafe_b64decode
|
||||
from collections import OrderedDict
|
||||
from inspect import getfullargspec
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
@@ -46,57 +45,62 @@ class StrategyResolver(IResolver):
|
||||
strategy_name, config=config,
|
||||
extra_dir=config.get('strategy_path'))
|
||||
|
||||
# make sure ask_strategy dict is available
|
||||
if 'ask_strategy' not in config:
|
||||
config['ask_strategy'] = {}
|
||||
|
||||
if hasattr(strategy, 'ticker_interval') and not hasattr(strategy, 'timeframe'):
|
||||
# Assign ticker_interval to timeframe to keep compatibility
|
||||
if 'timeframe' not in config:
|
||||
logger.warning(
|
||||
"DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'."
|
||||
)
|
||||
)
|
||||
strategy.timeframe = strategy.ticker_interval
|
||||
|
||||
if strategy._ft_params_from_file:
|
||||
# Set parameters from Hyperopt results file
|
||||
params = strategy._ft_params_from_file
|
||||
strategy.minimal_roi = params.get('roi', strategy.minimal_roi)
|
||||
|
||||
strategy.stoploss = params.get('stoploss', {}).get('stoploss', strategy.stoploss)
|
||||
trailing = params.get('trailing', {})
|
||||
strategy.trailing_stop = trailing.get('trailing_stop', strategy.trailing_stop)
|
||||
strategy.trailing_stop_positive = trailing.get('trailing_stop_positive',
|
||||
strategy.trailing_stop_positive)
|
||||
strategy.trailing_stop_positive_offset = trailing.get(
|
||||
'trailing_stop_positive_offset', strategy.trailing_stop_positive_offset)
|
||||
strategy.trailing_only_offset_is_reached = trailing.get(
|
||||
'trailing_only_offset_is_reached', strategy.trailing_only_offset_is_reached)
|
||||
|
||||
# Set attributes
|
||||
# Check if we need to override configuration
|
||||
# (Attribute name, default, subkey)
|
||||
attributes = [("minimal_roi", {"0": 10.0}, None),
|
||||
("timeframe", None, None),
|
||||
("stoploss", None, None),
|
||||
("trailing_stop", None, None),
|
||||
("trailing_stop_positive", None, None),
|
||||
("trailing_stop_positive_offset", 0.0, None),
|
||||
("trailing_only_offset_is_reached", None, None),
|
||||
("use_custom_stoploss", None, None),
|
||||
("process_only_new_candles", None, None),
|
||||
("order_types", None, None),
|
||||
("order_time_in_force", None, None),
|
||||
("stake_currency", None, None),
|
||||
("stake_amount", None, None),
|
||||
("protections", None, None),
|
||||
("startup_candle_count", None, None),
|
||||
("unfilledtimeout", None, None),
|
||||
("use_sell_signal", True, 'ask_strategy'),
|
||||
("sell_profit_only", False, 'ask_strategy'),
|
||||
("ignore_roi_if_buy_signal", False, 'ask_strategy'),
|
||||
("sell_profit_offset", 0.0, 'ask_strategy'),
|
||||
("disable_dataframe_checks", False, None),
|
||||
("ignore_buying_expired_candle_after", 0, 'ask_strategy')
|
||||
attributes = [("minimal_roi", {"0": 10.0}),
|
||||
("timeframe", None),
|
||||
("stoploss", None),
|
||||
("trailing_stop", None),
|
||||
("trailing_stop_positive", None),
|
||||
("trailing_stop_positive_offset", 0.0),
|
||||
("trailing_only_offset_is_reached", None),
|
||||
("use_custom_stoploss", None),
|
||||
("process_only_new_candles", None),
|
||||
("order_types", None),
|
||||
("order_time_in_force", None),
|
||||
("stake_currency", None),
|
||||
("stake_amount", None),
|
||||
("protections", None),
|
||||
("startup_candle_count", None),
|
||||
("unfilledtimeout", None),
|
||||
("use_sell_signal", True),
|
||||
("sell_profit_only", False),
|
||||
("ignore_roi_if_buy_signal", False),
|
||||
("sell_profit_offset", 0.0),
|
||||
("disable_dataframe_checks", False),
|
||||
("ignore_buying_expired_candle_after", 0)
|
||||
]
|
||||
for attribute, default, subkey in attributes:
|
||||
if subkey:
|
||||
StrategyResolver._override_attribute_helper(strategy, config.get(subkey, {}),
|
||||
attribute, default)
|
||||
else:
|
||||
StrategyResolver._override_attribute_helper(strategy, config,
|
||||
attribute, default)
|
||||
for attribute, default in attributes:
|
||||
StrategyResolver._override_attribute_helper(strategy, config,
|
||||
attribute, default)
|
||||
|
||||
# Loop this list again to have output combined
|
||||
for attribute, _, subkey in attributes:
|
||||
if subkey and attribute in config[subkey]:
|
||||
logger.info("Strategy using %s: %s", attribute, config[subkey][attribute])
|
||||
elif attribute in config:
|
||||
for attribute, _ in attributes:
|
||||
if attribute in config:
|
||||
logger.info("Strategy using %s: %s", attribute, config[attribute])
|
||||
|
||||
StrategyResolver._normalize_attributes(strategy)
|
||||
@@ -114,7 +118,9 @@ class StrategyResolver(IResolver):
|
||||
- Strategy
|
||||
- default (if not None)
|
||||
"""
|
||||
if attribute in config:
|
||||
if (attribute in config
|
||||
and not isinstance(getattr(type(strategy), attribute, None), property)):
|
||||
# Ensure Properties are not overwritten
|
||||
setattr(strategy, attribute, config[attribute])
|
||||
logger.info("Override strategy '%s' with value in config file: %s.",
|
||||
attribute, config[attribute])
|
||||
@@ -139,7 +145,7 @@ class StrategyResolver(IResolver):
|
||||
|
||||
# Sort and apply type conversions
|
||||
if hasattr(strategy, 'minimal_roi'):
|
||||
strategy.minimal_roi = OrderedDict(sorted(
|
||||
strategy.minimal_roi = dict(sorted(
|
||||
{int(key): value for (key, value) in strategy.minimal_roi.items()}.items(),
|
||||
key=lambda t: t[0]))
|
||||
if hasattr(strategy, 'stoploss'):
|
||||
@@ -196,9 +202,9 @@ class StrategyResolver(IResolver):
|
||||
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
|
||||
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
|
||||
strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
|
||||
if any([x == 2 for x in [strategy._populate_fun_len,
|
||||
strategy._buy_fun_len,
|
||||
strategy._sell_fun_len]]):
|
||||
if any(x == 2 for x in [strategy._populate_fun_len,
|
||||
strategy._buy_fun_len,
|
||||
strategy._sell_fun_len]):
|
||||
strategy.INTERFACE_VERSION = 1
|
||||
|
||||
return strategy
|
||||
|
@@ -1,3 +1,3 @@
|
||||
# flake8: noqa: F401
|
||||
from .rpc import RPC, RPCException, RPCHandler, RPCMessageType
|
||||
from .rpc import RPC, RPCException, RPCHandler
|
||||
from .rpc_manager import RPCManager
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user