adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into pr/cryptomeisternox/5150

Browse Source
This commit is contained in:
Matthias
2021-10-30 10:26:05 +02:00
parent 85979c3176 c34b8a95d7
commit 6f1e719216
245 changed files with 10802 additions and 6918 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
freqtrade/__init__.py
View File

@@ -22,7 +22,7 @@ if __version__ == 'develop':
# subprocess.check_output(
# ['git', 'log', '--format="%h"', '-n 1'],
# stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
except Exception:
except Exception: # pragma: no cover
# git not available, ignore
try:
# Try Fallback to freqtrade_commit file (created by CI while building docker image)

10
freqtrade/commands/__init__.py
View File

@@ -8,15 +8,17 @@ Note: Be careful with file-scoped imports in these subfiles.
"""
from freqtrade.commands.arguments import Arguments
from freqtrade.commands.build_config_commands import start_new_config
from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
start_list_data)
from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
start_download_data, start_list_data)
from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
start_new_hyperopt, start_new_strategy)
start_new_strategy)
from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts,
start_list_markets, start_list_strategies,
start_list_timeframes, start_show_trades)
from freqtrade.commands.optimize_commands import start_backtest_filter, start_backtesting, start_edge, start_hyperopt
from freqtrade.commands.optimize_commands import (start_backtest_filter, start_backtesting,
start_edge, start_hyperopt)
from freqtrade.commands.pairlist_commands import start_test_pairlist
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
from freqtrade.commands.trade_commands import start_trading
from freqtrade.commands.webserver_commands import start_webserver

79
freqtrade/commands/arguments.py
View File

@@ -17,12 +17,15 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
ARGS_WEBSERVER: List[str] = []
ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
"max_open_trades", "stake_amount", "fee", "pairs"]
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
"enable_protections", "dry_run_wallet",
"strategy_list", "export", "exportfilename"]
"enable_protections", "dry_run_wallet", "timeframe_detail",
"strategy_list", "export", "exportfilename",
"backtest_breakdown"]
ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"position_stacking", "use_max_market_positions",
@@ -30,7 +33,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"epochs", "spaces", "print_all",
"print_colorized", "print_json", "hyperopt_jobs",
"hyperopt_random_state", "hyperopt_min_trades",
"hyperopt_loss"]
"hyperopt_loss", "disableparamexport",
"hyperopt_ignore_missing_space"]
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
@@ -56,16 +60,16 @@ ARGS_BUILD_CONFIG = ["config"]
ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"]
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
"download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
"dataformat_trades"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive",
"timerange", "download_trades", "exchange", "timeframes",
"erase", "dataformat_ohlcv", "dataformat_trades"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
"db_url", "trade_source", "export", "exportfilename",
@@ -74,7 +78,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "timeframe", "plot_auto_open"]
ARGS_INSTALL_UI = ["erase_ui_only"]
ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version']
ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
@@ -88,14 +92,15 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
"hyperoptexportfilename", "export_csv"]
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header"]
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
"disableparamexport", "backtest_breakdown"]
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies", "list-data",
"list-hyperopts", "hyperopt-list", "backtest-filter", "hyperopt-show",
"plot-dataframe", "plot-profit", "show-trades"]
"hyperopt-list", "hyperopt-show", "backtest-filter",
"plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
class Arguments:
@@ -171,14 +176,16 @@ class Arguments:
self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
self._build_args(optionlist=['version'], parser=self.parser)
from freqtrade.commands import (start_backtesting, start_backtest_filter, start_convert_data, start_create_userdir,
start_download_data, start_edge, start_hyperopt,
start_hyperopt_list, start_hyperopt_show, start_install_ui,
start_list_data, start_list_exchanges, start_list_hyperopts,
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config, start_new_hyperopt,
start_new_strategy, start_plot_dataframe, start_plot_profit,
start_show_trades, start_test_pairlist, start_trading)
from freqtrade.commands import (
start_backtesting, start_backtest_filter, start_convert_data, start_convert_trades,
start_create_userdir, start_download_data, start_edge,
start_hyperopt, start_hyperopt_list, start_hyperopt_show,
start_install_ui, start_list_data, start_list_exchanges,
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config, start_new_strategy,
start_plot_dataframe, start_plot_profit, start_show_trades,
start_test_pairlist, start_trading, start_webserver
)
subparsers = self.parser.add_subparsers(dest='command',
# Use custom message when no subhandler is added
@@ -205,12 +212,6 @@ class Arguments:
build_config_cmd.set_defaults(func=start_new_config)
self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
# add new-hyperopt subcommand
build_hyperopt_cmd = subparsers.add_parser('new-hyperopt',
help="Create new hyperopt")
build_hyperopt_cmd.set_defaults(func=start_new_hyperopt)
self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd)
# add new-strategy subcommand
build_strategy_cmd = subparsers.add_parser('new-strategy',
help="Create new strategy")
@@ -244,6 +245,15 @@ class Arguments:
convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
# Add trades-to-ohlcv subcommand
convert_trade_data_cmd = subparsers.add_parser(
'trades-to-ohlcv',
help='Convert trade data to OHLCV data.',
parents=[_common_parser],
)
convert_trade_data_cmd.set_defaults(func=start_convert_trades)
self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd)
# Add list-data subcommand
list_data_cmd = subparsers.add_parser(
'list-data',
@@ -308,15 +318,6 @@ class Arguments:
list_exchanges_cmd.set_defaults(func=start_list_exchanges)
self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd)
# Add list-hyperopts subcommand
list_hyperopts_cmd = subparsers.add_parser(
'list-hyperopts',
help='Print available hyperopt classes.',
parents=[_common_parser],
)
list_hyperopts_cmd.set_defaults(func=start_list_hyperopts)
self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd)
# Add list-markets subcommand
list_markets_cmd = subparsers.add_parser(
'list-markets',
@@ -395,3 +396,9 @@ class Arguments:
)
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
# Add webserver subcommand
webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
parents=[_common_parser])
webserver_cmd.set_defaults(func=start_webserver)
self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)

31
freqtrade/commands/build_config_commands.py
View File

@@ -61,21 +61,27 @@ def ask_user_config() -> Dict[str, Any]:
"type": "text",
"name": "stake_currency",
"message": "Please insert your stake currency:",
"default": 'BTC',
"default": 'USDT',
},
{
"type": "text",
"name": "stake_amount",
"message": "Please insert your stake amount:",
"default": "0.01",
"message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
"default": "100",
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
if val == UNLIMITED_STAKE_AMOUNT
else val
},
{
"type": "text",
"name": "max_open_trades",
"message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):",
"default": "3",
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val)
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val),
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
if val == UNLIMITED_STAKE_AMOUNT
else val
},
{
"type": "text",
@@ -99,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]:
"bittrex",
"kraken",
"ftx",
"kucoin",
"gateio",
Separator(),
"other",
],
@@ -122,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]:
"message": "Insert Exchange Secret",
"when": lambda x: not x['dry_run']
},
{
"type": "password",
"name": "exchange_key_password",
"message": "Insert Exchange API Key password",
"when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin'
},
{
"type": "confirm",
"name": "telegram",
@@ -149,7 +163,8 @@ def ask_user_config() -> Dict[str, Any]:
{
"type": "text",
"name": "api_server_listen_addr",
"message": "Insert Api server Listen Address (best left untouched default!)",
"message": ("Insert Api server Listen Address (0.0.0.0 for docker, "
"otherwise best left untouched)"),
"default": "127.0.0.1",
"when": lambda x: x['api_server']
},
@@ -183,7 +198,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
"""
Applies selections to the template and writes the result to config_path
:param config_path: Path object for new config file. Should not exist yet
:param selecions: Dict containing selections taken by the user.
:param selections: Dict containing selections taken by the user.
"""
from jinja2.exceptions import TemplateNotFound
try:
@@ -193,7 +208,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
selections['exchange'] = render_template(
templatefile=f"subtemplates/exchange_{exchange_template}.j2",
arguments=selections
)
)
except TemplateNotFound:
selections['exchange'] = render_template(
templatefile="subtemplates/exchange_generic.j2",
@@ -213,7 +228,7 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None:
def start_new_config(args: Dict[str, Any]) -> None:
"""
Create a new strategy from a template
Asking the user questions to fill out the templateaccordingly.
Asking the user questions to fill out the template accordingly.
"""
config_path = Path(args['config'][0])

51
freqtrade/commands/cli_options.py
View File

@@ -1,7 +1,7 @@
"""
Definition of cli arguments used in arguments.py
"""
from argparse import ArgumentTypeError
from argparse import SUPPRESS, ArgumentTypeError
from freqtrade import __version__, constants
from freqtrade.constants import HYPEROPT_LOSS_BUILTIN
@@ -135,6 +135,10 @@ AVAILABLE_CLI_OPTIONS = {
help='Override the value of the `stake_amount` configuration setting.',
),
# Backtesting
"timeframe_detail": Arg(
'--timeframe-detail',
help='Specify detail timeframe for backtesting (`1m`, `5m`, `30m`, `1h`, `1d`).',
),
"position_stacking": Arg(
'--eps', '--enable-position-stacking',
help='Allow buying the same pair multiple times (position stacking).',
@@ -162,13 +166,14 @@ AVAILABLE_CLI_OPTIONS = {
'Please note that ticker-interval needs to be set either in config '
'or via command line. When using this together with `--export trades`, '
'the strategy-name is injected into the filename '
'(so `backtest-data.json` becomes `backtest-data-DefaultStrategy.json`',
'(so `backtest-data.json` becomes `backtest-data-SampleStrategy.json`',
nargs='+',
),
"export": Arg(
'--export',
help='Export backtest results, argument are: trades. '
'Example: `--export=trades`',
help='Export backtest results (default: trades).',
choices=constants.EXPORT_OPTIONS,
),
"exportfilename": Arg(
'--export-filename',
@@ -177,12 +182,23 @@ AVAILABLE_CLI_OPTIONS = {
'Example: `--export-filename=user_data/backtest_results/backtest_today.json`',
metavar='PATH',
),
"disableparamexport": Arg(
'--disable-param-export',
help="Disable automatic hyperopt parameter export.",
action='store_true',
),
"fee": Arg(
'--fee',
help='Specify fee ratio. Will be applied twice (on trade entry and exit).',
type=float,
metavar='FLOAT',
),
"backtest_breakdown": Arg(
'--breakdown',
help='Show backtesting breakdown per [day, week, month].',
nargs='+',
choices=constants.BACKTEST_BREAKDOWNS
),
# Edge
"stoploss_range": Arg(
'--stoplosses',
@@ -193,13 +209,13 @@ AVAILABLE_CLI_OPTIONS = {
# Hyperopt
"hyperopt": Arg(
'--hyperopt',
help='Specify hyperopt class name which will be used by the bot.',
help=SUPPRESS,
metavar='NAME',
required=False,
),
"hyperopt_path": Arg(
'--hyperopt-path',
help='Specify additional lookup path for Hyperopt and Hyperopt Loss functions.',
help='Specify additional lookup path for Hyperopt Loss functions.',
metavar='PATH',
),
"backtest_path": Arg(
@@ -217,7 +233,7 @@ AVAILABLE_CLI_OPTIONS = {
"spaces": Arg(
'--spaces',
help='Specify which parameters to hyperopt. Space-separated list.',
choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'default'],
choices=['all', 'buy', 'sell', 'roi', 'stoploss', 'trailing', 'protection', 'default'],
nargs='+',
default='default',
),
@@ -350,6 +366,11 @@ AVAILABLE_CLI_OPTIONS = {
type=check_int_positive,
metavar='INT',
),
"include_inactive": Arg(
'--include-inactive-pairs',
help='Also download data from inactive pairs.',
action='store_true',
),
"new_pairs_days": Arg(
'--new-pairs-days',
help='Download data of new pairs for given number of days. Default: `%(default)s`.',
@@ -376,12 +397,12 @@ AVAILABLE_CLI_OPTIONS = {
),
"dataformat_ohlcv": Arg(
'--data-format-ohlcv',
help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
choices=constants.AVAILABLE_DATAHANDLERS,
),
"dataformat_trades": Arg(
'--data-format-trades',
help='Storage format for downloaded trades data. (default: `%(default)s`).',
help='Storage format for downloaded trades data. (default: `jsongz`).',
choices=constants.AVAILABLE_DATAHANDLERS,
),
"exchange": Arg(
@@ -409,6 +430,12 @@ AVAILABLE_CLI_OPTIONS = {
action='store_true',
default=False,
),
"ui_version": Arg(
'--ui-version',
help=('Specify a specific version of FreqUI to install. '
'Not specifying this installs the latest version.'),
type=str,
),
# Templating options
"template": Arg(
'--template',
@@ -547,4 +574,10 @@ AVAILABLE_CLI_OPTIONS = {
help='Do not print epoch details header.',
action='store_true',
),
"hyperopt_ignore_missing_space": Arg(
"--ignore-missing-spaces", "--ignore-unparameterized-spaces",
help=("Suppress errors for any requested Hyperopt spaces "
"that do not contain any parameters."),
action="store_true",
),
}

45
freqtrade/commands/data_commands.py
View File

@@ -11,6 +11,7 @@ from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_oh
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.exchange.exchange import market_is_active
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver
@@ -47,10 +48,13 @@ def start_download_data(args: Dict[str, Any]) -> None:
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
markets = [p for p, m in exchange.markets.items() if market_is_active(m)
or config.get('include_inactive')]
expanded_pairs = expand_pairlist(config['pairs'], markets)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(expanded_pairs)
logger.info(f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
@@ -88,6 +92,41 @@ def start_download_data(args: Dict[str, Any]) -> None:
f"on exchange {exchange.name}.")
def start_convert_trades(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
timerange = TimeRange()
# Remove stake-currency to skip checks which are not relevant for datadownload
config['stake_currency'] = ''
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to Convert pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another

78
freqtrade/commands/deploy_commands.py
View File

@@ -7,7 +7,7 @@ import requests
from freqtrade.configuration import setup_utils_configuration
from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.constants import USERPATH_STRATEGIES
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.misc import render_template, render_template_with_fallback
@@ -38,15 +38,15 @@ def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: st
indicators = render_template_with_fallback(
templatefile=f"subtemplates/indicators_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/indicators_{fallback}.j2",
)
)
buy_trend = render_template_with_fallback(
templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2",
)
)
sell_trend = render_template_with_fallback(
templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2",
)
)
plot_config = render_template_with_fallback(
templatefile=f"subtemplates/plot_config_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2",
@@ -74,8 +74,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if "strategy" in args and args["strategy"]:
if args["strategy"] == "DefaultStrategy":
raise OperationalException("DefaultStrategy is not allowed as name.")
new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py')
@@ -89,58 +87,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None:
raise OperationalException("`new-strategy` requires --strategy to be set.")
def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None:
"""
Deploys a new hyperopt template to hyperopt_path
"""
fallback = 'full'
buy_guards = render_template_with_fallback(
templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2",
)
sell_guards = render_template_with_fallback(
templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2",
)
buy_space = render_template_with_fallback(
templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2",
)
sell_space = render_template_with_fallback(
templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",
templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2",
)
strategy_text = render_template(templatefile='base_hyperopt.py.j2',
arguments={"hyperopt": hyperopt_name,
"buy_guards": buy_guards,
"sell_guards": sell_guards,
"buy_space": buy_space,
"sell_space": sell_space,
})
logger.info(f"Writing hyperopt to `{hyperopt_path}`.")
hyperopt_path.write_text(strategy_text)
def start_new_hyperopt(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if 'hyperopt' in args and args['hyperopt']:
if args['hyperopt'] == 'DefaultHyperopt':
raise OperationalException("DefaultHyperopt is not allowed as name.")
new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py')
if new_path.exists():
raise OperationalException(f"`{new_path}` already exists. "
"Please choose another Hyperopt Name.")
deploy_new_hyperopt(args['hyperopt'], new_path, args['template'])
else:
raise OperationalException("`new-hyperopt` requires --hyperopt to be set.")
def clean_ui_subdir(directory: Path):
if directory.is_dir():
logger.info("Removing UI directory content.")
@@ -182,7 +128,7 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str):
f.write(version)
def get_ui_download_url() -> Tuple[str, str]:
def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]:
base_url = 'https://api.github.com/repos/freqtrade/frequi/'
# Get base UI Repo path
@@ -190,8 +136,16 @@ def get_ui_download_url() -> Tuple[str, str]:
resp.raise_for_status()
r = resp.json()
latest_version = r[0]['name']
assets = r[0].get('assets', [])
if version:
tmp = [x for x in r if x['name'] == version]
if tmp:
latest_version = tmp[0]['name']
assets = tmp[0].get('assets', [])
else:
raise ValueError("UI-Version not found.")
else:
latest_version = r[0]['name']
assets = r[0].get('assets', [])
dl_url = ''
if assets and len(assets) > 0:
dl_url = assets[0]['browser_download_url']
@@ -210,7 +164,7 @@ def start_install_ui(args: Dict[str, Any]) -> None:
dest_folder = Path(__file__).parents[1] / 'rpc/api_server/ui/installed/'
# First make sure the assets are removed.
dl_url, latest_version = get_ui_download_url()
dl_url, latest_version = get_ui_download_url(args.get('ui_version'))
curr_version = read_ui_version(dest_folder)
if curr_version == latest_version and not args.get('erase_ui_only'):

187
freqtrade/commands/hyperopt_commands.py
View File

@@ -1,6 +1,6 @@
import logging
from operator import itemgetter
from typing import Any, Dict, List
from typing import Any, Dict
from colorama import init as colorama_init
@@ -28,30 +28,12 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
no_details = config.get('hyperopt_list_no_details', False)
no_header = False
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
}
results_file = get_latest_hyperopt_file(
config['user_data_dir'] / 'hyperopt_results',
config.get('hyperoptexportfilename'))
# Previous evaluations
epochs = HyperoptTools.load_previous_results(results_file)
total_epochs = len(epochs)
epochs = hyperopt_filter_epochs(epochs, filteroptions)
epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
if print_colorized:
colorama_init(autoreset=True)
@@ -59,7 +41,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
if not export_csv:
try:
print(HyperoptTools.get_result_table(config, epochs, total_epochs,
not filteroptions['only_best'],
not config.get('hyperopt_list_best', False),
print_colorized, 0))
except KeyboardInterrupt:
print('User interrupted..')
@@ -71,7 +53,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None:
if epochs and export_csv:
HyperoptTools.export_csv_file(
config, epochs, total_epochs, not filteroptions['only_best'], export_csv
config, epochs, export_csv
)
@@ -91,26 +73,9 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
n = config.get('hyperopt_show_index', -1)
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
'filter_max_objective': config.get('hyperopt_list_max_objective', None)
}
# Previous evaluations
epochs = HyperoptTools.load_previous_results(results_file)
total_epochs = len(epochs)
epochs, total_epochs = HyperoptTools.load_filtered_results(results_file, config)
epochs = hyperopt_filter_epochs(epochs, filteroptions)
filtered_epochs = len(epochs)
if n > filtered_epochs:
@@ -129,143 +94,11 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
metrics = val['results_metrics']
if 'strategy_name' in metrics:
show_backtest_result(metrics['strategy_name'], metrics,
metrics['stake_currency'])
strategy_name = metrics['strategy_name']
show_backtest_result(strategy_name, metrics,
metrics['stake_currency'], config.get('backtest_breakdown', []))
HyperoptTools.try_export_params(config, strategy_name, val)
HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
header_str="Epoch details")
def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
"""
Filter our items from the list of hyperopt results
TODO: after 2021.5 remove all "legacy" mode queries.
"""
if filteroptions['only_best']:
epochs = [x for x in epochs if x['is_best']]
if filteroptions['only_profitable']:
epochs = [x for x in epochs if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total', 0)) > 0]
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
logger.info(f"{len(epochs)} " +
("best " if filteroptions['only_best'] else "") +
("profitable " if filteroptions['only_profitable'] else "") +
"epochs found.")
return epochs
def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
"""
Filter epochs with trade-counts > trades
"""
return [
x for x in epochs
if x['results_metrics'].get(
'trade_count', x['results_metrics'].get('total_trades', 0)
) > trade_count
]
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_trades'] > 0:
epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
if filteroptions['filter_max_trades'] > 0:
epochs = [
x for x in epochs
if x['results_metrics'].get(
'trade_count', x['results_metrics'].get('total_trades')
) < filteroptions['filter_max_trades']
]
return epochs
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
def get_duration_value(x):
# Duration in minutes ...
if 'duration' in x['results_metrics']:
return x['results_metrics']['duration']
else:
# New mode
if 'holding_avg_s' in x['results_metrics']:
avg = x['results_metrics']['holding_avg_s']
return avg // 60
raise OperationalException(
"Holding-average not available. Please omit the filter on average time, "
"or rerun hyperopt with this version")
if filteroptions['filter_min_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) > filteroptions['filter_min_avg_time']
]
if filteroptions['filter_max_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) < filteroptions['filter_max_avg_time']
]
return epochs
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
) > filteroptions['filter_min_avg_profit']
]
if filteroptions['filter_max_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
) < filteroptions['filter_max_avg_profit']
]
if filteroptions['filter_min_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total_abs', 0)
) > filteroptions['filter_min_total_profit']
]
if filteroptions['filter_max_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total_abs', 0)
) < filteroptions['filter_max_total_profit']
]
return epochs
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
if filteroptions['filter_max_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
return epochs

25
freqtrade/commands/list_commands.py
View File

@@ -10,11 +10,11 @@ from colorama import init as colorama_init
from tabulate import tabulate
from freqtrade.configuration import setup_utils_configuration
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.constants import USERPATH_STRATEGIES
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import market_is_active, validate_exchanges
from freqtrade.misc import plural
from freqtrade.misc import parse_db_uri_for_logging, plural
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -92,25 +92,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None:
_print_objs_tabular(strategy_objs, config.get('print_colorized', False))
def start_list_hyperopts(args: Dict[str, Any]) -> None:
"""
Print files with HyperOpt custom classes available in the directory
"""
from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS))
hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column'])
# Sort alphabetically
hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name'])
if args['print_one_column']:
print('\n'.join([s['name'] for s in hyperopt_objs]))
else:
_print_objs_tabular(hyperopt_objs, config.get('print_colorized', False))
def start_list_timeframes(args: Dict[str, Any]) -> None:
"""
Print timeframes available on Exchange
@@ -225,7 +206,7 @@ def start_show_trades(args: Dict[str, Any]) -> None:
if 'db_url' not in config:
raise OperationalException("--db-url is required for this command.")
logger.info(f'Using DB: "{config["db_url"]}"')
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
init_db(config['db_url'], clean_open_orders=False)
tfilter = []

1
freqtrade/commands/optimize_commands.py
View File

@@ -19,6 +19,7 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
"""
Prepare the configuration for the Hyperopt module
:param args: Cli args from Arguments()
:param method: Bot running mode
:return: Configuration
"""
config = setup_utils_configuration(args, method)

15
freqtrade/commands/webserver_commands.py Normal file
View File

@@ -0,0 +1,15 @@
from typing import Any, Dict
from freqtrade.enums import RunMode
def start_webserver(args: Dict[str, Any]) -> None:
"""
Main entry point for webserver mode
"""
from freqtrade.configuration import Configuration
from freqtrade.rpc.api_server import ApiServer
# Initialize configuration
config = Configuration(args, RunMode.WEBSERVER).get_config()
ApiServer(config, standalone=True)

19
freqtrade/configuration/PeriodicCache.py Normal file
View File

@@ -0,0 +1,19 @@
from datetime import datetime, timezone
from cachetools.ttl import TTLCache
class PeriodicCache(TTLCache):
"""
Special cache that expires at "straight" times
A timer with ttl of 3600 (1h) will expire at every full hour (:00).
"""
def __init__(self, maxsize, ttl, getsizeof=None):
def local_timer():
ts = datetime.now(timezone.utc).timestamp()
offset = (ts % ttl)
return ts - offset
# Init with smlight offset
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)

3
freqtrade/configuration/__init__.py
View File

@@ -1,7 +1,8 @@
# flake8: noqa: F401
from freqtrade.configuration.check_exchange import check_exchange, remove_credentials
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.config_setup import setup_utils_configuration
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.configuration.configuration import Configuration
from freqtrade.configuration.PeriodicCache import PeriodicCache
from freqtrade.configuration.timerange import TimeRange

21
freqtrade/configuration/check_exchange.py
View File

@@ -10,19 +10,6 @@ from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
logger = logging.getLogger(__name__)
def remove_credentials(config: Dict[str, Any]) -> None:
"""
Removes exchange keys from the configuration and specifies dry-run
Used for backtesting / hyperopt / edge and utils.
Modifies the input dict!
"""
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
config['exchange']['password'] = ''
config['exchange']['uid'] = ''
config['dry_run'] = True
def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
"""
Check if the exchange name in the config file is supported by Freqtrade
@@ -51,10 +38,10 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
if not is_exchange_known_ccxt(exchange):
raise OperationalException(
f'Exchange "{exchange}" is not known to the ccxt library '
f'and therefore not available for the bot.\n'
f'The following exchanges are available for Freqtrade: '
f'{", ".join(available_exchanges())}'
f'Exchange "{exchange}" is not known to the ccxt library '
f'and therefore not available for the bot.\n'
f'The following exchanges are available for Freqtrade: '
f'{", ".join(available_exchanges())}'
)
valid, reason = validate_exchange(exchange)

6
freqtrade/configuration/config_setup.py
View File

@@ -3,7 +3,6 @@ from typing import Any, Dict
from freqtrade.enums import RunMode
from .check_exchange import remove_credentials
from .config_validation import validate_config_consistency
from .configuration import Configuration
@@ -15,13 +14,14 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str
"""
Prepare the configuration for utils subcommands
:param args: Cli args from Arguments()
:param method: Bot running mode
:return: Configuration
"""
configuration = Configuration(args, method)
config = configuration.get_config()
# Ensure we do not use Exchange credentials
remove_credentials(config)
# Ensure these modes are using Dry-run
config['dry_run'] = True
validate_config_consistency(config)
return config

27
freqtrade/configuration/config_validation.py
View File

@@ -79,6 +79,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
_validate_whitelist(conf)
_validate_protections(conf)
_validate_unlimited_amount(conf)
_validate_ask_orderbook(conf)
# validate configuration before returning
logger.info('Validating configuration ...')
@@ -114,7 +115,7 @@ def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
if conf.get('stoploss') == 0.0:
raise OperationalException(
'The config stoploss needs to be different from 0 to avoid problems with sell orders.'
)
)
# Skip if trailing stoploss is not activated
if not conf.get('trailing_stop', False):
return
@@ -149,7 +150,7 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
if not conf.get('edge', {}).get('enabled'):
return
if not conf.get('ask_strategy', {}).get('use_sell_signal', True):
if not conf.get('use_sell_signal', True):
raise OperationalException(
"Edge requires `use_sell_signal` to be True, otherwise no sells will happen."
)
@@ -179,10 +180,30 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
raise OperationalException(
"Protections must specify either `stop_duration` or `stop_duration_candles`.\n"
f"Please fix the protection {prot.get('method')}"
)
)
if ('lookback_period' in prot and 'lookback_period_candles' in prot):
raise OperationalException(
"Protections must specify either `lookback_period` or `lookback_period_candles`.\n"
f"Please fix the protection {prot.get('method')}"
)
def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
ask_strategy = conf.get('ask_strategy', {})
ob_min = ask_strategy.get('order_book_min')
ob_max = ask_strategy.get('order_book_max')
if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
if ob_min != ob_max:
raise OperationalException(
"Using order_book_max != order_book_min in ask_strategy is no longer supported."
"Please pick one value and use `order_book_top` in the future."
)
else:
# Move value to order_book_top
ask_strategy['order_book_top'] = ob_min
logger.warning(
"DEPRECATED: "
"Please use `order_book_top` instead of `order_book_min` and `order_book_max` "
"for your `ask_strategy` configuration."
)

27
freqtrade/configuration/configuration.py
View File

@@ -11,11 +11,12 @@ from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
from freqtrade.configuration.load_config import load_config_file, load_file
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts
from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging
logger = logging.getLogger(__name__)
@@ -72,6 +73,11 @@ class Configuration:
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:
config['internals'] = {}
@@ -144,7 +150,7 @@ class Configuration:
config['db_url'] = constants.DEFAULT_DB_PROD_URL
logger.info('Dry run is disabled')
logger.info(f'Using DB: "{config["db_url"]}"')
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
def _process_common_options(self, config: Dict[str, Any]) -> None:
@@ -236,6 +242,9 @@ class Configuration:
except ValueError:
pass
self._args_to_config(config, argname='timeframe_detail',
logstring='Parameter --timeframe-detail detected, '
'using {} for intra-candle backtesting ...')
self._args_to_config(config, argname='stake_amount',
logstring='Parameter --stake-amount detected, '
'overriding stake_amount to: {} ...')
@@ -260,6 +269,12 @@ class Configuration:
self._args_to_config(config, argname='export',
logstring='Parameter --export detected: {} ...')
self._args_to_config(config, argname='backtest_breakdown',
logstring='Parameter --breakdown detected ...')
self._args_to_config(config, argname='disableparamexport',
logstring='Parameter --disableparamexport detected: {} ...')
# Edge section:
if 'stoploss_range' in self.args and self.args["stoploss_range"]:
txt_range = eval(self.args["stoploss_range"])
@@ -358,6 +373,9 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_show_no_header',
logstring='Parameter --no-header detected: {}')
self._args_to_config(config, argname="hyperopt_ignore_missing_space",
logstring="Paramter --ignore-missing-space detected: {}")
def _process_plot_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='pairs',
@@ -393,6 +411,9 @@ class Configuration:
self._args_to_config(config, argname='days',
logstring='Detected --days: {}')
self._args_to_config(config, argname='include_inactive',
logstring='Detected --include-inactive-pairs: {}')
self._args_to_config(config, argname='download_trades',
logstring='Detected --dl-trades: {}')
@@ -460,7 +481,7 @@ class Configuration:
pairs_file = Path(self.args["pairs_file"])
logger.info(f'Reading pairs file "{pairs_file}".')
# Download pairs from the pairs file if no config is specified
# or if pairs file is specified explicitely
# or if pairs file is specified explicitly
if not pairs_file.exists():
raise OperationalException(f'No pairs file found with path "{pairs_file}".')
config['pairs'] = load_file(pairs_file)

68
freqtrade/configuration/deprecated_settings.py
View File

@@ -3,7 +3,7 @@ Functions to handle deprecated settings
"""
import logging
from typing import Any, Dict
from typing import Any, Dict, Optional
from freqtrade.exceptions import OperationalException
@@ -12,23 +12,24 @@ logger = logging.getLogger(__name__)
def check_conflicting_settings(config: Dict[str, Any],
section1: str, name1: str,
section2: str, name2: str) -> None:
section1_config = config.get(section1, {})
section2_config = config.get(section2, {})
if name1 in section1_config and name2 in section2_config:
section_old: str, name_old: str,
section_new: Optional[str], name_new: str) -> None:
section_new_config = config.get(section_new, {}) if section_new else config
section_old_config = config.get(section_old, {})
if name_new in section_new_config and name_old in section_old_config:
new_name = f"{section_new}.{name_new}" if section_new else f"{name_new}"
raise OperationalException(
f"Conflicting settings `{section1}.{name1}` and `{section2}.{name2}` "
f"Conflicting settings `{new_name}` and `{section_old}.{name_old}` "
"(DEPRECATED) detected in the configuration file. "
"This deprecated setting will be removed in the next versions of Freqtrade. "
f"Please delete it from your configuration and use the `{section1}.{name1}` "
f"Please delete it from your configuration and use the `{new_name}` "
"setting instead."
)
def process_removed_setting(config: Dict[str, Any],
section1: str, name1: str,
section2: str, name2: str) -> None:
section2: Optional[str], name2: str) -> None:
"""
:param section1: Removed section
:param name1: Removed setting name
@@ -37,27 +38,32 @@ def process_removed_setting(config: Dict[str, Any],
"""
section1_config = config.get(section1, {})
if name1 in section1_config:
section_2 = f"{section2}.{name2}" if section2 else f"{name2}"
raise OperationalException(
f"Setting `{section1}.{name1}` has been moved to `{section2}.{name2}. "
f"Please delete it from your configuration and use the `{section2}.{name2}` "
f"Setting `{section1}.{name1}` has been moved to `{section_2}. "
f"Please delete it from your configuration and use the `{section_2}` "
"setting instead."
)
def process_deprecated_setting(config: Dict[str, Any],
section1: str, name1: str,
section2: str, name2: str) -> None:
section2_config = config.get(section2, {})
section_old: str, name_old: str,
section_new: Optional[str], name_new: str
) -> None:
check_conflicting_settings(config, section_old, name_old, section_new, name_new)
section_old_config = config.get(section_old, {})
if name2 in section2_config:
if name_old in section_old_config:
section_2 = f"{section_new}.{name_new}" if section_new else f"{name_new}"
logger.warning(
"DEPRECATED: "
f"The `{section2}.{name2}` setting is deprecated and "
f"The `{section_old}.{name_old}` setting is deprecated and "
"will be removed in the next versions of Freqtrade. "
f"Please use the `{section1}.{name1}` setting in your configuration instead."
f"Please use the `{section_2}` setting in your configuration instead."
)
section1_config = config.get(section1, {})
section1_config[name1] = section2_config[name2]
section_new_config = config.get(section_new, {}) if section_new else config
section_new_config[name_new] = section_old_config[name_old]
def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
@@ -65,15 +71,24 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
# Kept for future deprecated / moved settings
# check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
# 'experimental', 'use_sell_signal')
# process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
# 'experimental', 'use_sell_signal')
process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
None, 'use_sell_signal')
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only',
None, 'sell_profit_only')
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_offset',
None, 'sell_profit_offset')
process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_buy_signal')
process_deprecated_setting(config, 'ask_strategy', 'ignore_buying_expired_candle_after',
None, 'ignore_buying_expired_candle_after')
# Legacy way - having them in experimental ...
process_removed_setting(config, 'experimental', 'use_sell_signal',
'ask_strategy', 'use_sell_signal')
None, 'use_sell_signal')
process_removed_setting(config, 'experimental', 'sell_profit_only',
'ask_strategy', 'sell_profit_only')
None, 'sell_profit_only')
process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
'ask_strategy', 'ignore_roi_if_buy_signal')
None, 'ignore_roi_if_buy_signal')
if (config.get('edge', {}).get('enabled', False)
and 'capital_available_percentage' in config.get('edge', {})):
@@ -93,5 +108,8 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
raise OperationalException(
"Both 'timeframe' and 'ticker_interval' detected."
"Please remove 'ticker_interval' from your configuration to continue operating."
)
)
config['timeframe'] = config['ticker_interval']
if 'protections' in config:
logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")

54
freqtrade/configuration/environment_vars.py Normal file
View File

@@ -0,0 +1,54 @@
import logging
import os
from typing import Any, Dict
from freqtrade.constants import ENV_VAR_PREFIX
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
def get_var_typed(val):
try:
return int(val)
except ValueError:
try:
return float(val)
except ValueError:
if val.lower() in ('t', 'true'):
return True
elif val.lower() in ('f', 'false'):
return False
# keep as string
return val
def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
"""
Environment variables must be prefixed with FREQTRADE.
FREQTRADE__{section}__{key}
:param env_dict: Dictionary to validate - usually os.environ
:param prefix: Prefix to consider (usually FREQTRADE__)
:return: Nested dict based on available and relevant variables.
"""
relevant_vars: Dict[str, Any] = {}
for env_var, val in sorted(env_dict.items()):
if env_var.startswith(prefix):
logger.info(f"Loading variable '{env_var}'")
key = env_var.replace(prefix, '')
for k in reversed(key.split('__')):
val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
relevant_vars = deep_merge_dicts(val, relevant_vars)
return relevant_vars
def enironment_vars_to_dict() -> Dict[str, Any]:
"""
Read environment variables and return a nested dict for relevant variables
Relevant variables must follow the FREQTRADE__{section}__{key} pattern
:return: Nested dict based on available and relevant variables.
"""
return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)

76
freqtrade/constants.py
View File

@@ -12,6 +12,7 @@ PROCESS_THROTTLE_SECS = 5 # sec
HYPEROPT_EPOCH = 100 # epochs
RETRY_TIMEOUT = 30 # sec
TIMEOUT_UNITS = ['minutes', 'seconds']
EXPORT_OPTIONS = ['none', 'trades']
DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
UNLIMITED_STAKE_AMOUNT = 'unlimited'
@@ -23,13 +24,16 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market']
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily',
'CalmarHyperOptLoss',
'MaxDrawDownHyperOptLoss']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
'SpreadFilter', 'VolatilityFilter']
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
DRY_RUN_WALLET = 1000
DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons
@@ -39,13 +43,16 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
LAST_BT_RESULT_FN = '.last_result.json'
FTHYPT_FILEVERSION = 'fthypt_fileversion'
USERPATH_HYPEROPTS = 'hyperopts'
USERPATH_STRATEGIES = 'strategies'
USERPATH_NOTEBOOKS = 'notebooks'
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
ENV_VAR_PREFIX = 'FREQTRADE__'
NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
# Define decimals per coin for outputs
# Only used for outputs.
@@ -60,13 +67,10 @@ DUST_PER_COIN = {
'ETH': 0.01
}
# Soure files with destination directories within user-directory
# Source files with destination directories within user-directory
USER_DATA_FILES = {
'sample_strategy.py': USERPATH_STRATEGIES,
'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS,
'sample_hyperopt_loss.py': USERPATH_HYPEROPTS,
'sample_hyperopt.py': USERPATH_HYPEROPTS,
'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS,
}
@@ -107,10 +111,14 @@ CONF_SCHEMA = {
},
'tradable_balance_ratio': {
'type': 'number',
'minimum': 0.1,
'minimum': 0.0,
'maximum': 1,
'default': 0.99
},
'available_capital': {
'type': 'number',
'minimum': 0,
},
'amend_last_stake_amount': {'type': 'boolean', 'default': False},
'last_stake_amount_min_ratio': {
'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5
@@ -133,6 +141,15 @@ CONF_SCHEMA = {
'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1},
'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1},
'trailing_only_offset_is_reached': {'type': 'boolean'},
'use_sell_signal': {'type': 'boolean'},
'sell_profit_only': {'type': 'boolean'},
'sell_profit_offset': {'type': 'number'},
'ignore_roi_if_buy_signal': {'type': 'boolean'},
'ignore_buying_expired_candle_after': {'type': 'number'},
'backtest_breakdown': {
'type': 'array',
'items': {'type': 'string', 'enum': BACKTEST_BREAKDOWNS}
},
'bot_name': {'type': 'string'},
'unfilledtimeout': {
'type': 'object',
@@ -153,7 +170,7 @@ CONF_SCHEMA = {
},
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'},
'use_order_book': {'type': 'boolean'},
'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1},
'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
'check_depth_of_market': {
'type': 'object',
'properties': {
@@ -162,7 +179,7 @@ CONF_SCHEMA = {
}
},
},
'required': ['ask_last_balance']
'required': ['price_side']
},
'ask_strategy': {
'type': 'object',
@@ -175,13 +192,12 @@ CONF_SCHEMA = {
'exclusiveMaximum': False,
},
'use_order_book': {'type': 'boolean'},
'order_book_min': {'type': 'integer', 'minimum': 1},
'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50},
'use_sell_signal': {'type': 'boolean'},
'sell_profit_only': {'type': 'boolean'},
'sell_profit_offset': {'type': 'number'},
'ignore_roi_if_buy_signal': {'type': 'boolean'}
}
'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
},
'required': ['price_side']
},
'custom_price_max_distance_ratio': {
'type': 'number', 'minimum': 0.0
},
'order_types': {
'type': 'object',
@@ -272,9 +288,19 @@ CONF_SCHEMA = {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
},
},
'protection_trigger': {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
},
'protection_trigger_global': {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
},
}
}
},
'reload': {'type': 'boolean'},
},
'required': ['enabled', 'token', 'chat_id'],
},
@@ -308,6 +334,8 @@ CONF_SCHEMA = {
'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password']
},
'db_url': {'type': 'string'},
'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
'disableparamexport': {'type': 'boolean'},
'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
'forcebuy_enable': {'type': 'boolean'},
'disable_dataframe_checks': {'type': 'boolean'},
@@ -322,13 +350,13 @@ CONF_SCHEMA = {
},
'dataformat_ohlcv': {
'type': 'string',
'enum': AVAILABLE_DATAHANDLERS,
'default': 'json'
'enum': AVAILABLE_DATAHANDLERS,
'default': 'json'
},
'dataformat_trades': {
'type': 'string',
'enum': AVAILABLE_DATAHANDLERS,
'default': 'jsongz'
'enum': AVAILABLE_DATAHANDLERS,
'default': 'jsongz'
}
},
'definitions': {

4
freqtrade/data/btanalysis.py
View File

@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
"trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
# Mid-term format, crated by BacktestResult Named Tuple
# Mid-term format, created by BacktestResult Named Tuple
BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
'fee_close', 'amount', 'profit_abs', 'profit_ratio']
@@ -30,7 +30,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'fee_open', 'fee_close', 'trade_duration',
'profit_ratio', 'profit_abs', 'sell_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', ]
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag']
def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:

6
freqtrade/data/converter.py
View File

@@ -49,7 +49,7 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *,
fill_missing: bool = True,
drop_incomplete: bool = True) -> DataFrame:
"""
Clense a OHLCV dataframe by
Cleanse a OHLCV dataframe by
* Grouping it by date (removes duplicate tics)
* dropping last candles if requested
* Filling up missing data (if requested)
@@ -242,7 +242,7 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase souce data (does not apply if source and target format are identical)
:param erase: Erase source data (does not apply if source and target format are identical)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)
@@ -267,7 +267,7 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase souce data (does not apply if source and target format are identical)
:param erase: Erase source data (does not apply if source and target format are identical)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)

27
freqtrade/data/dataprovider.py
View File

@@ -10,11 +10,12 @@ from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
from freqtrade.data.history import load_pair_history
from freqtrade.enums import RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange
from freqtrade.exchange import Exchange, timeframe_to_seconds
logger = logging.getLogger(__name__)
@@ -31,6 +32,7 @@ class DataProvider:
self._pairlists = pairlists
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None
self.__cached_pairs_backtesting: Dict[PairWithTimeframe, DataFrame] = {}
def _set_dataframe_max_index(self, limit_index: int):
"""
@@ -62,11 +64,22 @@ class DataProvider:
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
"""
return load_pair_history(pair=pair,
timeframe=timeframe or self._config['timeframe'],
datadir=self._config['datadir'],
data_format=self._config.get('dataformat_ohlcv', 'json')
)
saved_pair = (pair, str(timeframe))
if saved_pair not in self.__cached_pairs_backtesting:
timerange = TimeRange.parse_timerange(None if self._config.get(
'timerange') is None else str(self._config.get('timerange')))
# Move informative start time respecting startup_candle_count
timerange.subtract_start(
timeframe_to_seconds(str(timeframe)) * self._config.get('startup_candle_count', 0)
)
self.__cached_pairs_backtesting[saved_pair] = load_pair_history(
pair=pair,
timeframe=timeframe or self._config['timeframe'],
datadir=self._config['datadir'],
timerange=timerange,
data_format=self._config.get('dataformat_ohlcv', 'json')
)
return self.__cached_pairs_backtesting[saved_pair].copy()
def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
"""
@@ -136,6 +149,8 @@ class DataProvider:
Clear pair dataframe cache.
"""
self.__cached_pairs = {}
self.__cached_pairs_backtesting = {}
self.__slice_index = 0
# Exchange functions

4
freqtrade/data/history/hdf5datahandler.py
View File

@@ -52,8 +52,8 @@ class HDF5DataHandler(IDataHandler):
"""
Store data in hdf5 file.
:param pair: Pair - used to generate filename
:timeframe: Timeframe - used to generate filename
:data: Dataframe containing OHLCV data
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
:return: None
"""
key = self._pair_ohlcv_key(pair, timeframe)

41
freqtrade/data/history/history_utils.py
View File

@@ -113,13 +113,15 @@ def refresh_data(datadir: Path,
:param timeframe: Timeframe (e.g. "5m")
:param pairs: List of pairs to load
:param exchange: Exchange object
:param data_format: dataformat to use
:param timerange: Limit data to be loaded to this timerange
"""
data_handler = get_datahandler(datadir, data_format)
for pair in pairs:
_download_pair_history(pair=pair, timeframe=timeframe,
datadir=datadir, timerange=timerange,
exchange=exchange, data_handler=data_handler)
for idx, pair in enumerate(pairs):
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
timeframe=timeframe, datadir=datadir,
timerange=timerange, exchange=exchange, data_handler=data_handler)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
@@ -152,13 +154,14 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
return data, start_ms
def _download_pair_history(datadir: Path,
def _download_pair_history(pair: str, *,
datadir: Path,
exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler = None) -> bool:
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
@@ -176,7 +179,7 @@ def _download_pair_history(datadir: Path,
try:
logger.info(
f'Download history data for pair: "{pair}", timeframe: {timeframe} '
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
f'and store in {datadir}.'
)
@@ -193,8 +196,9 @@ def _download_pair_history(datadir: Path,
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(
days=-new_pairs_days).float_timestamp) * 1000
arrow.utcnow().shift(
days=-new_pairs_days).int_timestamp * 1000,
is_new_pair=data.empty
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -233,7 +237,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
"""
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format)
for pair in pairs:
for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets:
pairs_not_available.append(pair)
logger.info(f"Skipping pair {pair}...")
@@ -246,10 +250,11 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
f'Deleting existing data for pair {pair}, interval {timeframe}.')
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
_download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe),
new_pairs_days=new_pairs_days,
timerange=timerange, data_handler=data_handler)
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
return pairs_not_available
@@ -271,7 +276,7 @@ def _download_trades_history(exchange: Exchange,
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:
since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
trades = data_handler.trades_load(pair)

8
freqtrade/data/history/idatahandler.py
View File

@@ -49,8 +49,8 @@ class IDataHandler(ABC):
"""
Store ohlcv data.
:param pair: Pair - used to generate filename
:timeframe: Timeframe - used to generate filename
:data: Dataframe containing OHLCV data
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
:return: None
"""
@@ -245,8 +245,8 @@ def get_datahandler(datadir: Path, data_format: str = None,
data_handler: IDataHandler = None) -> IDataHandler:
"""
:param datadir: Folder to save data
:data_format: dataformat to use
:data_handler: returns this datahandler if it exists or initializes a new one
:param data_format: dataformat to use
:param data_handler: returns this datahandler if it exists or initializes a new one
"""
if not data_handler:

6
freqtrade/data/history/jsondatahandler.py
View File

@@ -55,14 +55,14 @@ class JsonDataHandler(IDataHandler):
format looks as follows:
[[<date>,<open>,<high>,<low>,<close>]]
:param pair: Pair - used to generate filename
:timeframe: Timeframe - used to generate filename
:data: Dataframe containing OHLCV data
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
:return: None
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)
_data = data.copy()
# Convert date to int
_data['date'] = _data['date'].astype(np.int64) // 1000 // 1000
_data['date'] = _data['date'].view(np.int64) // 1000 // 1000
# Reset index, select only appropriate columns and save as json
_data.reset_index(drop=True).loc[:, self._columns].to_json(

12
freqtrade/edge/edge_positioning.py
View File

@@ -119,7 +119,7 @@ class Edge:
)
# Download informative pairs too
res = defaultdict(list)
for p, t in self.strategy.informative_pairs():
for p, t in self.strategy.gather_informative_pairs():
res[t].append(p)
for timeframe, inf_pairs in res.items():
timerange_startup = deepcopy(self._timerange)
@@ -151,7 +151,7 @@ class Edge:
# Fake run-mode to Edge
prior_rm = self.config['runmode']
self.config['runmode'] = RunMode.EDGE
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
preprocessed = self.strategy.advise_all_indicators(data)
self.config['runmode'] = prior_rm
# Print timeframe
@@ -231,12 +231,12 @@ class Edge:
'Minimum expectancy and minimum winrate are met only for %s,'
' so other pairs are filtered out.',
self._final_pairs
)
)
else:
logger.info(
'Edge removed all pairs as no pair with minimum expectancy '
'and minimum winrate was found !'
)
)
return self._final_pairs
@@ -247,7 +247,7 @@ class Edge:
final = []
for pair, info in self._cached_pairs.items():
if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
final.append({
'Pair': pair,
'Winrate': info.winrate,
@@ -301,7 +301,7 @@ class Edge:
def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
"""
This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
The calulation will be done per pair and per strategy.
The calculation will be done per pair and per strategy.
"""
# Removing pairs having less than min_trades_number
min_trades_number = self.edge_config.get('min_trade_number', 10)

3
freqtrade/enums/__init__.py
View File

@@ -1,6 +1,7 @@
# flake8: noqa: F401
from freqtrade.enums.backteststate import BacktestState
from freqtrade.enums.rpcmessagetype import RPCMessageType
from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
from freqtrade.enums.selltype import SellType
from freqtrade.enums.signaltype import SignalType
from freqtrade.enums.signaltype import SignalTagType, SignalType
from freqtrade.enums.state import State

15
freqtrade/enums/backteststate.py Normal file
View File

@@ -0,0 +1,15 @@
from enum import Enum
class BacktestState(Enum):
"""
Bot application states
"""
STARTUP = 1
DATALOAD = 2
ANALYZE = 3
CONVERT = 4
BACKTEST = 5
def __str__(self):
return f"{self.name.lower()}"

2
freqtrade/enums/rpcmessagetype.py
View File

@@ -11,6 +11,8 @@ class RPCMessageType(Enum):
SELL = 'sell'
SELL_FILL = 'sell_fill'
SELL_CANCEL = 'sell_cancel'
PROTECTION_TRIGGER = 'protection_trigger'
PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global'
def __repr__(self):
return self.value

1
freqtrade/enums/runmode.py
View File

@@ -14,6 +14,7 @@ class RunMode(Enum):
UTIL_EXCHANGE = "util_exchange"
UTIL_NO_EXCHANGE = "util_no_exchange"
PLOT = "plot"
WEBSERVER = "webserver"
OTHER = "other"

7
freqtrade/enums/signaltype.py
View File

@@ -7,3 +7,10 @@ class SignalType(Enum):
"""
BUY = "buy"
SELL = "sell"
class SignalTagType(Enum):
"""
Enum for signal columns
"""
BUY_TAG = "buy_tag"

4
freqtrade/exceptions.py
View File

@@ -47,7 +47,7 @@ class InvalidOrderException(ExchangeError):
class RetryableOrderError(InvalidOrderException):
"""
This is returned when the order is not found.
This Error will be repeated with increasing backof (in line with DDosError).
This Error will be repeated with increasing backoff (in line with DDosError).
"""
@@ -75,6 +75,6 @@ class DDosProtection(TemporaryError):
class StrategyError(FreqtradeException):
"""
Errors with custom user-code deteced.
Errors with custom user-code detected.
Usually caused by errors in the strategy.
"""

3
freqtrade/exchange/__init__.py
View File

@@ -1,6 +1,6 @@
# flake8: noqa: F401
# isort: off
from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS
from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS
from freqtrade.exchange.exchange import Exchange
# isort: on
from freqtrade.exchange.bibox import Bibox
@@ -15,6 +15,7 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
timeframe_to_seconds, validate_exchange,
validate_exchanges)
from freqtrade.exchange.ftx import Ftx
from freqtrade.exchange.gateio import Gateio
from freqtrade.exchange.hitbtc import Hitbtc
from freqtrade.exchange.kraken import Kraken
from freqtrade.exchange.kucoin import Kucoin

22
freqtrade/exchange/binance.py
View File

@@ -1,7 +1,8 @@
""" Binance exchange subclass """
import logging
from typing import Dict
from typing import Dict, List
import arrow
import ccxt
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
@@ -18,6 +19,7 @@ class Binance(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"time_in_force_parameter": "timeInForce",
"ohlcv_candle_limit": 1000,
"trades_pagination": "id",
"trades_pagination_arg": "fromId",
@@ -68,6 +70,7 @@ class Binance(Exchange):
amount=amount, price=rate, params=params)
logger.info('stoploss limit order added for %s. '
'stop price: %s. limit: %s', pair, stop_price, rate)
self._log_exchange_response('create_stoploss_order', order)
return order
except ccxt.InsufficientFunds as e:
raise InsufficientFundsError(
@@ -88,3 +91,20 @@ class Binance(Exchange):
f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, is_new_pair: bool
) -> List:
"""
Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
Does not work for other exchanges, which don't return the earliest data when called with "0"
"""
if is_new_pair:
x = await self._async_get_candle_history(pair, timeframe, 0)
if x and x[2] and x[2][0] and x[2][0][0] > since_ms:
# Set starting date to first available candle.
since_ms = x[2][0][0]
logger.info(f"Candle-data for {pair} available starting with "
f"{arrow.get(since_ms // 1000).isoformat()}.")
return await super()._async_get_historic_ohlcv(
pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair)

15
freqtrade/exchange/common.py
View File

@@ -16,8 +16,6 @@ API_FETCH_ORDER_RETRY_COUNT = 5
BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "
"Details in https://github.com/freqtrade/freqtrade/issues/1983",
"phemex": "Does not provide history. ",
"poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
}
@@ -51,6 +49,19 @@ EXCHANGE_HAS_OPTIONAL = [
]
def remove_credentials(config) -> None:
"""
Removes exchange keys from the configuration and specifies dry-run
Used for backtesting / hyperopt / edge and utils.
Modifies the input dict!
"""
if config.get('dry_run', False):
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
config['exchange']['password'] = ''
config['exchange']['uid'] = ''
def calculate_backoff(retrycount, max_retries):
"""
Calculate backoff

378
freqtrade/exchange/exchange.py
View File

@@ -19,15 +19,16 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRU
decimal_to_precision)
from pandas import DataFrame
from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes
from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES,
ListPairsWithTimeframes)
from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
InvalidOrderException, OperationalException, PricingError,
RetryableOrderError, TemporaryError)
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier,
retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED,
remove_credentials, retrier, retrier_async)
from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -53,12 +54,16 @@ class Exchange:
# Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
_params: Dict = {}
# Additional headers - added to the ccxt object
_headers: Dict = {}
# Dict to specify which options each exchange implements
# This defines defaults, which can be selectively overridden by subclasses using _ft_has
# or by specifying them in the configuration.
_ft_has_default: Dict = {
"stoploss_on_exchange": False,
"order_time_in_force": ["gtc"],
"time_in_force_parameter": "timeInForce",
"ohlcv_params": {},
"ohlcv_candle_limit": 500,
"ohlcv_partial_candle": True,
@@ -99,11 +104,13 @@ class Exchange:
# Holds all open sell orders for dry_run
self._dry_run_open_orders: Dict[str, Any] = {}
remove_credentials(config)
if config['dry_run']:
logger.info('Instance is running with dry_run enabled')
logger.info(f"Using CCXT {ccxt.__version__}")
exchange_config = config['exchange']
self.log_responses = exchange_config.get('log_responses', False)
# Deep merge ft_has with default ft_has options
self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default))
@@ -167,7 +174,7 @@ class Exchange:
asyncio.get_event_loop().run_until_complete(self._api_async.close())
def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt,
ccxt_kwargs: dict = None) -> ccxt.Exchange:
ccxt_kwargs: Dict = {}) -> ccxt.Exchange:
"""
Initialize ccxt with given config and return valid
ccxt instance.
@@ -186,6 +193,10 @@ class Exchange:
}
if ccxt_kwargs:
logger.info('Applying additional ccxt config: %s', ccxt_kwargs)
if self._headers:
# Inject static headers after the above output to not confuse users.
ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs)
if ccxt_kwargs:
ex_config.update(ccxt_kwargs)
try:
@@ -226,10 +237,15 @@ class Exchange:
"""exchange ccxt precisionMode"""
return self._api.precisionMode
def _log_exchange_response(self, endpoint, response) -> None:
""" Log exchange responses """
if self.log_responses:
logger.info(f"API {endpoint}: {response}")
def ohlcv_candle_limit(self, timeframe: str) -> int:
"""
Exchange ohlcv candle limit
Uses ohlcv_candle_limit_per_timeframe if the exchange has different limts
Uses ohlcv_candle_limit_per_timeframe if the exchange has different limits
per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit
:param timeframe: Timeframe to check
:return: Candle limit as integer
@@ -345,9 +361,16 @@ class Exchange:
def validate_stakecurrency(self, stake_currency: str) -> None:
"""
Checks stake-currency against available currencies on the exchange.
Only runs on startup. If markets have not been loaded, there's been a problem with
the connection to the exchange.
:param stake_currency: Stake-currency to validate
:raise: OperationalException if stake-currency is not available.
"""
if not self._markets:
raise OperationalException(
'Could not load markets, therefore cannot start. '
'Please investigate the above error for more details.'
)
quote_currencies = self.get_quote_currencies()
if stake_currency not in quote_currencies:
raise OperationalException(
@@ -381,7 +404,7 @@ class Exchange:
# its contents depend on the exchange.
# It can also be a string or similar ... so we need to verify that first.
elif (isinstance(self.markets[pair].get('info', None), dict)
and self.markets[pair].get('info', {}).get('IsRestricted', False)):
and self.markets[pair].get('info', {}).get('prohibitedIn', False)):
# Warn users about restricted pairs in whitelist.
# We cannot determine reliably if Users are affected.
logger.warning(f"Pair {pair} is restricted for some users on this exchange."
@@ -457,7 +480,7 @@ class Exchange:
if startup_candles + 5 > candle_limit:
raise OperationalException(
f"This strategy requires {startup_candles} candles to start. "
f"{self.name} only provides {candle_limit} for {timeframe}.")
f"{self.name} only provides {candle_limit - 5} for {timeframe}.")
def exchange_has(self, endpoint: str) -> bool:
"""
@@ -469,11 +492,11 @@ class Exchange:
return endpoint in self._api.has and self._api.has[endpoint]
def amount_to_precision(self, pair: str, amount: float) -> float:
'''
"""
Returns the amount to buy or sell to a precision the Exchange accepts
Re-implementation of ccxt internal methods - ensuring we can test the result is correct
based on our definitions.
'''
"""
if self.markets[pair]['precision']['amount']:
amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE,
precision=self.markets[pair]['precision']['amount'],
@@ -483,14 +506,14 @@ class Exchange:
return amount
def price_to_precision(self, pair: str, price: float) -> float:
'''
"""
Returns the price rounded up to the precision the Exchange accepts.
Partial Re-implementation of ccxt internal method decimal_to_precision(),
which does not support rounding up
TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
align with amount_to_precision().
Rounds up
'''
"""
if self.markets[pair]['precision']['price']:
# price = float(decimal_to_precision(price, rounding_mode=ROUND,
# precision=self.markets[pair]['precision']['price'],
@@ -500,7 +523,7 @@ class Exchange:
precision = self.markets[pair]['precision']['price']
missing = price % precision
if missing != 0:
price = price - missing + precision
price = round(price - missing + precision, 10)
else:
symbol_prec = self.markets[pair]['precision']['price']
big_price = price * pow(10, symbol_prec)
@@ -545,7 +568,7 @@ class Exchange:
amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent',
DEFAULT_AMOUNT_RESERVE_PERCENT)
amount_reserve_percent = (
amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
)
# it should not be more than 50%
amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1)
@@ -561,7 +584,7 @@ class Exchange:
rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount)
dry_order = {
dry_order: Dict[str, Any] = {
'id': order_id,
'symbol': pair,
'price': rate,
@@ -572,31 +595,110 @@ class Exchange:
'side': side,
'remaining': _amount,
'datetime': arrow.utcnow().isoformat(),
'timestamp': int(arrow.utcnow().int_timestamp * 1000),
'timestamp': arrow.utcnow().int_timestamp * 1000,
'status': "closed" if ordertype == "market" else "open",
'fee': None,
'info': {}
}
self._store_dry_order(dry_order, pair)
if dry_order["type"] in ["stop_loss_limit", "stop-loss-limit"]:
dry_order["info"] = {"stopPrice": dry_order["price"]}
if dry_order["type"] == "market":
# Update market order pricing
average = self.get_dry_market_fill_price(pair, side, amount, rate)
dry_order.update({
'average': average,
'cost': dry_order['amount'] * average,
})
dry_order = self.add_dry_order_fee(pair, dry_order)
dry_order = self.check_dry_limit_order_filled(dry_order)
self._dry_run_open_orders[dry_order["id"]] = dry_order
# Copy order and close it - so the returned order is open unless it's a market order
return dry_order
def _store_dry_order(self, dry_order: Dict, pair: str) -> None:
closed_order = dry_order.copy()
if closed_order['type'] in ["market", "limit"]:
closed_order.update({
'status': 'closed',
'filled': closed_order['amount'],
'remaining': 0,
'fee': {
'currency': self.get_pair_quote_currency(pair),
'cost': dry_order['cost'] * self.get_fee(pair),
'rate': self.get_fee(pair)
}
})
if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]:
closed_order["info"].update({"stopPrice": closed_order["price"]})
self._dry_run_open_orders[closed_order["id"]] = closed_order
def add_dry_order_fee(self, pair: str, dry_order: Dict[str, Any]) -> Dict[str, Any]:
dry_order.update({
'fee': {
'currency': self.get_pair_quote_currency(pair),
'cost': dry_order['cost'] * self.get_fee(pair),
'rate': self.get_fee(pair)
}
})
return dry_order
def get_dry_market_fill_price(self, pair: str, side: str, amount: float, rate: float) -> float:
"""
Get the market order fill price based on orderbook interpolation
"""
if self.exchange_has('fetchL2OrderBook'):
ob = self.fetch_l2_order_book(pair, 20)
ob_type = 'asks' if side == 'buy' else 'bids'
slippage = 0.05
max_slippage_val = rate * ((1 + slippage) if side == 'buy' else (1 - slippage))
remaining_amount = amount
filled_amount = 0
for book_entry in ob[ob_type]:
book_entry_price = book_entry[0]
book_entry_coin_volume = book_entry[1]
if remaining_amount > 0:
if remaining_amount < book_entry_coin_volume:
# Orderbook at this slot bigger than remaining amount
filled_amount += remaining_amount * book_entry_price
break
else:
filled_amount += book_entry_coin_volume * book_entry_price
remaining_amount -= book_entry_coin_volume
else:
break
else:
# If remaining_amount wasn't consumed completely (break was not called)
filled_amount += remaining_amount * book_entry_price
forecast_avg_filled_price = max(filled_amount, 0) / amount
# Limit max. slippage to specified value
if side == 'buy':
forecast_avg_filled_price = min(forecast_avg_filled_price, max_slippage_val)
else:
forecast_avg_filled_price = max(forecast_avg_filled_price, max_slippage_val)
return self.price_to_precision(pair, forecast_avg_filled_price)
return rate
def _is_dry_limit_order_filled(self, pair: str, side: str, limit: float) -> bool:
if not self.exchange_has('fetchL2OrderBook'):
return True
ob = self.fetch_l2_order_book(pair, 1)
if side == 'buy':
price = ob['asks'][0][0]
logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}")
if limit >= price:
return True
else:
price = ob['bids'][0][0]
logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}")
if limit <= price:
return True
return False
def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]:
"""
Check dry-run limit order fill and update fee (if it filled).
"""
if order['status'] != "closed" and order['type'] in ["limit"]:
pair = order['symbol']
if self._is_dry_limit_order_filled(pair, order['side'], order['price']):
order.update({
'status': 'closed',
'filled': order['amount'],
'remaining': 0,
})
self.add_dry_order_fee(pair, order)
return order
def fetch_dry_run_order(self, order_id) -> Dict[str, Any]:
"""
@@ -605,6 +707,7 @@ class Exchange:
"""
try:
order = self._dry_run_open_orders[order_id]
order = self.check_dry_limit_order_filled(order)
return order
except KeyError as e:
# Gracefully handle errors with dry-run orders.
@@ -614,7 +717,17 @@ class Exchange:
# Order handling
def create_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict:
rate: float, time_in_force: str = 'gtc') -> Dict:
if self._config['dry_run']:
dry_order = self.create_dry_run_order(pair, ordertype, side, amount, rate)
return dry_order
params = self._params.copy()
if time_in_force != 'gtc' and ordertype != 'market':
param = self._ft_has.get('time_in_force_parameter', '')
params.update({param: time_in_force})
try:
# Set the precision for amount and price(rate) as accepted by the exchange
amount = self.amount_to_precision(pair, amount)
@@ -622,8 +735,10 @@ class Exchange:
or self._api.options.get("createMarketBuyOrderRequiresPrice", False))
rate_for_order = self.price_to_precision(pair, rate) if needs_price else None
return self._api.create_order(pair, ordertype, side,
amount, rate_for_order, params)
order = self._api.create_order(pair, ordertype, side,
amount, rate_for_order, params)
self._log_exchange_response('create_order', order)
return order
except ccxt.InsufficientFunds as e:
raise InsufficientFundsError(
@@ -643,32 +758,6 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(e) from e
def buy(self, pair: str, ordertype: str, amount: float,
rate: float, time_in_force: str) -> Dict:
if self._config['dry_run']:
dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate)
return dry_order
params = self._params.copy()
if time_in_force != 'gtc' and ordertype != 'market':
params.update({'timeInForce': time_in_force})
return self.create_order(pair, ordertype, 'buy', amount, rate, params)
def sell(self, pair: str, ordertype: str, amount: float,
rate: float, time_in_force: str = 'gtc') -> Dict:
if self._config['dry_run']:
dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate)
return dry_order
params = self._params.copy()
if time_in_force != 'gtc' and ordertype != 'market':
params.update({'timeInForce': time_in_force})
return self.create_order(pair, ordertype, 'sell', amount, rate, params)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
@@ -694,7 +783,9 @@ class Exchange:
if self._config['dry_run']:
return self.fetch_dry_run_order(order_id)
try:
return self._api.fetch_order(order_id, pair)
order = self._api.fetch_order(order_id, pair)
self._log_exchange_response('fetch_order', order)
return order
except ccxt.OrderNotFound as e:
raise RetryableOrderError(
f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e
@@ -717,6 +808,8 @@ class Exchange:
"""
Simple wrapper calling either fetch_order or fetch_stoploss_order depending on
the stoploss_order parameter
:param order_id: OrderId to fetch order
:param pair: Pair corresponding to order_id
:param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order.
"""
if stoploss_order:
@@ -729,7 +822,7 @@ class Exchange:
:param order: Order dict as returned from fetch_order()
:return: True if order has been cancelled without being filled, False otherwise.
"""
return (order.get('status') in ('closed', 'canceled', 'cancelled')
return (order.get('status') in NON_OPEN_EXCHANGE_STATES
and order.get('filled') == 0.0)
@retrier
@@ -744,7 +837,9 @@ class Exchange:
return {}
try:
return self._api.cancel_order(order_id, pair)
order = self._api.cancel_order(order_id, pair)
self._log_exchange_response('cancel_order', order)
return order
except ccxt.InvalidOrder as e:
raise InvalidOrderException(
f'Could not cancel order. Message: {e}') from e
@@ -916,99 +1011,64 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(e) from e
def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1,
order_book_min: int = 1):
def get_rate(self, pair: str, refresh: bool, side: str) -> float:
"""
Helper generator to query orderbook in loop (used for early sell-order placing)
"""
order_book = self.fetch_l2_order_book(pair, order_book_max)
for i in range(order_book_min, order_book_max + 1):
yield order_book[side][i - 1][0]
def get_buy_rate(self, pair: str, refresh: bool) -> float:
"""
Calculates bid target between current ask price and last price
Calculates bid/ask target
bid rate - between current ask price and last price
ask rate - either using ticker bid or first bid based on orderbook
or remain static in any other case since it's not updating.
:param pair: Pair to get rate for
:param refresh: allow cached data
:param side: "buy" or "sell"
:return: float: Price
:raises PricingError if orderbook price could not be determined.
"""
cache_rate: TTLCache = self._buy_rate_cache if side == "buy" else self._sell_rate_cache
[strat_name, name] = ['bid_strategy', 'Buy'] if side == "buy" else ['ask_strategy', 'Sell']
if not refresh:
rate = self._buy_rate_cache.get(pair)
rate = cache_rate.get(pair)
# Check if cache has been invalidated
if rate:
logger.debug(f"Using cached buy rate for {pair}.")
logger.debug(f"Using cached {side} rate for {pair}.")
return rate
bid_strategy = self._config.get('bid_strategy', {})
if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
conf_strategy = self._config.get(strat_name, {})
order_book_top = bid_strategy.get('order_book_top', 1)
if conf_strategy.get('use_order_book', False) and ('use_order_book' in conf_strategy):
order_book_top = conf_strategy.get('order_book_top', 1)
order_book = self.fetch_l2_order_book(pair, order_book_top)
logger.debug('order_book %s', order_book)
# top 1 = index 0
try:
rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
rate = order_book[f"{conf_strategy['price_side']}s"][order_book_top - 1][0]
except (IndexError, KeyError) as e:
logger.warning(
"Buy Price from orderbook could not be determined."
f"Orderbook: {order_book}"
)
f"{name} Price at location {order_book_top} from orderbook could not be "
f"determined. Orderbook: {order_book}"
)
raise PricingError from e
logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
used_rate = rate_from_l2
price_side = {conf_strategy['price_side'].capitalize()}
logger.debug(f"{name} price from orderbook {price_side}"
f"side - top {order_book_top} order book {side} rate {rate:.8f}")
else:
logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
logger.debug(f"Using Last {conf_strategy['price_side'].capitalize()} / Last Price")
ticker = self.fetch_ticker(pair)
ticker_rate = ticker[bid_strategy['price_side']]
if ticker['last'] and ticker_rate > ticker['last']:
balance = bid_strategy['ask_last_balance']
ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
used_rate = ticker_rate
self._buy_rate_cache[pair] = used_rate
return used_rate
def get_sell_rate(self, pair: str, refresh: bool) -> float:
"""
Get sell rate - either using ticker bid or first bid based on orderbook
or remain static in any other case since it's not updating.
:param pair: Pair to get rate for
:param refresh: allow cached data
:return: Bid rate
:raises PricingError if price could not be determined.
"""
if not refresh:
rate = self._sell_rate_cache.get(pair)
# Check if cache has been invalidated
if rate:
logger.debug(f"Using cached sell rate for {pair}.")
return rate
ask_strategy = self._config.get('ask_strategy', {})
if ask_strategy.get('use_order_book', False):
# This code is only used for notifications, selling uses the generator directly
logger.info(
f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
)
try:
rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s"))
except (IndexError, KeyError) as e:
logger.warning("Sell Price at location from orderbook could not be determined.")
raise PricingError from e
else:
ticker = self.fetch_ticker(pair)
ticker_rate = ticker[ask_strategy['price_side']]
if ticker['last'] and ticker_rate < ticker['last']:
balance = ask_strategy.get('bid_last_balance', 0.0)
ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
ticker_rate = ticker[conf_strategy['price_side']]
if ticker['last'] and ticker_rate:
if side == 'buy' and ticker_rate > ticker['last']:
balance = conf_strategy.get('ask_last_balance', 0.0)
ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
elif side == 'sell' and ticker_rate < ticker['last']:
balance = conf_strategy.get('bid_last_balance', 0.0)
ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
rate = ticker_rate
if rate is None:
raise PricingError(f"Sell-Rate for {pair} was empty.")
self._sell_rate_cache[pair] = rate
raise PricingError(f"{name}-Rate for {pair} was empty.")
cache_rate[pair] = rate
return rate
# Fee handling
@@ -1042,6 +1102,7 @@ class Exchange:
pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000))
matched_trades = [trade for trade in my_trades if trade['order'] == order_id]
self._log_exchange_response('get_trades_for_order', matched_trades)
return matched_trades
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
@@ -1134,7 +1195,7 @@ class Exchange:
# Historic data
def get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int) -> List:
since_ms: int, is_new_pair: bool = False) -> List:
"""
Get candle history using asyncio and returns the list of candles.
Handles all async work for this.
@@ -1146,7 +1207,7 @@ class Exchange:
"""
return asyncio.get_event_loop().run_until_complete(
self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe,
since_ms=since_ms))
since_ms=since_ms, is_new_pair=is_new_pair))
def get_historic_ohlcv_as_df(self, pair: str, timeframe: str,
since_ms: int) -> DataFrame:
@@ -1161,11 +1222,12 @@ class Exchange:
return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
async def _async_get_historic_ohlcv(self, pair: str,
timeframe: str,
since_ms: int) -> List:
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, is_new_pair: bool
) -> List:
"""
Download historic ohlcv
:param is_new_pair: used by binance subclass to allow "fast" new pair downloading
"""
one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe)
@@ -1178,21 +1240,22 @@ class Exchange:
pair, timeframe, since) for since in
range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)]
results = await asyncio.gather(*input_coroutines, return_exceptions=True)
# Combine gathered results
data: List = []
for res in results:
if isinstance(res, Exception):
logger.warning("Async code raised an exception: %s", res.__class__.__name__)
continue
# Deconstruct tuple if it's not an exception
p, _, new_data = res
if p == pair:
data.extend(new_data)
# Chunk requests into batches of 100 to avoid overwelming ccxt Throttling
for input_coro in chunks(input_coroutines, 100):
results = await asyncio.gather(*input_coro, return_exceptions=True)
for res in results:
if isinstance(res, Exception):
logger.warning("Async code raised an exception: %s", res.__class__.__name__)
continue
# Deconstruct tuple if it's not an exception
p, _, new_data = res
if p == pair:
data.extend(new_data)
# Sort data again after extending the result - above calls return in "async order"
data = sorted(data, key=lambda x: x[0])
logger.info("Downloaded data for %s with length %s.", pair, len(data))
logger.info(f"Downloaded data for {pair} with length {len(data)}.")
return data
def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *,
@@ -1210,7 +1273,7 @@ class Exchange:
logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list))
input_coroutines = []
cached_pairs = []
# Gather coroutines to run
for pair, timeframe in set(pair_list):
if (((pair, timeframe) not in self._klines)
@@ -1222,6 +1285,7 @@ class Exchange:
"Using cached candle (OHLCV) data for pair %s, timeframe %s ...",
pair, timeframe
)
cached_pairs.append((pair, timeframe))
results = asyncio.get_event_loop().run_until_complete(
asyncio.gather(*input_coroutines, return_exceptions=True))
@@ -1239,11 +1303,15 @@ class Exchange:
self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
ohlcv_df = ohlcv_to_dataframe(
ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
results_df[(pair, timeframe)] = ohlcv_df
if cache:
self._klines[(pair, timeframe)] = ohlcv_df
# Return cached klines
for pair, timeframe in cached_pairs:
results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False)
return results_df
def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool:
@@ -1454,7 +1522,7 @@ class Exchange:
:returns List of trade data
"""
if not self.exchange_has("fetchTrades"):
raise OperationalException("This exchange does not suport downloading Trades.")
raise OperationalException("This exchange does not support downloading Trades.")
return asyncio.get_event_loop().run_until_complete(
self._async_get_trade_history(pair=pair, since=since,

7
freqtrade/exchange/ftx.py
View File

@@ -69,6 +69,7 @@ class Ftx(Exchange):
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
amount=amount, params=params)
self._log_exchange_response('create_stoploss_order', order)
logger.info('stoploss order added for %s. '
'stop price: %s.', pair, stop_price)
return order
@@ -99,12 +100,14 @@ class Ftx(Exchange):
orders = self._api.fetch_orders(pair, None, params={'type': 'stop'})
order = [order for order in orders if order['id'] == order_id]
self._log_exchange_response('fetch_stoploss_order', order)
if len(order) == 1:
if order[0].get('status') == 'closed':
# Trigger order was triggered ...
real_order_id = order[0].get('info', {}).get('orderId')
order1 = self._api.fetch_order(real_order_id, pair)
self._log_exchange_response('fetch_stoploss_order1', order1)
# Fake type to stop - as this was really a stop order.
order1['id_stop'] = order1['id']
order1['id'] = order_id
@@ -131,7 +134,9 @@ class Ftx(Exchange):
if self._config['dry_run']:
return {}
try:
return self._api.cancel_order(order_id, pair, params={'type': 'stop'})
order = self._api.cancel_order(order_id, pair, params={'type': 'stop'})
self._log_exchange_response('cancel_stoploss_order', order)
return order
except ccxt.InvalidOrder as e:
raise InvalidOrderException(
f'Could not cancel order. Message: {e}') from e

33
freqtrade/exchange/gateio.py Normal file
View File

@@ -0,0 +1,33 @@
""" Gate.io exchange subclass """
import logging
from typing import Dict
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Gateio(Exchange):
"""
Gate.io exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
}
_headers = {'X-Gate-Channel-Id': 'freqtrade'}
def validate_ordertypes(self, order_types: Dict) -> None:
super().validate_ordertypes(order_types)
if any(v == 'market' for k, v in order_types.items()):
raise OperationalException(
f'Exchange {self.name} does not support market orders.')

3
freqtrade/exchange/kraken.py
View File

@@ -49,7 +49,7 @@ class Kraken(Exchange):
orders = self._api.fetch_open_orders()
order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1],
x["remaining"] if x["side"] == "sell" else x["remaining"] * x["price"],
# Don't remove the below comment, this can be important for debuggung
# Don't remove the below comment, this can be important for debugging
# x["side"], x["amount"],
) for x in orders]
for bal in balances:
@@ -103,6 +103,7 @@ class Kraken(Exchange):
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
amount=amount, price=stop_price, params=params)
self._log_exchange_response('create_stoploss_order', order)
logger.info('stoploss order added for %s. '
'stop price: %s.', pair, stop_price)
return order

2
freqtrade/exchange/kucoin.py
View File

@@ -21,4 +21,6 @@ class Kucoin(Exchange):
_ft_has: Dict = {
"l2_limit_range": [20, 100],
"l2_limit_range_required": False,
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"time_in_force_parameter": "timeInForce",
}

296
freqtrade/freqtradebot.py
View File

@@ -70,7 +70,7 @@ class FreqtradeBot(LoggingMixin):
PairLocks.timeframe = self.config['timeframe']
self.protections = ProtectionManager(self.config)
self.protections = ProtectionManager(self.config, self.strategy.protections)
# RPC runs in separate threads, can start handling external commands just after
# initialization, even before Freqtradebot has a chance to start its throttling,
@@ -83,10 +83,10 @@ class FreqtradeBot(LoggingMixin):
self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists)
# Attach Dataprovider to Strategy baseclass
IStrategy.dp = self.dataprovider
# Attach Wallets to Strategy baseclass
IStrategy.wallets = self.wallets
# Attach Dataprovider to strategy instance
self.strategy.dp = self.dataprovider
# Attach Wallets to strategy instance
self.strategy.wallets = self.wallets
# Initializing Edge only if enabled
self.edge = Edge(self.config, self.exchange, self.strategy) if \
@@ -98,8 +98,8 @@ class FreqtradeBot(LoggingMixin):
initial_state = self.config.get('initial_state')
self.state = State[initial_state.upper()] if initial_state else State.STOPPED
# Protect sell-logic from forcesell and viceversa
self._sell_lock = Lock()
# Protect sell-logic from forcesell and vice versa
self._exit_lock = Lock()
LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe))
def notify_status(self, msg: str) -> None:
@@ -139,7 +139,7 @@ class FreqtradeBot(LoggingMixin):
# Only update open orders on startup
# This will update the database after the initial migration
self.update_open_orders()
self.startup_update_open_orders()
def process(self) -> None:
"""
@@ -160,20 +160,20 @@ class FreqtradeBot(LoggingMixin):
# Refreshing candles
self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist),
self.strategy.informative_pairs())
self.strategy.gather_informative_pairs())
strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)()
self.strategy.analyze(self.active_pair_whitelist)
with self._sell_lock:
with self._exit_lock:
# Check and handle any timed out open orders
self.check_handle_timedout()
# Protect from collisions with forcesell.
# Without this, freqtrade my try to recreate stoploss_on_exchange orders
# while selling is in process, since telegram messages arrive in an different thread.
with self._sell_lock:
with self._exit_lock:
trades = Trade.get_open_trades()
# First process current opened trades (positions)
self.exit_positions(trades)
@@ -237,7 +237,7 @@ class FreqtradeBot(LoggingMixin):
open_trades = len(Trade.get_open_trades())
return max(0, self.config['max_open_trades'] - open_trades)
def update_open_orders(self):
def startup_update_open_orders(self):
"""
Updates open orders based on order list kept in the database.
Mainly updates the state of orders - but may also close trades
@@ -296,9 +296,9 @@ class FreqtradeBot(LoggingMixin):
if sell_order:
self.refind_lost_order(trade)
else:
self.reupdate_buy_order_fees(trade)
self.reupdate_enter_order_fees(trade)
def reupdate_buy_order_fees(self, trade: Trade):
def reupdate_enter_order_fees(self, trade: Trade):
"""
Get buy order from database, and try to reupdate.
Handles trades where the initial fee-update did not work.
@@ -420,26 +420,24 @@ class FreqtradeBot(LoggingMixin):
return False
# running get_signal on historical data fetched
(buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
(buy, sell, buy_tag) = self.strategy.get_signal(
pair,
self.strategy.timeframe,
analyzed_df
)
if buy and not sell:
stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
if not stake_amount:
logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
return False
logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
f"{stake_amount} ...")
bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
if ((bid_check_dom.get('enabled', False)) and
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
if self._check_depth_of_market_buy(pair, bid_check_dom):
return self.execute_buy(pair, stake_amount)
return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
else:
return False
return self.execute_buy(pair, stake_amount)
return self.execute_entry(pair, stake_amount, buy_tag=buy_tag)
else:
return False
@@ -467,54 +465,70 @@ class FreqtradeBot(LoggingMixin):
logger.info(f"Bids to asks delta for {pair} does not satisfy condition.")
return False
def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None,
forcebuy: bool = False) -> bool:
def execute_entry(self, pair: str, stake_amount: float, price: Optional[float] = None,
forcebuy: bool = False, buy_tag: Optional[str] = None) -> bool:
"""
Executes a limit buy for the given pair
:param pair: pair for which we want to create a LIMIT_BUY
:param stake_amount: amount of stake-currency for the pair
:return: True if a buy order is created, false if it fails.
"""
time_in_force = self.strategy.order_time_in_force['buy']
if price:
buy_limit_requested = price
enter_limit_requested = price
else:
# Calculate price
buy_limit_requested = self.exchange.get_buy_rate(pair, True)
proposed_enter_rate = self.exchange.get_rate(pair, refresh=True, side="buy")
custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price,
default_retval=proposed_enter_rate)(
pair=pair, current_time=datetime.now(timezone.utc),
proposed_rate=proposed_enter_rate)
if not buy_limit_requested:
enter_limit_requested = self.get_valid_price(custom_entry_price, proposed_enter_rate)
if not enter_limit_requested:
raise PricingError('Could not determine buy price.')
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested,
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, enter_limit_requested,
self.strategy.stoploss)
if min_stake_amount is not None and min_stake_amount > stake_amount:
logger.warning(
f"Can't open a new trade for {pair}: stake amount "
f"is too small ({stake_amount} < {min_stake_amount})"
)
if not self.edge:
max_stake_amount = self.wallets.get_available_stake_amount()
stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
default_retval=stake_amount)(
pair=pair, current_time=datetime.now(timezone.utc),
current_rate=enter_limit_requested, proposed_stake=stake_amount,
min_stake=min_stake_amount, max_stake=max_stake_amount)
stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
if not stake_amount:
return False
amount = stake_amount / buy_limit_requested
logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
f"{stake_amount} ...")
amount = stake_amount / enter_limit_requested
order_type = self.strategy.order_types['buy']
if forcebuy:
# Forcebuy can define a different ordertype
order_type = self.strategy.order_types.get('forcebuy', order_type)
if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested,
pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested,
time_in_force=time_in_force, current_time=datetime.now(timezone.utc)):
logger.info(f"User requested abortion of buying {pair}")
return False
amount = self.exchange.amount_to_precision(pair, amount)
order = self.exchange.buy(pair=pair, ordertype=order_type,
amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force)
order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy",
amount=amount, rate=enter_limit_requested,
time_in_force=time_in_force)
order_obj = Order.parse_from_ccxt_object(order, pair, 'buy')
order_id = order['id']
order_status = order.get('status', None)
# we assume the order is executed at the price requested
buy_limit_filled_price = buy_limit_requested
enter_limit_filled_price = enter_limit_requested
amount_requested = amount
if order_status == 'expired' or order_status == 'rejected':
@@ -537,13 +551,13 @@ class FreqtradeBot(LoggingMixin):
)
stake_amount = order['cost']
amount = safe_value_fallback(order, 'filled', 'amount')
buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
enter_limit_filled_price = safe_value_fallback(order, 'average', 'price')
# in case of FOK the order may be filled immediately and fully
elif order_status == 'closed':
stake_amount = order['cost']
amount = safe_value_fallback(order, 'filled', 'amount')
buy_limit_filled_price = safe_value_fallback(order, 'average', 'price')
enter_limit_filled_price = safe_value_fallback(order, 'average', 'price')
# Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL
fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker')
@@ -555,12 +569,13 @@ class FreqtradeBot(LoggingMixin):
amount_requested=amount_requested,
fee_open=fee,
fee_close=fee,
open_rate=buy_limit_filled_price,
open_rate_requested=buy_limit_requested,
open_rate=enter_limit_filled_price,
open_rate_requested=enter_limit_requested,
open_date=datetime.utcnow(),
exchange=self.exchange.id,
open_order_id=order_id,
strategy=self.strategy.get_strategy_name(),
buy_tag=buy_tag,
timeframe=timeframe_to_minutes(self.config['timeframe'])
)
trade.orders.append(order_obj)
@@ -575,17 +590,18 @@ class FreqtradeBot(LoggingMixin):
# Updating wallets
self.wallets.update()
self._notify_buy(trade, order_type)
self._notify_enter(trade, order_type)
return True
def _notify_buy(self, trade: Trade, order_type: str) -> None:
def _notify_enter(self, trade: Trade, order_type: str) -> None:
"""
Sends rpc notification when a buy occurred.
"""
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY,
'buy_tag': trade.buy_tag,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'limit': trade.open_rate,
@@ -601,15 +617,16 @@ class FreqtradeBot(LoggingMixin):
# Send the message
self.rpc.send_msg(msg)
def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
"""
Sends rpc notification when a buy cancel occurred.
"""
current_rate = self.exchange.get_buy_rate(trade.pair, False)
current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="buy")
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_CANCEL,
'buy_tag': trade.buy_tag,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'limit': trade.open_rate,
@@ -626,10 +643,11 @@ class FreqtradeBot(LoggingMixin):
# Send the message
self.rpc.send_msg(msg)
def _notify_buy_fill(self, trade: Trade) -> None:
def _notify_enter_fill(self, trade: Trade) -> None:
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_FILL,
'buy_tag': trade.buy_tag,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'open_rate': trade.open_rate,
@@ -683,46 +701,21 @@ class FreqtradeBot(LoggingMixin):
(buy, sell) = (False, False)
config_ask_strategy = self.config.get('ask_strategy', {})
if (config_ask_strategy.get('use_sell_signal', True) or
config_ask_strategy.get('ignore_roi_if_buy_signal', False)):
if (self.config.get('use_sell_signal', True) or
self.config.get('ignore_roi_if_buy_signal', False)):
analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair,
self.strategy.timeframe)
(buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
(buy, sell, _) = self.strategy.get_signal(
trade.pair,
self.strategy.timeframe,
analyzed_df
)
if config_ask_strategy.get('use_order_book', False):
order_book_min = config_ask_strategy.get('order_book_min', 1)
order_book_max = config_ask_strategy.get('order_book_max', 1)
logger.debug(f'Using order book between {order_book_min} and {order_book_max} '
f'for selling {trade.pair}...')
order_book = self.exchange._order_book_gen(
trade.pair, f"{config_ask_strategy['price_side']}s",
order_book_min=order_book_min, order_book_max=order_book_max)
for i in range(order_book_min, order_book_max + 1):
try:
sell_rate = next(order_book)
except (IndexError, KeyError) as e:
logger.warning(
f"Sell Price at location {i} from orderbook could not be determined."
)
raise PricingError from e
logger.debug(f" order book {config_ask_strategy['price_side']} top {i}: "
f"{sell_rate:0.8f}")
# Assign sell-rate to cache - otherwise sell-rate is never updated in the cache,
# resulting in outdated RPC messages
self.exchange._sell_rate_cache[trade.pair] = sell_rate
if self._check_and_execute_sell(trade, sell_rate, buy, sell):
return True
else:
logger.debug('checking sell')
sell_rate = self.exchange.get_sell_rate(trade.pair, True)
if self._check_and_execute_sell(trade, sell_rate, buy, sell):
return True
logger.debug('checking sell')
exit_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell")
if self._check_and_execute_exit(trade, exit_rate, buy, sell):
return True
logger.debug('Found no sell signal for %s.', trade)
return False
@@ -751,8 +744,8 @@ class FreqtradeBot(LoggingMixin):
except InvalidOrderException as e:
trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}')
logger.warning('Selling the trade forcefully')
self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
logger.warning('Exiting the trade forcefully')
self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple(
sell_type=SellType.EMERGENCY_SELL))
except ExchangeError:
@@ -789,7 +782,7 @@ class FreqtradeBot(LoggingMixin):
# Lock pair for one candle to prevent immediate rebuys
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_sell(trade, "stoploss")
self._notify_exit(trade, "stoploss")
return True
if trade.open_order_id or not trade.is_open:
@@ -834,7 +827,7 @@ class FreqtradeBot(LoggingMixin):
"""
Check to see if stoploss on exchange should be updated
in case of trailing stoploss on exchange
:param Trade: Corresponding Trade
:param trade: Corresponding Trade
:param order: Current on exchange stoploss order
:return: None
"""
@@ -858,19 +851,19 @@ class FreqtradeBot(LoggingMixin):
logger.warning(f"Could not create trailing stoploss order "
f"for pair {trade.pair}.")
def _check_and_execute_sell(self, trade: Trade, sell_rate: float,
def _check_and_execute_exit(self, trade: Trade, exit_rate: float,
buy: bool, sell: bool) -> bool:
"""
Check and execute sell
Check and execute exit
"""
should_sell = self.strategy.should_sell(
trade, sell_rate, datetime.now(timezone.utc), buy, sell,
trade, exit_rate, datetime.now(timezone.utc), buy, sell,
force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0
)
if should_sell.sell_flag:
logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
self.execute_sell(trade, sell_rate, should_sell)
self.execute_trade_exit(trade, exit_rate, should_sell)
return True
return False
@@ -913,7 +906,7 @@ class FreqtradeBot(LoggingMixin):
default_retval=False)(pair=trade.pair,
trade=trade,
order=order))):
self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['TIMEOUT'])
self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT'])
elif (order['side'] == 'sell' and (order['status'] == 'open' or fully_cancelled) and (
fully_cancelled
@@ -922,7 +915,7 @@ class FreqtradeBot(LoggingMixin):
default_retval=False)(pair=trade.pair,
trade=trade,
order=order))):
self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['TIMEOUT'])
self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['TIMEOUT'])
def cancel_all_open_orders(self) -> None:
"""
@@ -938,13 +931,13 @@ class FreqtradeBot(LoggingMixin):
continue
if order['side'] == 'buy':
self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['ALL_CANCELLED'])
self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['ALL_CANCELLED'])
elif order['side'] == 'sell':
self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED'])
self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED'])
Trade.commit()
def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool:
def handle_cancel_enter(self, trade: Trade, order: Dict, reason: str) -> bool:
"""
Buy cancel - cancel order
:return: True if order was fully cancelled
@@ -952,7 +945,7 @@ class FreqtradeBot(LoggingMixin):
was_trade_fully_canceled = False
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in ('cancelled', 'canceled', 'closed'):
if order['status'] not in constants.NON_OPEN_EXCHANGE_STATES:
filled_val = order.get('filled', 0.0) or 0.0
filled_stake = filled_val * trade.open_rate
minstake = self.exchange.get_min_pair_stake_amount(
@@ -968,7 +961,7 @@ class FreqtradeBot(LoggingMixin):
# Avoid race condition where the order could not be cancelled coz its already filled.
# Simply bailing here is the only safe way - as this order will then be
# handled in the next iteration.
if corder.get('status') not in ('cancelled', 'canceled', 'closed'):
if corder.get('status') not in constants.NON_OPEN_EXCHANGE_STATES:
logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.")
return False
else:
@@ -990,7 +983,7 @@ class FreqtradeBot(LoggingMixin):
# if trade is partially complete, edit the stake details for the trade
# and close the order
# cancel_order may not contain the full order dict, so we need to fallback
# to the order dict aquired before cancelling.
# to the order dict acquired before cancelling.
# we need to fall back to the values from order if corder does not contain these keys.
trade.amount = filled_amount
trade.stake_amount = trade.amount * trade.open_rate
@@ -1001,11 +994,11 @@ class FreqtradeBot(LoggingMixin):
reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}"
self.wallets.update()
self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'],
reason=reason)
self._notify_enter_cancel(trade, order_type=self.strategy.order_types['buy'],
reason=reason)
return was_trade_fully_canceled
def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str:
def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> str:
"""
Sell cancel - cancel order and update trade
:return: Reason for cancel
@@ -1039,14 +1032,14 @@ class FreqtradeBot(LoggingMixin):
reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
self.wallets.update()
self._notify_sell_cancel(
self._notify_exit_cancel(
trade,
order_type=self.strategy.order_types['sell'],
reason=reason
)
return reason
def _safe_sell_amount(self, pair: str, amount: float) -> float:
def _safe_exit_amount(self, pair: str, amount: float) -> float:
"""
Get sellable amount.
Should be trade.amount - but will fall back to the available amount if necessary.
@@ -1071,9 +1064,9 @@ class FreqtradeBot(LoggingMixin):
raise DependencyException(
f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
def execute_trade_exit(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
"""
Executes a limit sell for the given trade and limit
Executes a trade exit for the given trade and limit
:param trade: Trade instance
:param limit: limit rate for the sell order
:param sell_reason: Reason the sell was triggered
@@ -1089,6 +1082,17 @@ class FreqtradeBot(LoggingMixin):
and self.strategy.order_types['stoploss_on_exchange']:
limit = trade.stop_loss
# set custom_exit_price if available
proposed_limit_rate = limit
current_profit = trade.calc_profit_ratio(limit)
custom_exit_price = strategy_safe_wrapper(self.strategy.custom_exit_price,
default_retval=proposed_limit_rate)(
pair=trade.pair, trade=trade,
current_time=datetime.now(timezone.utc),
proposed_rate=proposed_limit_rate, current_profit=current_profit)
limit = self.get_valid_price(custom_exit_price, proposed_limit_rate)
# First cancelling stoploss on exchange ...
if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
try:
@@ -1107,7 +1111,7 @@ class FreqtradeBot(LoggingMixin):
# but we allow this value to be changed)
order_type = self.strategy.order_types.get("forcesell", order_type)
amount = self._safe_sell_amount(trade.pair, trade.amount)
amount = self._safe_exit_amount(trade.pair, trade.amount)
time_in_force = self.strategy.order_time_in_force['sell']
if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
@@ -1119,11 +1123,11 @@ class FreqtradeBot(LoggingMixin):
try:
# Execute sell and update trade record
order = self.exchange.sell(pair=trade.pair,
ordertype=order_type,
amount=amount, rate=limit,
time_in_force=time_in_force
)
order = self.exchange.create_order(pair=trade.pair,
ordertype=order_type, side="sell",
amount=amount, rate=limit,
time_in_force=time_in_force
)
except InsufficientFundsError as e:
logger.warning(f"Unable to place order {e}.")
# Try to figure out what went wrong
@@ -1138,7 +1142,7 @@ class FreqtradeBot(LoggingMixin):
trade.close_rate_requested = limit
trade.sell_reason = sell_reason.sell_reason
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') == 'closed':
if order.get('status', 'unknown') in ('closed', 'expired'):
self.update_trade_state(trade, trade.open_order_id, order)
Trade.commit()
@@ -1146,18 +1150,19 @@ class FreqtradeBot(LoggingMixin):
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
self._notify_sell(trade, order_type)
self._notify_exit(trade, order_type)
return True
def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None:
def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False) -> None:
"""
Sends rpc notification when a sell occurred.
"""
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
# Use cached rates here - it was updated seconds ago.
current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None
current_rate = self.exchange.get_rate(
trade.pair, refresh=False, side="sell") if not fill else None
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"
@@ -1191,7 +1196,7 @@ class FreqtradeBot(LoggingMixin):
# Send the message
self.rpc.send_msg(msg)
def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
"""
Sends rpc notification when a sell cancel occurred.
"""
@@ -1202,7 +1207,7 @@ class FreqtradeBot(LoggingMixin):
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
current_rate = self.exchange.get_sell_rate(trade.pair, False)
current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="sell")
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"
@@ -1212,7 +1217,7 @@ class FreqtradeBot(LoggingMixin):
'exchange': trade.exchange.capitalize(),
'pair': trade.pair,
'gain': gain,
'limit': profit_rate,
'limit': profit_rate or 0,
'order_type': order_type,
'amount': trade.amount,
'open_rate': trade.open_rate,
@@ -1221,7 +1226,7 @@ class FreqtradeBot(LoggingMixin):
'profit_ratio': profit_ratio,
'sell_reason': trade.sell_reason,
'open_date': trade.open_date,
'close_date': trade.close_date,
'close_date': trade.close_date or datetime.now(timezone.utc),
'stake_currency': self.config['stake_currency'],
'fiat_currency': self.config.get('fiat_display_currency', None),
'reason': reason,
@@ -1286,16 +1291,28 @@ class FreqtradeBot(LoggingMixin):
# Updating wallets when order is closed
if not trade.is_open:
if not stoploss_order and not trade.open_order_id:
self._notify_sell(trade, '', True)
self.protections.stop_per_pair(trade.pair)
self.protections.global_stop()
self._notify_exit(trade, '', True)
self.handle_protections(trade.pair)
self.wallets.update()
elif not trade.open_order_id:
# Buy fill
self._notify_buy_fill(trade)
self._notify_enter_fill(trade)
return False
def handle_protections(self, pair: str) -> None:
prot_trig = self.protections.stop_per_pair(pair)
if prot_trig:
msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }
msg.update(prot_trig.to_json())
self.rpc.send_msg(msg)
prot_trig_glb = self.protections.global_stop()
if prot_trig_glb:
msg = {'type': RPCMessageType.PROTECTION_TRIGGER_GLOBAL, }
msg.update(prot_trig_glb.to_json())
self.rpc.send_msg(msg)
def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,
amount: float, fee_abs: float) -> float:
"""
@@ -1376,7 +1393,9 @@ class FreqtradeBot(LoggingMixin):
if fee_currency:
# fee_rate should use mean
fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None
trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
if fee_rate is not None and fee_rate < 0.02:
# Only update if fee-rate is < 2%
trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', ''))
if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC):
logger.warning(f"Amount {amount} does not match amount {trade.amount}")
@@ -1387,3 +1406,26 @@ class FreqtradeBot(LoggingMixin):
amount=amount, fee_abs=fee_abs)
else:
return amount
def get_valid_price(self, custom_price: float, proposed_price: float) -> float:
"""
Return the valid price.
Check if the custom price is of the good type if not return proposed_price
:return: valid price for the order
"""
if custom_price:
try:
valid_custom_price = float(custom_price)
except ValueError:
valid_custom_price = proposed_price
else:
valid_custom_price = proposed_price
cust_p_max_dist_r = self.config.get('custom_price_max_distance_ratio', 0.02)
min_custom_price_allowed = proposed_price - (proposed_price * cust_p_max_dist_r)
max_custom_price_allowed = proposed_price + (proposed_price * cust_p_max_dist_r)
# Bracket between min_custom_price_allowed and max_custom_price_allowed
return max(
min(valid_custom_price, max_custom_price_allowed),
min_custom_price_allowed)

2
freqtrade/loggers.py
View File

@@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None:
# syslog config. The messages should be equal for this.
handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
logging.root.addHandler(handler_sl)
elif s[0] == 'journald':
elif s[0] == 'journald': # pragma: no cover
try:
from systemd.journal import JournaldLogHandler
except ImportError:

8
freqtrade/main.py
View File

@@ -9,7 +9,7 @@ from typing import Any, List
# check min. python version
if sys.version_info < (3, 7):
if sys.version_info < (3, 7): # pragma: no cover
sys.exit("Freqtrade requires Python version >= 3.7")
from freqtrade.commands import Arguments
@@ -44,9 +44,9 @@ def main(sysargv: List[str] = None) -> None:
"as `freqtrade trade [options...]`.\n"
"To see the full list of options available, please use "
"`freqtrade --help` or `freqtrade <command> --help`."
)
)
except SystemExit as e:
except SystemExit as e: # pragma: no cover
return_code = e
except KeyboardInterrupt:
logger.info('SIGINT received, aborting ...')
@@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None:
sys.exit(return_code)
if __name__ == '__main__':
if __name__ == '__main__': # pragma: no cover
main()

15
freqtrade/misc.py
View File

@@ -8,6 +8,7 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Iterator, List
from typing.io import IO
from urllib.parse import urlparse
import rapidjson
@@ -56,6 +57,7 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
"""
Dump JSON data into a file
:param filename: file to create
:param is_zip: if file should be zip
:param data: JSON Data to save
:return:
"""
@@ -213,3 +215,16 @@ def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
"""
for chunk in range(0, len(lst), n):
yield (lst[chunk:chunk + n])
def parse_db_uri_for_logging(uri: str):
"""
Helper method to parse the DB URI and return the same DB URI with the password censored
if it contains it. Otherwise, return the DB URI unchanged
:param uri: DB URI to parse for logging
"""
parsed_db_uri = urlparse(uri)
if not parsed_db_uri.netloc: # No need for censoring as no password was provided
return uri
pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')

236
freqtrade/optimize/backtesting.py
View File

@@ -11,16 +11,17 @@ from typing import Any, Dict, List, Optional, Tuple
from pandas import DataFrame
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.configuration import TimeRange, validate_config_consistency
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
from freqtrade.data.btanalysis import trade_list_to_dataframe
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.converter import trim_dataframe, trim_dataframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import SellType
from freqtrade.enums import BacktestState, SellType
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.mixins import LoggingMixin
from freqtrade.optimize.bt_progress import BTProgress
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_stats)
from freqtrade.persistence import LocalTrade, PairLocks, Trade
@@ -42,6 +43,7 @@ CLOSE_IDX = 3
SELL_IDX = 4
LOW_IDX = 5
HIGH_IDX = 6
BUY_TAG_IDX = 7
class Backtesting:
@@ -57,9 +59,9 @@ class Backtesting:
LoggingMixin.show_output = False
self.config = config
self.results: Optional[Dict[str, Any]] = None
# Reset keys for backtesting
remove_credentials(self.config)
config['dry_run'] = True
self.strategylist: List[IStrategy] = []
self.all_results: Dict[str, Dict] = {}
@@ -83,7 +85,7 @@ class Backtesting:
"configuration or as cli argument `--timeframe 5m`")
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.init_backtest_detail()
self.pairlists = PairListManager(self.exchange, self.config)
if 'VolumePairList' in self.pairlists.name_list:
raise OperationalException("VolumePairList not allowed for backtesting.")
@@ -106,52 +108,79 @@ class Backtesting:
else:
self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0])
Trade.use_db = False
Trade.reset_trades()
PairLocks.timeframe = self.config['timeframe']
PairLocks.use_db = False
PairLocks.reset_locks()
self.wallets = Wallets(self.config, self.exchange, log=False)
self.timerange = TimeRange.parse_timerange(
None if self.config.get('timerange') is None else str(self.config.get('timerange')))
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
# Add maximum startup candle count to configuration for informative pairs support
self.config['startup_candle_count'] = self.required_startup
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
self.init_backtest()
def __del__(self):
self.cleanup()
def cleanup(self):
LoggingMixin.show_output = True
PairLocks.use_db = True
Trade.use_db = True
def init_backtest_detail(self):
# Load detail timeframe if specified
self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
if self.timeframe_detail:
self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
if self.timeframe_min <= self.timeframe_detail_min:
raise OperationalException(
"Detail timeframe must be smaller than strategy timeframe.")
else:
self.timeframe_detail_min = 0
self.detail_data: Dict[str, DataFrame] = {}
def init_backtest(self):
self.prepare_backtest(False)
self.wallets = Wallets(self.config, self.exchange, log=False)
self.progress = BTProgress()
self.abort = False
def _set_strategy(self, strategy: IStrategy):
"""
Load strategy into backtesting
"""
self.strategy: IStrategy = strategy
strategy.dp = self.dataprovider
# Attach Wallets to Strategy baseclass
strategy.wallets = self.wallets
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
def _load_protections(self, strategy: IStrategy):
if self.config.get('enable_protections', False):
conf = self.config
if hasattr(strategy, 'protections'):
conf = deepcopy(conf)
conf['protections'] = strategy.protections
self.protections = ProtectionManager(conf)
self.protections = ProtectionManager(self.config, strategy.protections)
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
"""
Loads backtest data and returns the data combined with the timerange
as tuple.
"""
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
self.progress.init_step(BacktestState.DATALOAD, 1)
data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe,
timerange=timerange,
timerange=self.timerange,
startup_candles=self.required_startup,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
@@ -164,10 +193,28 @@ class Backtesting:
f'({(max_date - min_date).days} days).')
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
self.timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
return data, timerange
self.progress.set_new_value(1)
return data, self.timerange
def load_bt_data_detail(self) -> None:
"""
Loads backtest detail data (smaller timeframe) if necessary.
"""
if self.timeframe_detail:
self.detail_data = history.load_data(
datadir=self.config['datadir'],
pairs=self.pairlists.whitelist,
timeframe=self.timeframe_detail,
timerange=self.timerange,
startup_candles=0,
fail_without_data=True,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
else:
self.detail_data = {}
def prepare_backtest(self, enable_protections):
"""
@@ -180,6 +227,17 @@ class Backtesting:
Trade.reset_trades()
self.rejected_trades = 0
self.dataprovider.clear_cache()
if enable_protections:
self._load_protections(self.strategy)
def check_abort(self):
"""
Check if abort was requested, raise DependencyException if that's the case
Only applies to Interactive backtest mode (webserver mode)
"""
if self.abort:
self.abort = False
raise DependencyException("Stop requested")
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
"""
@@ -189,27 +247,38 @@ class Backtesting:
"""
# Every change to this headers list must evaluate further usages of the resulting tuple
# and eventually change the constants for indexes at the top
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_tag']
data: Dict = {}
self.progress.init_step(BacktestState.CONVERT, len(processed))
# Create dict with data
for pair, pair_data in processed.items():
self.check_abort()
self.progress.increment()
if not pair_data.empty:
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
pair_data.loc[:, 'buy_tag'] = None # cleanup if buy_tag is exist
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair}).copy()
# Trim startup period from analyzed dataframe
df_analyzed = trim_dataframe(df_analyzed, self.timerange,
startup_candles=self.required_startup)
# To avoid using data from future, we use buy/sell signals shifted
# from the previous candle
df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1)
df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1)
df_analyzed.loc[:, 'buy_tag'] = df_analyzed.loc[:, 'buy_tag'].shift(1)
df_analyzed.drop(df_analyzed.head(1).index, inplace=True)
# Update dataprovider cache
self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed)
df_analyzed = df_analyzed.drop(df_analyzed.head(1).index)
# Convert from Pandas to list for performance reasons
# (Looping Pandas is slow.)
data[pair] = df_analyzed.values.tolist()
data[pair] = df_analyzed[headers].values.tolist()
return data
def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple,
@@ -225,6 +294,26 @@ class Backtesting:
# sell at open price.
return sell_row[OPEN_IDX]
# Special case: trailing triggers within same candle as trade opened. Assume most
# pessimistic price movement, which is moving just enough to arm stoploss and
# immediately going down to stop price.
if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0:
if (
not self.strategy.use_custom_stoploss and self.strategy.trailing_stop
and self.strategy.trailing_only_offset_is_reached
and self.strategy.trailing_stop_positive_offset is not None
and self.strategy.trailing_stop_positive
):
# Worst case: price reaches stop_positive_offset and dives down.
stop_rate = (sell_row[OPEN_IDX] *
(1 + abs(self.strategy.trailing_stop_positive_offset) -
abs(self.strategy.trailing_stop_positive)))
else:
# Worst case: price ticks tiny bit above open and dives down.
stop_rate = sell_row[OPEN_IDX] * (1 - abs(trade.stop_loss_pct))
assert stop_rate < sell_row[HIGH_IDX]
return stop_rate
# Set close_rate to stoploss
return trade.stop_loss
elif sell.sell_type == (SellType.ROI):
@@ -258,15 +347,16 @@ class Backtesting:
else:
return sell_row[OPEN_IDX]
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
def _get_sell_trade_entry_for_candle(self, trade: LocalTrade,
sell_row: Tuple) -> Optional[LocalTrade]:
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore
sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
sell_candle_time, sell_row[BUY_IDX],
sell_row[SELL_IDX],
low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
if sell.sell_flag:
trade.close_date = sell_row[DATE_IDX].to_pydatetime()
trade.close_date = sell_candle_time
trade.sell_reason = sell.sell_reason
trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
@@ -278,7 +368,7 @@ class Backtesting:
rate=closerate,
time_in_force=time_in_force,
sell_reason=sell.sell_reason,
current_time=sell_row[DATE_IDX].to_pydatetime()):
current_time=sell_candle_time):
return None
trade.close(closerate, show_msg=False)
@@ -286,12 +376,49 @@ class Backtesting:
return None
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
if self.timeframe_detail and trade.pair in self.detail_data:
sell_candle_time = sell_row[DATE_IDX].to_pydatetime()
sell_candle_end = sell_candle_time + timedelta(minutes=self.timeframe_min)
detail_data = self.detail_data[trade.pair]
detail_data = detail_data.loc[
(detail_data['date'] >= sell_candle_time) &
(detail_data['date'] < sell_candle_end)
].copy()
if len(detail_data) == 0:
# Fall back to "regular" data if no detail data was found for this candle
return self._get_sell_trade_entry_for_candle(trade, sell_row)
detail_data.loc[:, 'buy'] = sell_row[BUY_IDX]
detail_data.loc[:, 'sell'] = sell_row[SELL_IDX]
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high']
for det_row in detail_data[headers].values.tolist():
res = self._get_sell_trade_entry_for_candle(trade, det_row)
if res:
return res
return None
else:
return self._get_sell_trade_entry_for_candle(trade, sell_row)
def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
try:
stake_amount = self.wallets.get_trade_stake_amount(pair, None)
except DependencyException:
return None
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) or 0
max_stake_amount = self.wallets.get_available_stake_amount()
stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
default_retval=stake_amount)(
pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX],
proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount)
stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
if not stake_amount:
return None
order_type = self.strategy.order_types['buy']
time_in_force = self.strategy.order_time_in_force['sell']
@@ -303,6 +430,7 @@ class Backtesting:
if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
# Enter trade
has_buy_tag = len(row) >= BUY_TAG_IDX + 1
trade = LocalTrade(
pair=pair,
open_rate=row[OPEN_IDX],
@@ -312,6 +440,7 @@ class Backtesting:
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
buy_tag=row[BUY_TAG_IDX] if has_buy_tag else None,
exchange='backtesting',
)
return trade
@@ -368,10 +497,6 @@ class Backtesting:
trades: List[LocalTrade] = []
self.prepare_backtest(enable_protections)
# Update dataprovider cache
for pair, dataframe in processed.items():
self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
@@ -383,13 +508,18 @@ class Backtesting:
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
open_trade_count = 0
self.progress.init_step(BacktestState.BACKTEST, int(
(end_date - start_date) / timedelta(minutes=self.timeframe_min)))
# Loop timerange and get candle for each pair at that point in time
while tmp <= end_date:
open_trade_count_start = open_trade_count
self.check_abort()
for i, pair in enumerate(data):
row_index = indexes[pair]
try:
# Row is treated as "current incomplete candle".
# Buy / sell signals are shifted by 1 to compensate for this.
row = data[pair][row_index]
except IndexError:
# missing Data for one pair at the end.
@@ -401,8 +531,8 @@ class Backtesting:
continue
row_index += 1
self.dataprovider._set_dataframe_max_index(row_index)
indexes[pair] = row_index
self.dataprovider._set_dataframe_max_index(row_index)
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
@@ -426,10 +556,10 @@ class Backtesting:
open_trades[pair].append(trade)
LocalTrade.add_bt_trade(trade)
for trade in open_trades[pair]:
for trade in list(open_trades[pair]):
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(trade, row)
# Sell occured
# Sell occurred
if trade_entry:
# logger.debug(f"{pair} - Backtesting sell {trade}")
open_trade_count -= 1
@@ -442,6 +572,7 @@ class Backtesting:
self.protections.global_stop(tmp)
# Move time one configured time_interval ahead.
self.progress.increment()
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
@@ -456,7 +587,10 @@ class Backtesting:
'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
}
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, DataFrame],
timerange: TimeRange):
self.progress.init_step(BacktestState.ANALYZE, 0)
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
backtest_start_time = datetime.now(timezone.utc)
self._set_strategy(strat)
@@ -473,16 +607,18 @@ class Backtesting:
max_open_trades = 0
# need to reprocess data every time to populate signals
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
preprocessed = self.strategy.advise_all_indicators(data)
# Trim startup period from analyzed dataframe
preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
preprocessed_tmp = trim_dataframes(preprocessed, timerange, self.required_startup)
if not preprocessed:
if not preprocessed_tmp:
raise OperationalException(
"No data left after adjusting for startup candles.")
min_date, max_date = history.get_timerange(preprocessed)
# Use preprocessed_tmp for date generation (the trimmed dataframe).
# Backtesting will re-trim the dataframes after buy/sell signal generation.
min_date, max_date = history.get_timerange(preprocessed_tmp)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days).')
@@ -512,16 +648,18 @@ class Backtesting:
data: Dict[str, Any] = {}
data, timerange = self.load_bt_data()
self.load_bt_data_detail()
logger.info("Dataload complete. Calculating indicators")
for strat in self.strategylist:
min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
if len(self.strategylist) > 0:
stats = generate_backtest_stats(data, self.all_results,
min_date=min_date, max_date=max_date)
if self.config.get('export', False):
store_backtest_stats(self.config['exportfilename'], stats)
self.results = generate_backtest_stats(data, self.all_results,
min_date=min_date, max_date=max_date)
if self.config.get('export', 'none') == 'trades':
store_backtest_stats(self.config['exportfilename'], self.results)
# Show backtest results
show_backtest_results(self.config, stats)
show_backtest_results(self.config, self.results)

33
freqtrade/optimize/bt_progress.py Normal file
View File

@@ -0,0 +1,33 @@
from freqtrade.enums import BacktestState
class BTProgress:
_action: BacktestState = BacktestState.STARTUP
_progress: float = 0
_max_steps: float = 0
def __init__(self):
pass
def init_step(self, action: BacktestState, max_steps: float):
self._action = action
self._max_steps = max_steps
self._proress = 0
def set_new_value(self, new_value: float):
self._progress = new_value
def increment(self):
self._progress += 1
@property
def progress(self):
"""
Get progress as ratio, capped to be between 0 and 1 (to avoid small calculation errors).
"""
return max(min(round(self._progress / self._max_steps, 5)
if self._max_steps > 0 else 0, 1), 0)
@property
def action(self):
return str(self._action)

8
freqtrade/optimize/edge_cli.py
View File

@@ -7,7 +7,8 @@ import logging
from typing import Any, Dict
from freqtrade import constants
from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency
from freqtrade.configuration import TimeRange, validate_config_consistency
from freqtrade.data.dataprovider import DataProvider
from freqtrade.edge import Edge
from freqtrade.optimize.optimize_reports import generate_edge_table
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -28,11 +29,12 @@ class EdgeCli:
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
# Reset keys for edge
remove_credentials(self.config)
# Ensure using dry-run
self.config['dry_run'] = True
self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
self.strategy = StrategyResolver.load_strategy(self.config)
self.strategy.dp = DataProvider(config, None)
validate_config_consistency(self.config)

138
freqtrade/optimize/hyperopt.py
View File

@@ -12,7 +12,6 @@ from math import ceil
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import progressbar
import rapidjson
from colorama import Fore, Style
@@ -20,18 +19,19 @@ from colorama import init as colorama_init
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.misc import file_dump_json, plural
from freqtrade.exceptions import OperationalException
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_auto import HyperOptAuto
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.optimize.hyperopt_tools import HyperoptTools
from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
from freqtrade.optimize.optimize_reports import generate_strategy_stats
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver
# Suppress scikit-learn FutureWarnings from skopt
@@ -67,6 +67,7 @@ class Hyperopt:
def __init__(self, config: Dict[str, Any]) -> None:
self.buy_space: List[Dimension] = []
self.sell_space: List[Dimension] = []
self.protection_space: List[Dimension] = []
self.roi_space: List[Dimension] = []
self.stoploss_space: List[Dimension] = []
self.trailing_space: List[Dimension] = []
@@ -79,7 +80,10 @@ class Hyperopt:
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
else:
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
raise OperationalException(
"Using separate Hyperopt files has been removed in 2021.9. Please convert "
"your existing Hyperopt file to the new Hyperoptable strategy interface")
self.backtesting._set_strategy(self.backtesting.strategylist[0])
self.custom_hyperopt.strategy = self.backtesting.strategy
@@ -100,17 +104,6 @@ class Hyperopt:
self.num_epochs_saved = 0
self.current_best_epoch: Optional[Dict[str, Any]] = None
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
self.backtesting.strategy.advise_indicators = ( # type: ignore
self.custom_hyperopt.populate_indicators) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_buy_trend'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.populate_buy_trend) # type: ignore
if hasattr(self.custom_hyperopt, 'populate_sell_trend'):
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.populate_sell_trend) # type: ignore
# Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
self.max_open_trades = self.config['max_open_trades']
@@ -163,13 +156,9 @@ class Hyperopt:
While not a valid json object - this allows appending easily.
:param epoch: result dictionary for this epoch.
"""
def default_parser(x):
if isinstance(x, np.integer):
return int(x)
return str(x)
epoch[FTHYPT_FILEVERSION] = 2
with self.results_file.open('a') as f:
rapidjson.dump(epoch, f, default=default_parser,
rapidjson.dump(epoch, f, default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
f.write("\n")
@@ -191,6 +180,8 @@ class Hyperopt:
result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
if HyperoptTools.has_space(self.config, 'sell'):
result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
if HyperoptTools.has_space(self.config, 'protection'):
result['protection'] = {p.name: params.get(p.name) for p in self.protection_space}
if HyperoptTools.has_space(self.config, 'roi'):
result['roi'] = {str(k): v for k, v in
self.custom_hyperopt.generate_roi_table(params).items()}
@@ -201,6 +192,25 @@ class Hyperopt:
return result
def _get_no_optimize_details(self) -> Dict[str, Any]:
"""
Get non-optimized parameters
"""
result: Dict[str, Any] = {}
strategy = self.backtesting.strategy
if not HyperoptTools.has_space(self.config, 'roi'):
result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()}
if not HyperoptTools.has_space(self.config, 'stoploss'):
result['stoploss'] = {'stoploss': strategy.stoploss}
if not HyperoptTools.has_space(self.config, 'trailing'):
result['trailing'] = {
'trailing_stop': strategy.trailing_stop,
'trailing_stop_positive': strategy.trailing_stop_positive,
'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset,
'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached,
}
return result
def print_results(self, results) -> None:
"""
Log results if it is better than any previous evaluation
@@ -222,10 +232,16 @@ class Hyperopt:
"""
Assign the dimensions in the hyperoptimization space.
"""
if HyperoptTools.has_space(self.config, 'protection'):
# Protections can only be optimized when using the Parameter interface
logger.debug("Hyperopt has 'protection' space")
# Enable Protections if protection space is selected.
self.config['enable_protections'] = True
self.protection_space = self.custom_hyperopt.protection_space()
if HyperoptTools.has_space(self.config, 'buy'):
logger.debug("Hyperopt has 'buy' space")
self.buy_space = self.custom_hyperopt.indicator_space()
self.buy_space = self.custom_hyperopt.buy_indicator_space()
if HyperoptTools.has_space(self.config, 'sell'):
logger.debug("Hyperopt has 'sell' space")
@@ -242,30 +258,42 @@ class Hyperopt:
if HyperoptTools.has_space(self.config, 'trailing'):
logger.debug("Hyperopt has 'trailing' space")
self.trailing_space = self.custom_hyperopt.trailing_space()
self.dimensions = (self.buy_space + self.sell_space + self.roi_space +
self.stoploss_space + self.trailing_space)
self.dimensions = (self.buy_space + self.sell_space + self.protection_space
+ self.roi_space + self.stoploss_space + self.trailing_space)
def assign_params(self, params_dict: Dict, category: str) -> None:
"""
Assign hyperoptable parameters
"""
for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params_dict[attr_name]
def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
"""
Used Optimize function. Called once per epoch to optimize whatever is configured.
Used Optimize function.
Called once per epoch to optimize whatever is configured.
Keep this function as optimized as possible!
"""
backtest_start_time = datetime.now(timezone.utc)
params_dict = self._get_params_dict(self.dimensions, raw_params)
# Apply parameters
if HyperoptTools.has_space(self.config, 'buy'):
self.assign_params(params_dict, 'buy')
if HyperoptTools.has_space(self.config, 'sell'):
self.assign_params(params_dict, 'sell')
if HyperoptTools.has_space(self.config, 'protection'):
self.assign_params(params_dict, 'protection')
if HyperoptTools.has_space(self.config, 'roi'):
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.custom_hyperopt.generate_roi_table(params_dict))
if HyperoptTools.has_space(self.config, 'buy'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.buy_strategy_generator(params_dict))
if HyperoptTools.has_space(self.config, 'sell'):
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.sell_strategy_generator(params_dict))
if HyperoptTools.has_space(self.config, 'stoploss'):
self.backtesting.strategy.stoploss = params_dict['stoploss']
@@ -310,7 +338,8 @@ class Hyperopt:
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
not_optimized = self.backtesting.strategy.get_params_dict()
not_optimized = self.backtesting.strategy.get_no_optimize_params()
not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details())
trade_count = strat_stats['total_trades']
total_profit = strat_stats['profit_total']
@@ -324,7 +353,8 @@ class Hyperopt:
loss = self.calculate_loss(results=backtesting_results['results'],
trade_count=trade_count,
min_date=min_date, max_date=max_date,
config=self.config, processed=processed)
config=self.config, processed=processed,
backtest_stats=strat_stats)
return {
'loss': loss,
'params_dict': params_dict,
@@ -336,10 +366,20 @@ class Hyperopt:
}
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
estimator = self.custom_hyperopt.generate_estimator()
acq_optimizer = "sampling"
if isinstance(estimator, str):
if estimator not in ("GP", "RF", "ET", "GBRT"):
raise OperationalException(f"Estimator {estimator} not supported.")
else:
acq_optimizer = "auto"
logger.info(f"Using estimator {estimator}.")
return Optimizer(
dimensions,
base_estimator="ET",
acq_optimizer="auto",
base_estimator=estimator,
acq_optimizer=acq_optimizer,
n_initial_points=INITIAL_POINTS,
acq_optimizer_kwargs={'n_jobs': cpu_count},
random_state=self.random_state,
@@ -357,18 +397,17 @@ class Hyperopt:
data, timerange = self.backtesting.load_bt_data()
logger.info("Dataload complete. Calculating indicators")
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
preprocessed = self.backtesting.strategy.advise_all_indicators(data)
# Trim startup period from analyzed dataframe
# Trim startup period from analyzed dataframe to get correct dates for output.
processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
self.min_date, self.max_date = get_timerange(processed)
logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(self.max_date - self.min_date).days} days)..')
dump(processed, self.data_pickle_file)
# Store non-trimmed data - will be trimmed after signal generation.
dump(preprocessed, self.data_pickle_file)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
@@ -423,9 +462,9 @@ class Hyperopt:
' [', progressbar.ETA(), ', ', progressbar.Timer(), ']',
]
with progressbar.ProgressBar(
max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
widgets=widgets
) as pbar:
max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False,
widgets=widgets
) as pbar:
EVALS = ceil(self.total_epochs / jobs)
for i in range(EVALS):
# Correct the number of epochs to be processed for the last
@@ -469,6 +508,11 @@ class Hyperopt:
f"saved to '{self.results_file}'.")
if self.current_best_epoch:
HyperoptTools.try_export_params(
self.config,
self.backtesting.strategy.get_strategy_name(),
self.current_best_epoch)
HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
self.print_json)
else:

62
freqtrade/optimize/hyperopt_auto.py
View File

@@ -3,16 +3,32 @@ HyperOptAuto class.
This module implements a convenience auto-hyperopt class, which can be used together with strategies
that implement IHyperStrategy interface.
"""
import logging
from contextlib import suppress
from typing import Any, Callable, Dict, List
from typing import Callable, Dict, List
from pandas import DataFrame
from freqtrade.exceptions import OperationalException
with suppress(ImportError):
from skopt.space import Dimension
from freqtrade.optimize.hyperopt_interface import IHyperOpt
from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
logger = logging.getLogger(__name__)
def _format_exception_message(space: str, ignore_missing_space: bool) -> None:
msg = (f"The '{space}' space is included into the hyperoptimization "
f"but no parameter for this space was not found in your Strategy. "
)
if ignore_missing_space:
logger.warning(msg + "This space will be ignored.")
else:
raise OperationalException(
msg + f"Please make sure to have parameters for this space enabled for optimization "
f"or remove the '{space}' space from hyperoptimization.")
class HyperOptAuto(IHyperOpt):
@@ -22,26 +38,6 @@ class HyperOptAuto(IHyperOpt):
sell_indicator_space methods, but other hyperopt methods can be overridden as well.
"""
def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def populate_buy_trend(dataframe: DataFrame, metadata: dict):
for attr_name, attr in self.strategy.enumerate_parameters('buy'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params[attr_name]
return self.strategy.populate_buy_trend(dataframe, metadata)
return populate_buy_trend
def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def populate_sell_trend(dataframe: DataFrame, metadata: dict):
for attr_name, attr in self.strategy.enumerate_parameters('sell'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params[attr_name]
return self.strategy.populate_sell_trend(dataframe, metadata)
return populate_sell_trend
def _get_func(self, name) -> Callable:
"""
Return a function defined in Strategy.HyperOpt class, or one defined in super() class.
@@ -60,18 +56,25 @@ class HyperOptAuto(IHyperOpt):
if attr.optimize:
yield attr.get_space(attr_name)
def _get_indicator_space(self, category, fallback_method_name):
def _get_indicator_space(self, category) -> List:
# TODO: is this necessary, or can we call "generate_space" directly?
indicator_space = list(self._generate_indicator_space(category))
if len(indicator_space) > 0:
return indicator_space
else:
return self._get_func(fallback_method_name)()
_format_exception_message(
category,
self.config.get("hyperopt_ignore_missing_space", False))
return []
def indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy', 'indicator_space')
def buy_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy')
def sell_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('sell', 'sell_indicator_space')
return self._get_indicator_space('sell')
def protection_space(self) -> List['Dimension']:
return self._get_indicator_space('protection')
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
return self._get_func('generate_roi_table')(params)
@@ -87,3 +90,6 @@ class HyperOptAuto(IHyperOpt):
def trailing_space(self) -> List['Dimension']:
return self._get_func('trailing_space')()
def generate_estimator(self) -> EstimatorType:
return self._get_func('generate_estimator')()

128
freqtrade/optimize/hyperopt_epoch_filters.py Normal file
View File

@@ -0,0 +1,128 @@
import logging
from typing import List
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
def hyperopt_filter_epochs(epochs: List, filteroptions: dict, log: bool = True) -> List:
"""
Filter our items from the list of hyperopt results
"""
if filteroptions['only_best']:
epochs = [x for x in epochs if x['is_best']]
if filteroptions['only_profitable']:
epochs = [x for x in epochs
if x['results_metrics'].get('profit_total', 0) > 0]
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions)
epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions)
if log:
logger.info(f"{len(epochs)} " +
("best " if filteroptions['only_best'] else "") +
("profitable " if filteroptions['only_profitable'] else "") +
"epochs found.")
return epochs
def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
"""
Filter epochs with trade-counts > trades
"""
return [
x for x in epochs if x['results_metrics'].get('total_trades', 0) > trade_count
]
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_trades'] > 0:
epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
if filteroptions['filter_max_trades'] > 0:
epochs = [
x for x in epochs
if x['results_metrics'].get('total_trades') < filteroptions['filter_max_trades']
]
return epochs
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
def get_duration_value(x):
# Duration in minutes ...
if 'holding_avg_s' in x['results_metrics']:
avg = x['results_metrics']['holding_avg_s']
return avg // 60
raise OperationalException(
"Holding-average not available. Please omit the filter on average time, "
"or rerun hyperopt with this version")
if filteroptions['filter_min_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) > filteroptions['filter_min_avg_time']
]
if filteroptions['filter_max_avg_time'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if get_duration_value(x) < filteroptions['filter_max_avg_time']
]
return epochs
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_mean', 0) * 100
> filteroptions['filter_min_avg_profit']
]
if filteroptions['filter_max_avg_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_mean', 0) * 100
< filteroptions['filter_max_avg_profit']
]
if filteroptions['filter_min_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_total_abs', 0)
> filteroptions['filter_min_total_profit']
]
if filteroptions['filter_max_total_profit'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics'].get('profit_total_abs', 0)
< filteroptions['filter_max_total_profit']
]
return epochs
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
if filteroptions['filter_max_objective'] is not None:
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]
return epochs

37
freqtrade/optimize/hyperopt_interface.py
View File

@@ -5,11 +5,11 @@ This module defines the interface to apply for hyperopt
import logging
import math
from abc import ABC
from typing import Any, Callable, Dict, List
from typing import Dict, List, Union
from sklearn.base import RegressorMixin
from skopt.space import Categorical, Dimension, Integer
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import round_dict
from freqtrade.optimize.space import SKDecimal
@@ -18,12 +18,7 @@ from freqtrade.strategy import IStrategy
logger = logging.getLogger(__name__)
def _format_exception_message(method: str, space: str) -> str:
return (f"The '{space}' space is included into the hyperoptimization "
f"but {method}() method is not found in your "
f"custom Hyperopt class. You should either implement this "
f"method or remove the '{space}' space from hyperoptimization.")
EstimatorType = Union[RegressorMixin, str]
class IHyperOpt(ABC):
@@ -45,29 +40,13 @@ class IHyperOpt(ABC):
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
IHyperOpt.timeframe = str(config['timeframe'])
def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def generate_estimator(self) -> EstimatorType:
"""
Create a buy strategy generator.
Return base_estimator.
Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
inheriting from RegressorMixin (from sklearn).
"""
raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy'))
def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
"""
Create a sell strategy generator.
"""
raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
def indicator_space(self) -> List[Dimension]:
"""
Create an indicator space.
"""
raise OperationalException(_format_exception_message('indicator_space', 'buy'))
def sell_indicator_space(self) -> List[Dimension]:
"""
Create a sell indicator space.
"""
raise OperationalException(_format_exception_message('sell_indicator_space', 'sell'))
return 'ET'
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
"""

64
freqtrade/optimize/hyperopt_loss_calmar.py Normal file
View File

@@ -0,0 +1,64 @@
"""
CalmarHyperOptLoss
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
from datetime import datetime
from math import sqrt as msqrt
from typing import Any, Dict
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss
class CalmarHyperOptLoss(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation uses the Calmar Ratio calculation.
"""
@staticmethod
def hyperopt_loss_function(
results: DataFrame,
trade_count: int,
min_date: datetime,
max_date: datetime,
config: Dict,
processed: Dict[str, DataFrame],
backtest_stats: Dict[str, Any],
*args,
**kwargs
) -> float:
"""
Objective function, returns smaller number for more optimal results.
Uses Calmar Ratio calculation.
"""
total_profit = backtest_stats["profit_total"]
days_period = (max_date - min_date).days
# adding slippage of 0.1% per trade
total_profit = total_profit - 0.0005
expected_returns_mean = total_profit.sum() / days_period * 100
# calculate max drawdown
try:
_, _, _, high_val, low_val = calculate_max_drawdown(
results, value_col="profit_abs"
)
max_drawdown = (high_val - low_val) / high_val
except ValueError:
max_drawdown = 0
if max_drawdown != 0:
calmar_ratio = expected_returns_mean / max_drawdown * msqrt(365)
else:
# Define high (negative) calmar ratio to be clear that this is NOT optimal.
calmar_ratio = -20.0
# print(expected_returns_mean, max_drawdown, calmar_ratio)
return -calmar_ratio

3
freqtrade/optimize/hyperopt_loss_interface.py
View File

@@ -5,7 +5,7 @@ This module defines the interface for the loss-function for hyperopt
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Dict
from typing import Any, Dict
from pandas import DataFrame
@@ -22,6 +22,7 @@ class IHyperOptLoss(ABC):
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
config: Dict, processed: Dict[str, DataFrame],
backtest_stats: Dict[str, Any],
*args, **kwargs) -> float:
"""
Objective function, returns smaller number for better results

41
freqtrade/optimize/hyperopt_loss_max_drawdown.py Normal file
View File

@@ -0,0 +1,41 @@
"""
MaxDrawDownHyperOptLoss
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
from datetime import datetime
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss
class MaxDrawDownHyperOptLoss(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation optimizes for max draw down and profit
Less max drawdown more profit -> Lower return value
"""
@staticmethod
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
*args, **kwargs) -> float:
"""
Objective function.
Uses profit ratio weighted max_drawdown when drawdown is available.
Otherwise directly optimizes profit ratio.
"""
total_profit = results['profit_abs'].sum()
try:
max_drawdown = calculate_max_drawdown(results, value_col='profit_abs')
except ValueError:
# No losing trade, therefore no drawdown.
return -total_profit
return -total_profit / max_drawdown[0]

0
freqtrade/optimize/default_hyperopt_loss.py → freqtrade/optimize/hyperopt_loss_short_trade_dur.py
View File

250
freqtrade/optimize/hyperopt_tools.py Normal file → Executable file
View File

@@ -1,75 +1,160 @@
import io
import logging
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, Iterator, List, Optional, Tuple
import numpy as np
import pandas as pd
import rapidjson
import tabulate
from colorama import Fore, Style
from pandas import isna, json_normalize
from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.misc import round_coin_value, round_dict
from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
from freqtrade.optimize.hyperopt_epoch_filters import hyperopt_filter_epochs
logger = logging.getLogger(__name__)
NON_OPT_PARAM_APPENDIX = " # value loaded from strategy"
def hyperopt_serializer(x):
if isinstance(x, np.integer):
return int(x)
if isinstance(x, np.bool_):
return bool(x)
return str(x)
class HyperoptTools():
@staticmethod
def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]:
"""
Get Strategy-location (filename) from strategy_name
"""
from freqtrade.resolvers.strategy_resolver import StrategyResolver
directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
strategy_objs = StrategyResolver.search_all_objects(directory, False)
strategies = [s for s in strategy_objs if s['name'] == strategy_name]
if strategies:
strategy = strategies[0]
return Path(strategy['location'])
return None
@staticmethod
def export_params(params, strategy_name: str, filename: Path):
"""
Generate files
"""
final_params = deepcopy(params['params_not_optimized'])
final_params = deep_merge_dicts(params['params_details'], final_params)
final_params = {
'strategy_name': strategy_name,
'params': final_params,
'ft_stratparam_v': 1,
'export_time': datetime.now(timezone.utc),
}
logger.info(f"Dumping parameters to {filename}")
with filename.open('w') as f:
rapidjson.dump(final_params, f, indent=2,
default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
)
@staticmethod
def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict):
if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False):
# Export parameters ...
fn = HyperoptTools.get_strategy_filename(config, strategy_name)
if fn:
HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json'))
else:
logger.warning("Strategy not found, not exporting parameter file.")
@staticmethod
def has_space(config: Dict[str, Any], space: str) -> bool:
"""
Tell if the space value is contained in the configuration
"""
# The 'trailing' space is not included in the 'default' set of spaces
if space == 'trailing':
# 'trailing' and 'protection spaces are not included in the 'default' set of spaces
if space in ('trailing', 'protection'):
return any(s in config['spaces'] for s in [space, 'all'])
else:
return any(s in config['spaces'] for s in [space, 'all', 'default'])
@staticmethod
def _read_results_pickle(results_file: Path) -> List:
def _read_results(results_file: Path, batch_size: int = 10) -> Iterator[List[Any]]:
"""
Read hyperopt results from pickle file
LEGACY method - new files are written as json and cannot be read with this method.
"""
from joblib import load
logger.info(f"Reading pickled epochs from '{results_file}'")
data = load(results_file)
return data
@staticmethod
def _read_results(results_file: Path) -> List:
"""
Read hyperopt results from file
Stream hyperopt results from file
"""
import rapidjson
logger.info(f"Reading epochs from '{results_file}'")
with results_file.open('r') as f:
data = [rapidjson.loads(line) for line in f]
return data
data = []
for line in f:
data += [rapidjson.loads(line)]
if len(data) >= batch_size:
yield data
data = []
yield data
@staticmethod
def load_previous_results(results_file: Path) -> List:
"""
Load data for epochs from the file if we have one
"""
epochs: List = []
def _test_hyperopt_results_exist(results_file) -> bool:
if results_file.is_file() and results_file.stat().st_size > 0:
if results_file.suffix == '.pickle':
epochs = HyperoptTools._read_results_pickle(results_file)
else:
epochs = HyperoptTools._read_results(results_file)
# Detection of some old format, without 'is_best' field saved
if epochs[0].get('is_best') is None:
raise OperationalException(
"Legacy hyperopt results are no longer supported."
"Please rerun hyperopt or use an older version to load this file."
)
return True
else:
# No file found.
return False
@staticmethod
def load_filtered_results(results_file: Path, config: Dict[str, Any]) -> Tuple[List, int]:
filteroptions = {
'only_best': config.get('hyperopt_list_best', False),
'only_profitable': config.get('hyperopt_list_profitable', False),
'filter_min_trades': config.get('hyperopt_list_min_trades', 0),
'filter_max_trades': config.get('hyperopt_list_max_trades', 0),
'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None),
'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None),
'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None),
'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None),
'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None),
'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None),
'filter_min_objective': config.get('hyperopt_list_min_objective', None),
'filter_max_objective': config.get('hyperopt_list_max_objective', None),
}
if not HyperoptTools._test_hyperopt_results_exist(results_file):
# No file found.
return [], 0
epochs = []
total_epochs = 0
for epochs_tmp in HyperoptTools._read_results(results_file):
if total_epochs == 0 and epochs_tmp[0].get('is_best') is None:
raise OperationalException(
"The file with HyperoptTools results is incompatible with this version "
"of Freqtrade and cannot be loaded.")
logger.info(f"Loaded {len(epochs)} previous evaluations from disk.")
return epochs
total_epochs += len(epochs_tmp)
epochs += hyperopt_filter_epochs(epochs_tmp, filteroptions, log=False)
logger.info(f"Loaded {total_epochs} previous evaluations from disk.")
# Final filter run ...
epochs = hyperopt_filter_epochs(epochs, filteroptions, log=True)
return epochs, total_epochs
@staticmethod
def show_epoch_details(results, total_epochs: int, print_json: bool,
@@ -90,8 +175,8 @@ class HyperoptTools():
if print_json:
result_dict: Dict = {}
for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']:
HyperoptTools._params_update_for_json(result_dict, params, s)
for s in ['buy', 'sell', 'protection', 'roi', 'stoploss', 'trailing']:
HyperoptTools._params_update_for_json(result_dict, params, non_optimized, s)
print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
else:
@@ -99,44 +184,64 @@ class HyperoptTools():
non_optimized)
HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
non_optimized)
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
HyperoptTools._params_pretty_print(params, 'protection',
"Protection hyperspace params:", non_optimized)
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized)
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized)
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized)
@staticmethod
def _params_update_for_json(result_dict, params, space: str) -> None:
if space in params:
def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None:
if (space in params) or (space in non_optimized):
space_params = HyperoptTools._space_params(params, space)
space_non_optimized = HyperoptTools._space_params(non_optimized, space)
all_space_params = space_params
# Merge non optimized params if there are any
if len(space_non_optimized) > 0:
all_space_params = {**space_params, **space_non_optimized}
if space in ['buy', 'sell']:
result_dict.setdefault('params', {}).update(space_params)
result_dict.setdefault('params', {}).update(all_space_params)
elif space == 'roi':
# Convert keys in min_roi dict to strings because
# rapidjson cannot dump dicts with integer keys...
result_dict['minimal_roi'] = {str(k): v for k, v in space_params.items()}
result_dict['minimal_roi'] = {str(k): v for k, v in all_space_params.items()}
else: # 'stoploss', 'trailing'
result_dict.update(space_params)
result_dict.update(all_space_params)
@staticmethod
def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
if space in params or space in non_optimized:
space_params = HyperoptTools._space_params(params, space, 5)
no_params = HyperoptTools._space_params(non_optimized, space, 5)
appendix = ''
if not space_params and not no_params:
# No parameters - don't print
return
if not space_params:
# Not optimized parameters - append string
appendix = NON_OPT_PARAM_APPENDIX
result = f"\n# {header}\n"
if space == 'stoploss':
result += f"stoploss = {space_params.get('stoploss')}"
elif space == 'roi':
if space == "stoploss":
stoploss = safe_value_fallback2(space_params, no_params, space, space)
result += (f"stoploss = {stoploss}{appendix}")
elif space == "roi":
result = result[:-1] + f'{appendix}\n'
minimal_roi_result = rapidjson.dumps({
str(k): v for k, v in space_params.items()
str(k): v for k, v in (space_params or no_params).items()
}, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
result += f"minimal_roi = {minimal_roi_result}"
elif space == 'trailing':
for k, v in space_params.items():
result += f'{k} = {v}\n'
elif space == "trailing":
for k, v in (space_params or no_params).items():
result += f"{k} = {v}{appendix}\n"
else:
no_params = HyperoptTools._space_params(non_optimized, space, 5)
# Buy / sell parameters
result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}"
result = result.replace("\n", "\n ")
print(result)
@@ -150,7 +255,7 @@ class HyperoptTools():
return {}
@staticmethod
def _pprint(params, non_optimized, indent: int = 4):
def _pprint_dict(params, non_optimized, indent: int = 4):
"""
Pretty-print hyperopt results (based on 2 dicts - with add. comment)
"""
@@ -162,7 +267,7 @@ class HyperoptTools():
result += " " * indent + f'"{k}": '
result += f'"{param}",' if isinstance(param, str) else f'{param},'
if k in non_optimized:
result += " # value loaded from strategy"
result += NON_OPT_PARAM_APPENDIX
result += "\n"
result += '}'
return result
@@ -194,8 +299,8 @@ class HyperoptTools():
f"Objective: {results['loss']:.5f}")
@staticmethod
def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str:
def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool,
has_drawdown: bool) -> pd.DataFrame:
trials['Best'] = ''
if 'results_metrics.winsdrawslosses' not in trials.columns:
@@ -331,8 +436,7 @@ class HyperoptTools():
return table
@staticmethod
def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool,
csv_file: str) -> None:
def export_csv_file(config: dict, results: list, csv_file: str) -> None:
"""
Log result to csv-file
"""
@@ -354,21 +458,14 @@ class HyperoptTools():
trials['Best'] = ''
trials['Stake currency'] = config['stake_currency']
if 'results_metrics.total_trades' in trials:
base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.profit_mean', 'results_metrics.profit_median',
'results_metrics.profit_total',
'Stake currency',
'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
'loss', 'is_initial_point', 'is_best']
perc_multi = 100
else:
perc_multi = 1
base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.median_profit',
'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']
base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.profit_mean', 'results_metrics.profit_median',
'results_metrics.profit_total',
'Stake currency',
'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
'loss', 'is_initial_point', 'is_best']
perc_multi = 100
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
trials = trials[base_metrics + param_metrics]
@@ -396,11 +493,6 @@ class HyperoptTools():
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
)
if perc_multi == 1:
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: f'{x:,.1f} m' if isinstance(
x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
)
trials['Objective'] = trials['Objective'].apply(
lambda x: f'{x:,.5f}' if x != 100000 else ""
)

135
freqtrade/optimize/optimize_reports.py
View File

@@ -4,7 +4,7 @@ from pathlib import Path
from typing import Any, Dict, List, Union
from numpy import int64
from pandas import DataFrame
from pandas import DataFrame, to_datetime
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
@@ -21,7 +21,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
Stores backtest results
:param recordfilename: Path object, which can either be a filename or a directory.
Filenames will be appended with a timestamp right before the suffix
while for diectories, <directory>/backtest-result-<datetime>.json will be used as filename
while for directories, <directory>/backtest-result-<datetime>.json will be used as filename
:param stats: Dataframe containing the backtesting statistics
"""
if recordfilename.is_dir():
@@ -31,7 +31,7 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
filename = Path.joinpath(
recordfilename.parent,
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
).with_suffix(recordfilename.suffix)
).with_suffix(recordfilename.suffix)
file_dump_json(filename, stats)
latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
@@ -173,7 +173,7 @@ def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
for strategy, results in all_results.items():
tabular_data.append(_generate_result_line(
results['results'], results['config']['dry_run_wallet'], strategy)
)
)
try:
max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_ratio')
@@ -189,7 +189,6 @@ def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
def generate_edge_table(results: dict) -> str:
floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', 'd', 'd')
tabular_data = []
headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio',
@@ -214,6 +213,41 @@ def generate_edge_table(results: dict) -> str:
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def _get_resample_from_period(period: str) -> str:
if period == 'day':
return '1d'
if period == 'week':
return '1w'
if period == 'month':
return '1M'
raise ValueError(f"Period {period} is not supported.")
def generate_periodic_breakdown_stats(trade_list: List, period: str) -> List[Dict[str, Any]]:
results = DataFrame.from_records(trade_list)
if len(results) == 0:
return []
results['close_date'] = to_datetime(results['close_date'], utc=True)
resample_period = _get_resample_from_period(period)
resampled = results.resample(resample_period, on='close_date')
stats = []
for name, day in resampled:
profit_abs = day['profit_abs'].sum().round(10)
wins = sum(day['profit_abs'] > 0)
draws = sum(day['profit_abs'] == 0)
loses = sum(day['profit_abs'] < 0)
stats.append(
{
'date': name.strftime('%d/%m/%Y'),
'profit_abs': profit_abs,
'wins': wins,
'draws': draws,
'loses': loses
}
)
return stats
def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
""" Generate overall trade statistics """
if len(results) == 0:
@@ -229,8 +263,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
winning_trades = results.loc[results['profit_ratio'] > 0]
draw_trades = results.loc[results['profit_ratio'] == 0]
losing_trades = results.loc[results['profit_ratio'] < 0]
zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) &
(results['sell_reason'] == 'trailing_stop_loss')])
holding_avg = (timedelta(minutes=round(results['trade_duration'].mean()))
if not results.empty else timedelta())
@@ -249,7 +281,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
'winner_holding_avg_s': winner_holding_avg.total_seconds(),
'loser_holding_avg': loser_holding_avg,
'loser_holding_avg_s': loser_holding_avg.total_seconds(),
'zero_duration_trades': zero_duration_trades,
}
@@ -264,6 +295,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
'winning_days': 0,
'draw_days': 0,
'losing_days': 0,
'daily_profit_list': [],
}
daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
@@ -274,6 +306,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
winning_days = sum(daily_profit > 0)
draw_days = sum(daily_profit == 0)
losing_days = sum(daily_profit < 0)
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
return {
'backtest_best_day': best_rel,
@@ -283,6 +316,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
'winning_days': winning_days,
'draw_days': draw_days,
'losing_days': losing_days,
'daily_profit': daily_profit_list,
}
@@ -300,7 +334,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
:param min_date: Backtest start date
:param max_date: Backtest end date
:param market_change: float indicating the market change
:return: Dictionary containing results per strategy and a stratgy summary.
:return: Dictionary containing results per strategy and a strategy summary.
"""
results: Dict[str, DataFrame] = content['results']
if not isinstance(results, DataFrame):
@@ -325,10 +359,11 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
if not results.empty:
results['open_timestamp'] = results['open_date'].view(int64) // 1e6
results['close_timestamp'] = results['close_date'].view(int64) // 1e6
backtest_days = (max_date - min_date).days
backtest_days = (max_date - min_date).days or 1
strat_stats = {
'trades': results.to_dict(orient='records'),
'locks': [lock.to_json() for lock in content['locks']],
@@ -337,6 +372,8 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'results_per_pair': pair_results,
'sell_reason_summary': sell_reason_stats,
'left_open_trades': left_open_results,
# 'days_breakdown_stats': days_breakdown_stats,
'total_trades': len(results),
'total_volume': float(results['stake_amount'].sum()),
'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
@@ -353,7 +390,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'backtest_run_start_ts': content['backtest_start_time'],
'backtest_run_end_ts': content['backtest_end_time'],
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
'trades_per_day': round(len(results) / backtest_days, 2),
'market_change': market_change,
'pairlist': list(btdata.keys()),
'stake_amount': config['stake_amount'],
@@ -367,6 +404,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'max_open_trades_setting': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
'timeframe_detail': config.get('timeframe_detail', ''),
'timerange': config.get('timerange', ''),
'enable_protections': config.get('enable_protections', False),
'strategy_name': strategy,
@@ -378,10 +416,10 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
'use_custom_stoploss': config.get('use_custom_stoploss', False),
'minimal_roi': config['minimal_roi'],
'use_sell_signal': config['ask_strategy']['use_sell_signal'],
'sell_profit_only': config['ask_strategy']['sell_profit_only'],
'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
'use_sell_signal': config['use_sell_signal'],
'sell_profit_only': config['sell_profit_only'],
'sell_profit_offset': config['sell_profit_offset'],
'ignore_roi_if_buy_signal': config['ignore_roi_if_buy_signal'],
**daily_stats,
**trade_stats
}
@@ -436,7 +474,7 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
{ Strategy: {'results: results, 'config: config}}.
:param min_date: Backtest start date
:param max_date: Backtest end date
:return: Dictionary containing results per strategy and a stratgy summary.
:return: Dictionary containing results per strategy and a strategy summary.
"""
result: Dict[str, Any] = {'strategy': {}}
market_change = calculate_market_change(btdata, 'close')
@@ -504,12 +542,33 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
def text_table_periodic_breakdown(days_breakdown_stats: List[Dict[str, Any]],
stake_currency: str, period: str) -> str:
"""
Generate small table with Backtest results by days
:param days_breakdown_stats: Days breakdown metrics
:param stake_currency: Stakecurrency used
:return: pretty printed table with tabulate as string
"""
headers = [
period.capitalize(),
f'Tot Profit {stake_currency}',
'Wins',
'Draws',
'Losses',
]
output = [[
d['date'], round_coin_value(d['profit_abs'], stake_currency, False),
d['wins'], d['draws'], d['loses'],
] for d in days_breakdown_stats]
return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
def text_table_strategy(strategy_results, stake_currency: str) -> str:
"""
Generate summary table per strategy
:param strategy_results: Dict of <Strategyname: DataFrame> containing results for all strategies
:param stake_currency: stake-currency - used to correctly name headers
:param max_open_trades: Maximum allowed open trades used for backtest
:param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
:return: pretty printed table with tabulate as string
"""
floatfmt = _get_line_floatfmt(stake_currency)
@@ -543,28 +602,23 @@ def text_table_add_metrics(strat_results: Dict) -> str:
# Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
# command stores these results and newer version of freqtrade must be able to handle old
# results with missing new fields.
zero_duration_trades = '--'
if 'zero_duration_trades' in strat_results:
zero_duration_trades_per = \
100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades']
zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \
f'({strat_results["zero_duration_trades"]})'
metrics = [
('Backtesting from', strat_results['backtest_start']),
('Backtesting to', strat_results['backtest_end']),
('Max open trades', strat_results['max_open_trades']),
('', ''), # Empty line to improve readability
('Total trades', strat_results['total_trades']),
('Total/Daily Avg Trades',
f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"),
('Starting balance', round_coin_value(strat_results['starting_balance'],
strat_results['stake_currency'])),
('Final balance', round_coin_value(strat_results['final_balance'],
strat_results['stake_currency'])),
('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
strat_results['stake_currency'])),
('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"),
('Total profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
('Trades per day', strat_results['trades_per_day']),
('Avg. daily profit %',
f"{round(strat_results['profit_total'] / strat_results['backtest_days'] * 100, 2)}%"),
('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'],
strat_results['stake_currency'])),
('Total trade volume', round_coin_value(strat_results['total_volume'],
@@ -586,7 +640,6 @@ def text_table_add_metrics(strat_results: Dict) -> str:
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
('Zero Duration Trades', zero_duration_trades),
('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
('', ''), # Empty line to improve readability
@@ -613,7 +666,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])
stake_amount = round_coin_value(
strat_results['stake_amount'], strat_results['stake_currency']
) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited'
message = ("No trades made. "
f"Your starting balance was {start_balance}, "
@@ -622,7 +675,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
return message
def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str):
def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str,
backtest_breakdown=[]):
"""
Print results for one strategy
"""
@@ -644,6 +698,15 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
for period in backtest_breakdown:
days_breakdown_stats = generate_periodic_breakdown_stats(
trade_list=results['trades'], period=period)
table = text_table_periodic_breakdown(days_breakdown_stats=days_breakdown_stats,
stake_currency=stake_currency, period=period)
if isinstance(table, str) and len(table) > 0:
print(f' {period.upper()} BREAKDOWN '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_add_metrics(results)
if isinstance(table, str) and len(table) > 0:
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
@@ -658,12 +721,16 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
stake_currency = config['stake_currency']
for strategy, results in backtest_stats['strategy'].items():
show_backtest_result(strategy, results, stake_currency)
show_backtest_result(
strategy, results, stake_currency,
config.get('backtest_breakdown', []))
if len(backtest_stats['strategy']) > 1:
# Print Strategy summary table
table = text_table_strategy(backtest_stats['strategy_comparison'], stake_currency)
print(f"{results['backtest_start']} -> {results['backtest_end']} |"
f" Max open trades : {results['max_open_trades']}")
print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '='))
print(table)
print('=' * len(table.splitlines()[0]))

18
freqtrade/optimize/space/decimalspace.py
View File

@@ -7,11 +7,15 @@ class SKDecimal(Integer):
def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None,
name=None, dtype=np.int64):
self.decimals = decimals
_low = int(low * pow(10, self.decimals))
_high = int(high * pow(10, self.decimals))
self.pow_dot_one = pow(0.1, self.decimals)
self.pow_ten = pow(10, self.decimals)
_low = int(low * self.pow_ten)
_high = int(high * self.pow_ten)
# trunc to precision to avoid points out of space
self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals)
self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals)
self.low_orig = round(_low * self.pow_dot_one, self.decimals)
self.high_orig = round(_high * self.pow_dot_one, self.decimals)
super().__init__(_low, _high, prior, base, transform, name, dtype)
@@ -25,9 +29,9 @@ class SKDecimal(Integer):
return self.low_orig <= point <= self.high_orig
def transform(self, Xt):
aa = [int(x * pow(10, self.decimals)) for x in Xt]
return super().transform(aa)
return super().transform([int(v * self.pow_ten) for v in Xt])
def inverse_transform(self, Xt):
res = super().inverse_transform(Xt)
return [round(x * pow(0.1, self.decimals), self.decimals) for x in res]
# equivalent to [round(x * pow(0.1, self.decimals), self.decimals) for x in res]
return [int(v) / self.pow_ten for v in res]

25
freqtrade/persistence/migrations.py
View File

@@ -47,6 +47,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
strategy = get_column_def(cols, 'strategy', 'null')
buy_tag = get_column_def(cols, 'buy_tag', 'null')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
@@ -64,7 +65,8 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
# Schema migration necessary
with engine.begin() as connection:
connection.execute(text(f"alter table trades rename to {table_back_name}"))
# drop indexes on backup table
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_back_name):
connection.execute(text(f"drop index {index['name']}"))
# let SQLAlchemy create the schema as required
@@ -75,22 +77,15 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
connection.execute(text(f"""insert into trades
(id, exchange, pair, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_open_currency, open_rate,
fee_close, fee_close_cost, fee_close_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy,
max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag,
timeframe, open_trade_value, close_profit_abs
)
select id, lower(exchange),
case
when instr(pair, '_') != 0 then
substr(pair, instr(pair, '_') + 1) || '/' ||
substr(pair, 1, instr(pair, '_') - 1)
else pair
end
pair,
select id, lower(exchange), pair,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -103,7 +98,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {timeframe} timeframe,
{strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe,
{open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs
from {table_back_name}
"""))
@@ -131,7 +126,9 @@ def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, col
with engine.begin() as connection:
connection.execute(text(f"alter table orders rename to {table_back_name}"))
# drop indexes on backup table
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_back_name):
connection.execute(text(f"drop index {index['name']}"))
@@ -160,7 +157,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
table_back_name = get_backup_name(tabs, 'trades_bak')
# Check for latest column
if not has_column(cols, 'open_trade_value'):
if not has_column(cols, 'buy_tag'):
logger.info(f'Running database migration for trades - backup: {table_back_name}')
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
# Reread columns - the above recreated the table!

45
freqtrade/persistence/models.py
View File

@@ -2,7 +2,7 @@
This module contains the class to persist trades into SQLite
"""
import logging
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from typing import Any, Dict, List, Optional
@@ -13,7 +13,7 @@ from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session
from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.schema import UniqueConstraint
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.constants import DATETIME_PRINT_FORMAT, NON_OPEN_EXCHANGE_STATES
from freqtrade.enums import SellType
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.misc import safe_value_fallback
@@ -159,9 +159,9 @@ class Order(_DECL_BASE):
self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc)
self.ft_is_open = True
if self.status in ('closed', 'canceled', 'cancelled'):
if self.status in NON_OPEN_EXCHANGE_STATES:
self.ft_is_open = False
if order.get('filled', 0) > 0:
if (order.get('filled', 0.0) or 0.0) > 0:
self.order_filled_date = datetime.now(timezone.utc)
self.order_update_date = datetime.now(timezone.utc)
@@ -257,6 +257,7 @@ class LocalTrade():
sell_reason: str = ''
sell_order_status: str = ''
strategy: str = ''
buy_tag: Optional[str] = None
timeframe: Optional[int] = None
def __init__(self, **kwargs):
@@ -288,6 +289,7 @@ class LocalTrade():
'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None,
'stake_amount': round(self.stake_amount, 8),
'strategy': self.strategy,
'buy_tag': self.buy_tag,
'timeframe': self.timeframe,
'fee_open': self.fee_open,
@@ -352,12 +354,12 @@ class LocalTrade():
LocalTrade.trades_open = []
LocalTrade.total_profit = 0
def adjust_min_max_rates(self, current_price: float) -> None:
def adjust_min_max_rates(self, current_price: float, current_price_low: float) -> None:
"""
Adjust the max_rate and min_rate.
"""
self.max_rate = max(current_price, self.max_rate or self.open_rate)
self.min_rate = min(current_price, self.min_rate or self.open_rate)
self.min_rate = min(current_price_low, self.min_rate or self.open_rate)
def _set_new_stoploss(self, new_loss: float, stoploss: float):
"""Assign new stop value"""
@@ -636,7 +638,7 @@ class LocalTrade():
# skip case if trailing-stop changed the stoploss already.
if (trade.stop_loss == trade.initial_stop_loss
and trade.initial_stop_loss_pct != desired_stoploss):
and trade.initial_stop_loss_pct != desired_stoploss):
# Stoploss value got changed
logger.info(f"Stoploss for {trade} needs adjustment...")
@@ -703,6 +705,7 @@ class Trade(_DECL_BASE, LocalTrade):
sell_reason = Column(String(100), nullable=True)
sell_order_status = Column(String(100), nullable=True)
strategy = Column(String(100), nullable=True)
buy_tag = Column(String(100), nullable=True)
timeframe = Column(Integer, nullable=True)
def __init__(self, **kwargs):
@@ -801,6 +804,19 @@ class Trade(_DECL_BASE, LocalTrade):
Trade.is_open.is_(False),
]).all()
@staticmethod
def get_total_closed_profit() -> float:
"""
Retrieves total realized profit
"""
if Trade.use_db:
total_profit = Trade.query.with_entities(
func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar()
else:
total_profit = sum(
t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False))
return total_profit or 0
@staticmethod
def total_open_trades_stakes() -> float:
"""
@@ -816,17 +832,21 @@ class Trade(_DECL_BASE, LocalTrade):
return total_open_stake_amount or 0
@staticmethod
def get_overall_performance() -> List[Dict[str, Any]]:
def get_overall_performance(minutes=None) -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, including profit and trade count
NOTE: Not supported in Backtesting.
"""
filters = [Trade.is_open.is_(False)]
if minutes:
start_date = datetime.now(timezone.utc) - timedelta(minutes=minutes)
filters.append(Trade.close_date >= start_date)
pair_rates = Trade.query.with_entities(
Trade.pair,
func.sum(Trade.close_profit).label('profit_sum'),
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
func.count(Trade.pair).label('count')
).filter(Trade.is_open.is_(False))\
).filter(*filters)\
.group_by(Trade.pair) \
.order_by(desc('profit_sum_abs')) \
.all()
@@ -841,7 +861,7 @@ class Trade(_DECL_BASE, LocalTrade):
]
@staticmethod
def get_best_pair():
def get_best_pair(start_date: datetime = datetime.fromtimestamp(0)):
"""
Get best pair with closed trade.
NOTE: Not supported in Backtesting.
@@ -849,7 +869,7 @@ class Trade(_DECL_BASE, LocalTrade):
"""
best_pair = Trade.query.with_entities(
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
).filter(Trade.is_open.is_(False)) \
).filter(Trade.is_open.is_(False) & (Trade.close_date >= start_date)) \
.group_by(Trade.pair) \
.order_by(desc('profit_sum')).first()
return best_pair
@@ -876,7 +896,7 @@ class PairLock(_DECL_BASE):
lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT)
lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT)
return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, '
f'lock_end_time={lock_end_time})')
f'lock_end_time={lock_end_time}, reason={self.reason}, active={self.active})')
@staticmethod
def query_pair_locks(pair: Optional[str], now: datetime) -> Query:
@@ -885,7 +905,6 @@ class PairLock(_DECL_BASE):
:param pair: Pair to check for. Returns all current locks if pair is empty
:param now: Datetime object (generated via datetime.now(timezone.utc)).
"""
filters = [PairLock.lock_end_time > now,
# Only active locks
PairLock.active.is_(True), ]

38
freqtrade/persistence/pairlock_middleware.py
View File

@@ -30,7 +30,8 @@ class PairLocks():
PairLocks.locks = []
@staticmethod
def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None:
def lock_pair(pair: str, until: datetime, reason: str = None, *,
now: datetime = None) -> PairLock:
"""
Create PairLock from now to "until".
Uses database by default, unless PairLocks.use_db is set to False,
@@ -52,6 +53,7 @@ class PairLocks():
PairLock.query.session.commit()
else:
PairLocks.locks.append(lock)
return lock
@staticmethod
def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]:
@@ -101,6 +103,36 @@ class PairLocks():
if PairLocks.use_db:
PairLock.query.session.commit()
@staticmethod
def unlock_reason(reason: str, now: Optional[datetime] = None) -> None:
"""
Release all locks for this reason.
:param reason: Which reason to unlock
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
"""
if not now:
now = datetime.now(timezone.utc)
if PairLocks.use_db:
# used in live modes
logger.info(f"Releasing all locks with reason '{reason}':")
filters = [PairLock.lock_end_time > now,
PairLock.active.is_(True),
PairLock.reason == reason
]
locks = PairLock.query.filter(*filters)
for lock in locks:
logger.info(f"Releasing lock for {lock.pair} with reason '{reason}'.")
lock.active = False
PairLock.query.session.commit()
else:
# used in backtesting mode; don't show log messages for speed
locks = PairLocks.get_pair_locks(None)
for lock in locks:
if lock.reason == reason:
lock.active = False
@staticmethod
def is_global_lock(now: Optional[datetime] = None) -> bool:
"""
@@ -126,7 +158,9 @@ class PairLocks():
@staticmethod
def get_all_locks() -> List[PairLock]:
"""
Return all locks, also locks with expired end date
"""
if PairLocks.use_db:
return PairLock.query.all()
else:

14
freqtrade/plot/plotting.py
View File

@@ -288,8 +288,8 @@ def plot_area(fig, row: int, data: pd.DataFrame, indicator_a: str,
:param fig: Plot figure to append to
:param row: row number for this plot
:param data: candlestick DataFrame
:param indicator_a: indicator name as populated in stragetie
:param indicator_b: indicator name as populated in stragetie
:param indicator_a: indicator name as populated in strategy
:param indicator_b: indicator name as populated in strategy
:param label: label for the filled area
:param fill_color: color to be used for the filled area
:return: fig with added filled_traces plot
@@ -334,8 +334,8 @@ def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots:
)
elif indicator_b not in data:
logger.info(
'fill_to: "%s" ignored. Reason: This indicator is not '
'in your strategy.', indicator_b
'fill_to: "%s" ignored. Reason: This indicator is not '
'in your strategy.', indicator_b
)
return fig
@@ -373,6 +373,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
for i, name in enumerate(plot_config['subplots']):
fig['layout'][f'yaxis{3 + i}'].update(title=name)
fig['layout']['xaxis']['rangeslider'].update(visible=False)
fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
# Common information
candles = go.Candlestick(
@@ -452,6 +453,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
data=data)
# fill area between indicators ( 'fill_to': 'other_indicator')
fig = add_areas(fig, row, data, sub_config)
return fig
@@ -484,6 +486,7 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
fig['layout']['xaxis']['rangeslider'].update(visible=False)
fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
fig.add_trace(avgclose, 1, 1)
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
@@ -497,7 +500,6 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
fig = add_profit(fig, 3, df_comb, profit_col, f"Profit {pair}")
except ValueError:
pass
return fig
@@ -536,7 +538,7 @@ def load_and_plot_trades(config: Dict[str, Any]):
- Initializes plot-script
- Get candle (OHLCV) data
- Generate Dafaframes populated with indicators and signals based on configured strategy
- Load trades excecuted during the selected period
- Load trades executed during the selected period
- Generate Plotly plot objects
- Generate plot files
:return: None

55
freqtrade/plugins/pairlist/AgeFilter.py
View File

@@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional
import arrow
from pandas import DataFrame
from freqtrade.configuration import PeriodicCache
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -18,15 +19,17 @@ logger = logging.getLogger(__name__)
class AgeFilter(IPairList):
# Checked symbols cache (dictionary of ticker symbol => timestamp)
_symbolsChecked: Dict[str, int] = {}
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
# Checked symbols cache (dictionary of ticker symbol => timestamp)
self._symbolsChecked: Dict[str, int] = {}
self._symbolsCheckFailed = PeriodicCache(maxsize=1000, ttl=86_400)
self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
self._max_days_listed = pairlistconfig.get('max_days_listed', None)
if self._min_days_listed < 1:
raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
@@ -34,6 +37,12 @@ class AgeFilter(IPairList):
raise OperationalException("AgeFilter requires min_days_listed to not exceed "
"exchange max request size "
f"({exchange.ohlcv_candle_limit('1d')})")
if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
raise OperationalException("AgeFilter requires max_days_listed to not exceed "
"exchange max request size "
f"({exchange.ohlcv_candle_limit('1d')})")
@property
def needstickers(self) -> bool:
@@ -48,8 +57,13 @@ class AgeFilter(IPairList):
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Filtering pairs with age less than "
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
return (
f"{self.name} - Filtering pairs with age less than "
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
) + ((
" or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else '')
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
@@ -57,13 +71,19 @@ class AgeFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked]
needed_pairs = [
(p, '1d') for p in pairlist
if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
if not needed_pairs:
return pairlist
# Remove pairs that have been removed before
return [p for p in pairlist if p not in self._symbolsCheckFailed]
since_days = -(
self._max_days_listed if self._max_days_listed else self._min_days_listed
) - 1
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._min_days_listed - 1)
.shift(days=since_days)
.float_timestamp) * 1000
candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
if self._enabled:
@@ -86,14 +106,23 @@ class AgeFilter(IPairList):
return True
if daily_candles is not None:
if len(daily_candles) >= self._min_days_listed:
if (
len(daily_candles) >= self._min_days_listed
and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed)
):
# We have fetched at least the minimum required number of daily candles
# Add to cache, store the time we last checked this symbol
self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
return True
else:
self.log_once(f"Removed {pair} from whitelist, because age "
f"{len(daily_candles)} is less than {self._min_days_listed} "
f"{plural(self._min_days_listed, 'day')}", logger.info)
self.log_once((
f"Removed {pair} from whitelist, because age "
f"{len(daily_candles)} is less than {self._min_days_listed} "
f"{plural(self._min_days_listed, 'day')}"
) + ((
" or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else ''), logger.info)
self._symbolsCheckFailed[pair] = arrow.utcnow().int_timestamp * 1000
return False
return False

16
freqtrade/plugins/pairlist/IPairList.py
View File

@@ -144,24 +144,26 @@ class IPairList(LoggingMixin, ABC):
markets = self._exchange.markets
if not markets:
raise OperationalException(
'Markets not loaded. Make sure that exchange is initialized correctly.')
'Markets not loaded. Make sure that exchange is initialized correctly.')
sanitized_whitelist: List[str] = []
for pair in pairlist:
# pair is not in the generated dynamic market or has the wrong stake currency
if pair not in markets:
logger.warning(f"Pair {pair} is not compatible with exchange "
f"{self._exchange.name}. Removing it from whitelist..")
self.log_once(f"Pair {pair} is not compatible with exchange "
f"{self._exchange.name}. Removing it from whitelist..",
logger.warning)
continue
if not self._exchange.market_is_tradable(markets[pair]):
logger.warning(f"Pair {pair} is not tradable with Freqtrade."
"Removing it from whitelist..")
self.log_once(f"Pair {pair} is not tradable with Freqtrade."
"Removing it from whitelist..", logger.warning)
continue
if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']:
logger.warning(f"Pair {pair} is not compatible with your stake currency "
f"{self._config['stake_currency']}. Removing it from whitelist..")
self.log_once(f"Pair {pair} is not compatible with your stake currency "
f"{self._config['stake_currency']}. Removing it from whitelist..",
logger.warning)
continue
# Check if market is active

54
freqtrade/plugins/pairlist/OffsetFilter.py Normal file
View File

@@ -0,0 +1,54 @@
"""
Offset pair list filter
"""
import logging
from typing import Any, Dict, List
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class OffsetFilter(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._offset = pairlistconfig.get('offset', 0)
if self._offset < 0:
raise OperationalException("OffsetFilter requires offset to be >= 0")
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return False
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return f"{self.name} - Offseting pairs by {self._offset}."
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
if self._offset > len(pairlist):
self.log_once(f"Offset of {self._offset} is larger than " +
f"pair count of {len(pairlist)}", logger.warning)
pairs = pairlist[self._offset:]
self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
return pairs

22
freqtrade/plugins/pairlist/PerformanceFilter.py
View File

@@ -2,7 +2,7 @@
Performance pair list filter
"""
import logging
from typing import Dict, List
from typing import Any, Dict, List
import pandas as pd
@@ -15,11 +15,19 @@ logger = logging.getLogger(__name__)
class PerformanceFilter(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._minutes = pairlistconfig.get('minutes', 0)
self._min_profit = pairlistconfig.get('min_profit', None)
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requries tickers, an empty List is passed
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return False
@@ -40,7 +48,7 @@ class PerformanceFilter(IPairList):
"""
# Get the trading performance for pairs from database
try:
performance = pd.DataFrame(Trade.get_overall_performance())
performance = pd.DataFrame(Trade.get_overall_performance(self._minutes))
except AttributeError:
# Performancefilter does not work in backtesting.
self.log_once("PerformanceFilter is not available in this mode.", logger.warning)
@@ -61,6 +69,14 @@ class PerformanceFilter(IPairList):
sorted_df = list_df.merge(performance, on='pair', how='left')\
.fillna(0).sort_values(by=['count', 'pair'], ascending=True)\
.sort_values(by=['profit'], ascending=False)
if self._min_profit is not None:
removed = sorted_df[sorted_df['profit'] < self._min_profit]
for _, row in removed.iterrows():
self.log_once(
f"Removing pair {row['pair']} since {row['profit']} is "
f"below {self._min_profit}", logger.info)
sorted_df = sorted_df[sorted_df['profit'] >= self._min_profit]
pairlist = sorted_df['pair'].tolist()
return pairlist

12
freqtrade/plugins/pairlist/StaticPairList.py
View File

@@ -4,9 +4,9 @@ Static Pair List provider
Provides pair white list as it configured in config
"""
import logging
from copy import deepcopy
from typing import Any, Dict, List
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -20,10 +20,6 @@ class StaticPairList(IPairList):
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
if self._pairlist_pos != 0:
raise OperationalException(f"{self.name} can only be used in the first position "
"in the list of Pairlist Handlers.")
self._allow_inactive = self._pairlistconfig.get('allow_inactive', False)
@property
@@ -64,4 +60,8 @@ class StaticPairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
return pairlist
pairlist_ = deepcopy(pairlist)
for pair in self._config['exchange']['pair_whitelist']:
if pair not in pairlist_:
pairlist_.append(pair)
return pairlist_

12
freqtrade/plugins/pairlist/VolatilityFilter.py
View File

@@ -20,9 +20,9 @@ logger = logging.getLogger(__name__)
class VolatilityFilter(IPairList):
'''
"""
Filters pairs by volatility
'''
"""
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
@@ -69,10 +69,10 @@ class VolatilityFilter(IPairList):
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.float_timestamp) * 1000
since_ms = (arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.int_timestamp) * 1000
# Get all candles
candles = {}
if needed_pairs:

109
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -4,11 +4,15 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from functools import partial
from typing import Any, Dict, List
import arrow
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import format_ms_time
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -36,6 +40,35 @@ class VolumePairList(IPairList):
self._min_value = self._pairlistconfig.get('min_value', 0)
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
if (self._lookback_days > 0) & (self._lookback_period > 0):
raise OperationalException(
'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
'config. Please set lookback_days only or lookback_period and lookback_timeframe '
'and restart the bot.'
)
# overwrite lookback timeframe and days when lookback_days is set
if self._lookback_days > 0:
self._lookback_timeframe = '1d'
self._lookback_period = self._lookback_days
# get timeframe in minutes and seconds
self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
self._tf_in_sec = self._tf_in_min * 60
# wether to use range lookback or not
self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
if self._use_range & (self._refresh_period < self._tf_in_sec):
raise OperationalException(
f'Refresh period of {self._refresh_period} seconds is smaller than one '
f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
f'to at least {self._tf_in_sec} and restart the bot.'
)
if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException(
@@ -47,6 +80,13 @@ class VolumePairList(IPairList):
raise OperationalException(
f'key {self._sort_key} not in {SORT_VALUES}')
if self._lookback_period < 0:
raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
raise OperationalException("VolumeFilter requires lookback_period to not "
"exceed exchange max request size "
f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
@property
def needstickers(self) -> bool:
"""
@@ -76,19 +116,18 @@ class VolumePairList(IPairList):
pairlist = self._pair_cache.get('pairlist')
if pairlist:
# Item found - no refresh necessary
return pairlist
return pairlist.copy()
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
filtered_tickers = [
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and v[self._sort_key] is not None)]
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and (self._use_range or v[self._sort_key] is not None))]
pairlist = [s['symbol'] for s in filtered_tickers]
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache['pairlist'] = pairlist
self._pair_cache['pairlist'] = pairlist.copy()
return pairlist
@@ -103,15 +142,69 @@ class VolumePairList(IPairList):
# Use the incoming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
if self._use_range:
since_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-(self._lookback_period * self._tf_in_min)
- self._tf_in_min)
.int_timestamp) * 1000
to_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-self._tf_in_min)
.int_timestamp) * 1000
# todo: utc date output for starting date
self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}", logger.info)
needed_pairs = [
(p, self._lookback_timeframe) for p in
[
s['symbol'] for s in filtered_tickers
] if p not in self._pair_cache
]
# Get all candles
candles = {}
if needed_pairs:
candles = self._exchange.refresh_latest_ohlcv(
needed_pairs, since_ms=since_ms, cache=False
)
for i, p in enumerate(filtered_tickers):
pair_candles = candles[
(p['symbol'], self._lookback_timeframe)
] if (p['symbol'], self._lookback_timeframe) in candles else None
# in case of candle data calculate typical price and quoteVolume for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
quoteVolume = (pair_candles['quoteVolume']
.rolling(self._lookback_period)
.sum()
.iloc[-1])
# replace quoteVolume with range quoteVolume sum calculated above
filtered_tickers[i]['quoteVolume'] = quoteVolume
else:
filtered_tickers[i]['quoteVolume'] = 0
if self._min_value > 0:
filtered_tickers = [
v for v in filtered_tickers if v[self._sort_key] > self._min_value]
v for v in filtered_tickers if v[self._sort_key] > self._min_value]
sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
pairs = self.verify_blacklist(pairs, logger.info)
pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]

4
freqtrade/plugins/pairlist/pairlist_helpers.py
View File

@@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
if keep_invalid:
for pair_wc in wildcardpl:
try:
comp = re.compile(pair_wc)
comp = re.compile(pair_wc, re.IGNORECASE)
result_partial = [
pair for pair in available_pairs if re.fullmatch(comp, pair)
]
@@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
else:
for pair_wc in wildcardpl:
try:
comp = re.compile(pair_wc)
comp = re.compile(pair_wc, re.IGNORECASE)
result += [
pair for pair in available_pairs if re.fullmatch(comp, pair)
]

28
freqtrade/plugins/pairlist/rangestabilityfilter.py
View File

@@ -26,6 +26,7 @@ class RangeStabilityFilter(IPairList):
self._days = pairlistconfig.get('lookback_days', 10)
self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None)
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
@@ -50,8 +51,12 @@ class RangeStabilityFilter(IPairList):
"""
Short whitelist method description - used for startup-messages
"""
max_rate_desc = ""
if self._max_rate_of_change:
max_rate_desc = (f" and above {self._max_rate_of_change}")
return (f"{self.name} - Filtering pairs with rate of change below "
f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.")
f"{self._min_rate_of_change}{max_rate_desc} over the "
f"last {plural(self._days, 'day')}.")
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
@@ -62,10 +67,10 @@ class RangeStabilityFilter(IPairList):
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.float_timestamp) * 1000
since_ms = (arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.int_timestamp) * 1000
# Get all candles
candles = {}
if needed_pairs:
@@ -104,6 +109,17 @@ class RangeStabilityFilter(IPairList):
f"which is below the threshold of {self._min_rate_of_change}.",
logger.info)
result = False
if self._max_rate_of_change:
if pct_change <= self._max_rate_of_change:
result = True
else:
self.log_once(
f"Removed {pair} from whitelist, because rate of change "
f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, "
f"which is above the threshold of {self._max_rate_of_change}.",
logger.info)
result = False
self._pair_cache[pair] = result
else:
self.log_once(f"Removed {pair} from whitelist, no candles found.", logger.info)
return result

17
freqtrade/plugins/pairlistmanager.py
View File

@@ -28,13 +28,13 @@ class PairListManager():
self._tickers_needed = False
for pairlist_handler_config in self._config.get('pairlists', None):
pairlist_handler = PairListResolver.load_pairlist(
pairlist_handler_config['method'],
exchange=exchange,
pairlistmanager=self,
config=config,
pairlistconfig=pairlist_handler_config,
pairlist_pos=len(self._pairlist_handlers)
)
pairlist_handler_config['method'],
exchange=exchange,
pairlistmanager=self,
config=config,
pairlistconfig=pairlist_handler_config,
pairlist_pos=len(self._pairlist_handlers)
)
self._tickers_needed |= pairlist_handler.needstickers
self._pairlist_handlers.append(pairlist_handler)
@@ -83,7 +83,8 @@ class PairListManager():
pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
# Process all Pairlist Handlers in the chain
for pairlist_handler in self._pairlist_handlers:
# except for the first one, which is the generator.
for pairlist_handler in self._pairlist_handlers[1:]:
pairlist = pairlist_handler.filter_pairlist(pairlist, tickers)
# Validation against blacklist happens after the chain of Pairlist Handlers

27
freqtrade/plugins/protectionmanager.py
View File

@@ -6,6 +6,7 @@ from datetime import datetime, timezone
from typing import Dict, List, Optional
from freqtrade.persistence import PairLocks
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.protections import IProtection
from freqtrade.resolvers import ProtectionResolver
@@ -15,11 +16,11 @@ logger = logging.getLogger(__name__)
class ProtectionManager():
def __init__(self, config: dict) -> None:
def __init__(self, config: Dict, protections: List) -> None:
self._config = config
self._protection_handlers: List[IProtection] = []
for protection_handler_config in self._config.get('protections', []):
for protection_handler_config in protections:
protection_handler = ProtectionResolver.load_protection(
protection_handler_config['method'],
config=config,
@@ -43,30 +44,28 @@ class ProtectionManager():
"""
return [{p.name: p.short_desc()} for p in self._protection_handlers]
def global_stop(self, now: Optional[datetime] = None) -> bool:
def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]:
if not now:
now = datetime.now(timezone.utc)
result = False
result = None
for protection_handler in self._protection_handlers:
if protection_handler.has_global_stop:
result, until, reason = protection_handler.global_stop(now)
lock, until, reason = protection_handler.global_stop(now)
# Early stopping - first positive result blocks further trades
if result and until:
if lock and until:
if not PairLocks.is_global_lock(until):
PairLocks.lock_pair('*', until, reason, now=now)
result = True
result = PairLocks.lock_pair('*', until, reason, now=now)
return result
def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool:
def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]:
if not now:
now = datetime.now(timezone.utc)
result = False
result = None
for protection_handler in self._protection_handlers:
if protection_handler.has_local_stop:
result, until, reason = protection_handler.stop_per_pair(pair, now)
if result and until:
lock, until, reason = protection_handler.stop_per_pair(pair, now)
if lock and until:
if not PairLocks.is_pair_locked(pair, until):
PairLocks.lock_pair(pair, until, reason, now=now)
result = True
result = PairLocks.lock_pair(pair, until, reason, now=now)
return result

9
freqtrade/plugins/protections/iprotection.py
View File

@@ -25,19 +25,22 @@ class IProtection(LoggingMixin, ABC):
def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
self._config = config
self._protection_config = protection_config
self._stop_duration_candles: Optional[int] = None
self._lookback_period_candles: Optional[int] = None
tf_in_min = timeframe_to_minutes(config['timeframe'])
if 'stop_duration_candles' in protection_config:
self._stop_duration_candles = protection_config.get('stop_duration_candles', 1)
self._stop_duration_candles = int(protection_config.get('stop_duration_candles', 1))
self._stop_duration = (tf_in_min * self._stop_duration_candles)
else:
self._stop_duration_candles = None
self._stop_duration = protection_config.get('stop_duration', 60)
if 'lookback_period_candles' in protection_config:
self._lookback_period_candles = protection_config.get('lookback_period_candles', 1)
self._lookback_period_candles = int(protection_config.get('lookback_period_candles', 1))
self._lookback_period = tf_in_min * self._lookback_period_candles
else:
self._lookback_period_candles = None
self._lookback_period = protection_config.get('lookback_period', 60)
self._lookback_period = int(protection_config.get('lookback_period', 60))
LoggingMixin.__init__(self, logger)

6
freqtrade/plugins/protections/stoploss_guard.py
View File

@@ -54,9 +54,9 @@ class StoplossGuard(IProtection):
trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until)
trades = [trade for trade in trades1 if (str(trade.sell_reason) in (
SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
SellType.STOPLOSS_ON_EXCHANGE.value)
and trade.close_profit and trade.close_profit < 0)]
SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
SellType.STOPLOSS_ON_EXCHANGE.value)
and trade.close_profit and trade.close_profit < 0)]
if len(trades) < self._trade_limit:
return False, None, None

3
freqtrade/resolvers/__init__.py
View File

@@ -8,6 +8,3 @@ from freqtrade.resolvers.exchange_resolver import ExchangeResolver
from freqtrade.resolvers.pairlist_resolver import PairListResolver
from freqtrade.resolvers.protection_resolver import ProtectionResolver
from freqtrade.resolvers.strategy_resolver import StrategyResolver

1
freqtrade/resolvers/exchange_resolver.py
View File

@@ -21,6 +21,7 @@ class ExchangeResolver(IResolver):
def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange:
"""
Load the custom class from config parameter
:param exchange_name: name of the Exchange to load
:param config: configuration dictionary
"""
# Map exchange name to avoid duplicate classes for identical exchanges

38
freqtrade/resolvers/hyperopt_resolver.py
View File

@@ -9,7 +9,6 @@ from typing import Dict
from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS
from freqtrade.exceptions import OperationalException
from freqtrade.optimize.hyperopt_interface import IHyperOpt
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss
from freqtrade.resolvers import IResolver
@@ -17,43 +16,6 @@ from freqtrade.resolvers import IResolver
logger = logging.getLogger(__name__)
class HyperOptResolver(IResolver):
"""
This class contains all the logic to load custom hyperopt class
"""
object_type = IHyperOpt
object_type_str = "Hyperopt"
user_subdir = USERPATH_HYPEROPTS
initial_search_path = None
@staticmethod
def load_hyperopt(config: Dict) -> IHyperOpt:
"""
Load the custom hyperopt class from config parameter
:param config: configuration dictionary
"""
if not config.get('hyperopt'):
raise OperationalException("No Hyperopt set. Please use `--hyperopt` to specify "
"the Hyperopt class to use.")
hyperopt_name = config['hyperopt']
hyperopt = HyperOptResolver.load_object(hyperopt_name, config,
kwargs={'config': config},
extra_dir=config.get('hyperopt_path'))
if not hasattr(hyperopt, 'populate_indicators'):
logger.info("Hyperopt class does not provide populate_indicators() method. "
"Using populate_indicators from the strategy.")
if not hasattr(hyperopt, 'populate_buy_trend'):
logger.info("Hyperopt class does not provide populate_buy_trend() method. "
"Using populate_buy_trend from the strategy.")
if not hasattr(hyperopt, 'populate_sell_trend'):
logger.info("Hyperopt class does not provide populate_sell_trend() method. "
"Using populate_sell_trend from the strategy.")
return hyperopt
class HyperOptLossResolver(IResolver):
"""
This class contains all the logic to load custom hyperopt loss class

11
freqtrade/resolvers/iresolver.py
View File

@@ -58,6 +58,9 @@ class IResolver:
# Generate spec based on absolute path
# Pass object_name as first argument to have logging print a reasonable name.
spec = importlib.util.spec_from_file_location(object_name or "", str(module_path))
if not spec:
return iter([None])
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module) # type: ignore # importlib does not use typehints
@@ -88,7 +91,7 @@ class IResolver:
logger.debug(f"Searching for {cls.object_type.__name__} {object_name} in '{directory}'")
for entry in directory.iterdir():
# Only consider python files
if not str(entry).endswith('.py'):
if entry.suffix != '.py':
logger.debug('Ignoring %s', entry)
continue
if entry.is_symlink() and not entry.is_file():
@@ -132,7 +135,7 @@ class IResolver:
extra_dir: Optional[str] = None) -> Any:
"""
Search and loads the specified object as configured in hte child class.
:param objectname: name of the module to import
:param object_name: name of the module to import
:param config: configuration dictionary
:param extra_dir: additional directory to search for the given pairlist
:raises: OperationalException if the class is invalid or does not exist.
@@ -160,13 +163,13 @@ class IResolver:
:param directory: Path to search
:param enum_failed: If True, will return None for modules which fail.
Otherwise, failing modules are skipped.
:return: List of dicts containing 'name', 'class' and 'location' entires
:return: List of dicts containing 'name', 'class' and 'location' entries
"""
logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'")
objects = []
for entry in directory.iterdir():
# Only consider python files
if not str(entry).endswith('.py'):
if entry.suffix != '.py':
logger.debug('Ignoring %s', entry)
continue
module_path = entry.resolve()

89
freqtrade/resolvers/strategy_resolver.py
View File

@@ -45,57 +45,66 @@ class StrategyResolver(IResolver):
strategy_name, config=config,
extra_dir=config.get('strategy_path'))
# make sure ask_strategy dict is available
if 'ask_strategy' not in config:
config['ask_strategy'] = {}
if hasattr(strategy, 'ticker_interval') and not hasattr(strategy, 'timeframe'):
# Assign ticker_interval to timeframe to keep compatibility
if 'timeframe' not in config:
logger.warning(
"DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'."
)
)
strategy.timeframe = strategy.ticker_interval
if strategy._ft_params_from_file:
# Set parameters from Hyperopt results file
params = strategy._ft_params_from_file
strategy.minimal_roi = params.get('roi', getattr(strategy, 'minimal_roi', {}))
strategy.stoploss = params.get('stoploss', {}).get(
'stoploss', getattr(strategy, 'stoploss', -0.1))
trailing = params.get('trailing', {})
strategy.trailing_stop = trailing.get(
'trailing_stop', getattr(strategy, 'trailing_stop', False))
strategy.trailing_stop_positive = trailing.get(
'trailing_stop_positive', getattr(strategy, 'trailing_stop_positive', None))
strategy.trailing_stop_positive_offset = trailing.get(
'trailing_stop_positive_offset',
getattr(strategy, 'trailing_stop_positive_offset', 0))
strategy.trailing_only_offset_is_reached = trailing.get(
'trailing_only_offset_is_reached',
getattr(strategy, 'trailing_only_offset_is_reached', 0.0))
# Set attributes
# Check if we need to override configuration
# (Attribute name, default, subkey)
attributes = [("minimal_roi", {"0": 10.0}, None),
("timeframe", None, None),
("stoploss", None, None),
("trailing_stop", None, None),
("trailing_stop_positive", None, None),
("trailing_stop_positive_offset", 0.0, None),
("trailing_only_offset_is_reached", None, None),
("use_custom_stoploss", None, None),
("process_only_new_candles", None, None),
("order_types", None, None),
("order_time_in_force", None, None),
("stake_currency", None, None),
("stake_amount", None, None),
("protections", None, None),
("startup_candle_count", None, None),
("unfilledtimeout", None, None),
("use_sell_signal", True, 'ask_strategy'),
("sell_profit_only", False, 'ask_strategy'),
("ignore_roi_if_buy_signal", False, 'ask_strategy'),
("sell_profit_offset", 0.0, 'ask_strategy'),
("disable_dataframe_checks", False, None),
("ignore_buying_expired_candle_after", 0, 'ask_strategy')
attributes = [("minimal_roi", {"0": 10.0}),
("timeframe", None),
("stoploss", None),
("trailing_stop", None),
("trailing_stop_positive", None),
("trailing_stop_positive_offset", 0.0),
("trailing_only_offset_is_reached", None),
("use_custom_stoploss", None),
("process_only_new_candles", None),
("order_types", None),
("order_time_in_force", None),
("stake_currency", None),
("stake_amount", None),
("protections", None),
("startup_candle_count", None),
("unfilledtimeout", None),
("use_sell_signal", True),
("sell_profit_only", False),
("ignore_roi_if_buy_signal", False),
("sell_profit_offset", 0.0),
("disable_dataframe_checks", False),
("ignore_buying_expired_candle_after", 0)
]
for attribute, default, subkey in attributes:
if subkey:
StrategyResolver._override_attribute_helper(strategy, config.get(subkey, {}),
attribute, default)
else:
StrategyResolver._override_attribute_helper(strategy, config,
attribute, default)
for attribute, default in attributes:
StrategyResolver._override_attribute_helper(strategy, config,
attribute, default)
# Loop this list again to have output combined
for attribute, _, subkey in attributes:
if subkey and attribute in config[subkey]:
logger.info("Strategy using %s: %s", attribute, config[subkey][attribute])
elif attribute in config:
for attribute, _ in attributes:
if attribute in config:
logger.info("Strategy using %s: %s", attribute, config[attribute])
StrategyResolver._normalize_attributes(strategy)
@@ -113,7 +122,9 @@ class StrategyResolver(IResolver):
- Strategy
- default (if not None)
"""
if attribute in config:
if (attribute in config
and not isinstance(getattr(type(strategy), attribute, None), property)):
# Ensure Properties are not overwritten
setattr(strategy, attribute, config[attribute])
logger.info("Override strategy '%s' with value in config file: %s.",
attribute, config[attribute])

182
freqtrade/rpc/api_server/api_backtest.py Normal file
View File

@@ -0,0 +1,182 @@
import asyncio
import logging
from copy import deepcopy
from fastapi import APIRouter, BackgroundTasks, Depends
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.enums import BacktestState
from freqtrade.exceptions import DependencyException
from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse
from freqtrade.rpc.api_server.deps import get_config
from freqtrade.rpc.api_server.webserver import ApiServer
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# Private API, protected by authentication
router = APIRouter()
@router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
config=Depends(get_config)):
"""Start backtesting if not done so already"""
if ApiServer._bgtask_running:
raise RPCException('Bot Background task already running')
btconfig = deepcopy(config)
settings = dict(bt_settings)
# Pydantic models will contain all keys, but non-provided ones are None
for setting in settings.keys():
if settings[setting] is not None:
btconfig[setting] = settings[setting]
# Start backtesting
# Initialize backtesting object
def run_backtest():
from freqtrade.optimize.optimize_reports import generate_backtest_stats
from freqtrade.resolvers import StrategyResolver
asyncio.set_event_loop(asyncio.new_event_loop())
try:
# Reload strategy
lastconfig = ApiServer._bt_last_config
strat = StrategyResolver.load_strategy(btconfig)
validate_config_consistency(btconfig)
if (
not ApiServer._bt
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail')
or lastconfig.get('timerange') != btconfig['timerange']
):
from freqtrade.optimize.backtesting import Backtesting
ApiServer._bt = Backtesting(btconfig)
if ApiServer._bt.timeframe_detail:
ApiServer._bt.load_bt_data_detail()
else:
ApiServer._bt.config = btconfig
ApiServer._bt.init_backtest()
# Only reload data if timeframe changed.
if (
not ApiServer._bt_data
or not ApiServer._bt_timerange
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('timerange') != btconfig['timerange']
):
ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
lastconfig['timerange'] = btconfig['timerange']
lastconfig['timeframe'] = strat.timeframe
lastconfig['protections'] = btconfig.get('protections', [])
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
ApiServer._bt.abort = False
min_date, max_date = ApiServer._bt.backtest_one_strategy(
strat, ApiServer._bt_data, ApiServer._bt_timerange)
ApiServer._bt.results = generate_backtest_stats(
ApiServer._bt_data, ApiServer._bt.all_results,
min_date=min_date, max_date=max_date)
logger.info("Backtest finished.")
except DependencyException as e:
logger.info(f"Backtesting caused an error: {e}")
pass
finally:
ApiServer._bgtask_running = False
background_tasks.add_task(run_backtest)
ApiServer._bgtask_running = True
return {
"status": "running",
"running": True,
"progress": 0,
"step": str(BacktestState.STARTUP),
"status_msg": "Backtest started",
}
@router.get('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_get_backtest():
"""
Get backtesting result.
Returns Result after backtesting has been ran.
"""
from freqtrade.persistence import LocalTrade
if ApiServer._bgtask_running:
return {
"status": "running",
"running": True,
"step": ApiServer._bt.progress.action if ApiServer._bt else str(BacktestState.STARTUP),
"progress": ApiServer._bt.progress.progress if ApiServer._bt else 0,
"trade_count": len(LocalTrade.trades),
"status_msg": "Backtest running",
}
if not ApiServer._bt:
return {
"status": "not_started",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest not yet executed"
}
return {
"status": "ended",
"running": False,
"status_msg": "Backtest ended",
"step": "finished",
"progress": 1,
"backtest_result": ApiServer._bt.results,
}
@router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_delete_backtest():
"""Reset backtesting"""
if ApiServer._bgtask_running:
return {
"status": "running",
"running": True,
"step": "",
"progress": 0,
"status_msg": "Backtest running",
}
if ApiServer._bt:
del ApiServer._bt
ApiServer._bt = None
del ApiServer._bt_data
ApiServer._bt_data = None
logger.info("Backtesting reset")
return {
"status": "reset",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest reset",
}
@router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_backtest_abort():
if not ApiServer._bgtask_running:
return {
"status": "not_running",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest ended",
}
ApiServer._bt.abort = True
return {
"status": "stopping",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest ended",
}

48
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -46,6 +46,12 @@ class Balances(BaseModel):
value: float
stake: str
note: str
starting_capital: float
starting_capital_ratio: float
starting_capital_pct: float
starting_capital_fiat: float
starting_capital_fiat_ratio: float
starting_capital_fiat_pct: float
class Count(BaseModel):
@@ -67,12 +73,16 @@ class Profit(BaseModel):
profit_closed_ratio_mean: float
profit_closed_percent_sum: float
profit_closed_ratio_sum: float
profit_closed_percent: float
profit_closed_ratio: float
profit_closed_fiat: float
profit_all_coin: float
profit_all_percent_mean: float
profit_all_ratio_mean: float
profit_all_percent_sum: float
profit_all_ratio_sum: float
profit_all_percent: float
profit_all_ratio: float
profit_all_fiat: float
trade_count: int
closed_trade_count: int
@@ -115,19 +125,21 @@ class ShowConfig(BaseModel):
dry_run: bool
stake_currency: str
stake_amount: Union[float, str]
available_capital: Optional[float]
stake_currency_decimals: int
max_open_trades: int
minimal_roi: Dict[str, Any]
stoploss: float
trailing_stop: bool
stoploss: Optional[float]
trailing_stop: Optional[bool]
trailing_stop_positive: Optional[float]
trailing_stop_positive_offset: Optional[float]
trailing_only_offset_is_reached: Optional[bool]
use_custom_stoploss: Optional[bool]
timeframe: str
timeframe: Optional[str]
timeframe_ms: int
timeframe_min: int
exchange: str
strategy: str
strategy: Optional[str]
forcebuy_enabled: bool
ask_strategy: Dict[str, Any]
bid_strategy: Dict[str, Any]
@@ -145,6 +157,7 @@ class TradeSchema(BaseModel):
amount_requested: float
stake_amount: float
strategy: str
buy_tag: Optional[str]
timeframe: int
fee_open: Optional[float]
fee_open_cost: Optional[float]
@@ -312,3 +325,30 @@ class PairHistory(BaseModel):
json_encoders = {
datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT),
}
class BacktestRequest(BaseModel):
strategy: str
timeframe: Optional[str]
timeframe_detail: Optional[str]
timerange: Optional[str]
max_open_trades: Optional[int]
stake_amount: Optional[Union[float, str]]
enable_protections: bool
dry_run_wallet: Optional[float]
class BacktestResponse(BaseModel):
status: str
running: bool
status_msg: str
step: str
progress: float
trade_count: Optional[float]
# TODO: Properly type backtestresult...
backtest_result: Optional[Dict[str, Any]]
class SysInfo(BaseModel):
cpu_pct: List[float]
ram_pct: float

27
freqtrade/rpc/api_server/api_v1.py
View File

@@ -1,3 +1,4 @@
import logging
from copy import deepcopy
from pathlib import Path
from typing import List, Optional
@@ -17,11 +18,14 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac
OpenTradeSchema, PairHistory, PerformanceEntry,
Ping, PlotConfig, Profit, ResultMsg, ShowConfig,
Stats, StatusMsg, StrategyListResponse,
StrategyResponse, Version, WhitelistResponse)
StrategyResponse, SysInfo, Version,
WhitelistResponse)
from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# Public API, requires no auth.
router_public = APIRouter()
# Private API, protected by authentication
@@ -162,8 +166,8 @@ def delete_lock_pair(payload: DeleteLockRequest, rpc: RPC = Depends(get_rpc)):
@router.get('/logs', response_model=Logs, tags=['info'])
def logs(limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)):
return rpc._rpc_get_logs(limit)
def logs(limit: Optional[int] = None):
return RPC._rpc_get_logs(limit)
@router.post('/start', response_model=StatusMsg, tags=['botcontrol'])
@@ -196,8 +200,8 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str,
config=Depends(get_config)):
config = deepcopy(config)
config.update({
'strategy': strategy,
})
'strategy': strategy,
})
return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange)
@@ -220,11 +224,11 @@ def list_strategies(config=Depends(get_config)):
@router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
def get_strategy(strategy: str, config=Depends(get_config)):
config = deepcopy(config)
config_ = deepcopy(config)
from freqtrade.resolvers.strategy_resolver import StrategyResolver
try:
strategy_obj = StrategyResolver._load_strategy(strategy, config,
extra_dir=config.get('strategy_path'))
strategy_obj = StrategyResolver._load_strategy(strategy, config_,
extra_dir=config_.get('strategy_path'))
except OperationalException:
raise HTTPException(status_code=404, detail='Strategy not found')
@@ -249,10 +253,15 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option
pair_interval = sorted(pair_interval, key=lambda x: x[0])
pairs = list({x[0] for x in pair_interval})
pairs.sort()
result = {
'length': len(pairs),
'pairs': pairs,
'pair_interval': pair_interval,
}
return result
@router.get('/sysinfo', response_model=SysInfo, tags=['info'])
def sysinfo():
return RPC._rpc_sysinfo()

9
freqtrade/rpc/api_server/deps.py
View File

@@ -1,5 +1,6 @@
from typing import Any, Dict, Optional
from typing import Any, Dict, Iterator, Optional
from freqtrade.persistence import Trade
from freqtrade.rpc.rpc import RPC, RPCException
from .webserver import ApiServer
@@ -11,10 +12,12 @@ def get_rpc_optional() -> Optional[RPC]:
return None
def get_rpc() -> Optional[RPC]:
def get_rpc() -> Optional[Iterator[RPC]]:
_rpc = get_rpc_optional()
if _rpc:
return _rpc
Trade.query.session.rollback()
yield _rpc
Trade.query.session.rollback()
else:
raise RPCException('Bot is not in the correct state')

23
freqtrade/rpc/api_server/uvicorn_threaded.py
View File

@@ -5,6 +5,20 @@ import time
import uvicorn
def asyncio_setup() -> None: # pragma: no cover
# Set eventloop for win32 setups
# Reverts a change done in uvicorn 0.15.0 - which now sets the eventloop
# via policy.
import sys
if sys.version_info >= (3, 8) and sys.platform == "win32":
import asyncio
import selectors
selector = selectors.SelectSelector()
loop = asyncio.SelectorEventLoop(selector)
asyncio.set_event_loop(loop)
class UvicornServer(uvicorn.Server):
"""
Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742
@@ -28,12 +42,15 @@ class UvicornServer(uvicorn.Server):
try:
import uvloop # noqa
except ImportError: # pragma: no cover
from uvicorn.loops.asyncio import asyncio_setup
asyncio_setup()
else:
asyncio.set_event_loop(uvloop.new_event_loop())
loop = asyncio.get_event_loop()
try:
loop = asyncio.get_event_loop()
except RuntimeError:
# When running in a thread, we'll not have an eventloop yet.
loop = asyncio.new_event_loop()
loop.run_until_complete(self.serve(sockets=sockets))
@contextlib.contextmanager

28
freqtrade/rpc/api_server/web_ui.py
View File

@@ -18,6 +18,27 @@ async def fallback():
return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html'))
@router_ui.get('/ui_version', include_in_schema=False)
async def ui_version():
from freqtrade.commands.deploy_commands import read_ui_version
uibase = Path(__file__).parent / 'ui/installed/'
version = read_ui_version(uibase)
return {
"version": version if version else "not_installed",
}
def is_relative_to(path, base) -> bool:
# Helper function simulating behaviour of is_relative_to, which was only added in python 3.9
try:
path.relative_to(base)
return True
except ValueError:
pass
return False
@router_ui.get('/{rest_of_path:path}', include_in_schema=False)
async def index_html(rest_of_path: str):
"""
@@ -26,8 +47,11 @@ async def index_html(rest_of_path: str):
if rest_of_path.startswith('api') or rest_of_path.startswith('.'):
raise HTTPException(status_code=404, detail="Not Found")
uibase = Path(__file__).parent / 'ui/installed/'
if (uibase / rest_of_path).is_file():
return FileResponse(str(uibase / rest_of_path))
filename = uibase / rest_of_path
# It's security relevant to check "relative_to".
# Without this, Directory-traversal is possible.
if filename.is_file() and is_relative_to(filename, uibase):
return FileResponse(str(filename))
index_file = uibase / 'index.html'
if not index_file.is_file():

69
freqtrade/rpc/api_server/webserver.py
View File

@@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
from starlette.responses import JSONResponse
from freqtrade.exceptions import OperationalException
from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer
from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler
@@ -28,17 +29,37 @@ class FTJSONResponse(JSONResponse):
class ApiServer(RPCHandler):
__instance = None
__initialized = False
_rpc: RPC
# Backtesting type: Backtesting
_bt = None
_bt_data = None
_bt_timerange = None
_bt_last_config: Dict[str, Any] = {}
_has_rpc: bool = False
_bgtask_running: bool = False
_config: Dict[str, Any] = {}
def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None:
super().__init__(rpc, config)
self._server = None
def __new__(cls, *args, **kwargs):
"""
This class is a singleton.
We'll only have one instance of it around.
"""
if ApiServer.__instance is None:
ApiServer.__instance = object.__new__(cls)
ApiServer.__initialized = False
return ApiServer.__instance
ApiServer._rpc = rpc
ApiServer._has_rpc = True
def __init__(self, config: Dict[str, Any], standalone: bool = False) -> None:
ApiServer._config = config
if self.__initialized and (standalone or self._standalone):
return
self._standalone: bool = standalone
self._server = None
ApiServer.__initialized = True
api_config = self._config['api_server']
self.app = FastAPI(title="Freqtrade API",
@@ -50,12 +71,33 @@ class ApiServer(RPCHandler):
self.start_api()
def add_rpc_handler(self, rpc: RPC):
"""
Attach rpc handler
"""
if not self._has_rpc:
ApiServer._rpc = rpc
ApiServer._has_rpc = True
else:
# This should not happen assuming we didn't mess up.
raise OperationalException('RPC Handler already attached.')
def cleanup(self) -> None:
""" Cleanup pending module resources """
if self._server:
ApiServer._has_rpc = False
del ApiServer._rpc
if self._server and not self._standalone:
logger.info("Stopping API Server")
self._server.cleanup()
@classmethod
def shutdown(cls):
cls.__initialized = False
del cls.__instance
cls.__instance = None
cls._has_rpc = False
cls._rpc = None
def send_msg(self, msg: Dict[str, str]) -> None:
pass
@@ -68,6 +110,7 @@ class ApiServer(RPCHandler):
def configure_app(self, app: FastAPI, config):
from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login
from freqtrade.rpc.api_server.api_backtest import router as api_backtest
from freqtrade.rpc.api_server.api_v1 import router as api_v1
from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public
from freqtrade.rpc.api_server.web_ui import router_ui
@@ -77,6 +120,9 @@ class ApiServer(RPCHandler):
app.include_router(api_v1, prefix="/api/v1",
dependencies=[Depends(http_basic_or_jwt_token)],
)
app.include_router(api_backtest, prefix="/api/v1",
dependencies=[Depends(http_basic_or_jwt_token)],
)
app.include_router(router_login, prefix="/api/v1", tags=["auth"])
# UI Router MUST be last!
app.include_router(router_ui, prefix='')
@@ -115,18 +161,19 @@ class ApiServer(RPCHandler):
logger.info('Starting Local Rest Server.')
verbosity = self._config['api_server'].get('verbosity', 'error')
log_config = uvicorn.config.LOGGING_CONFIG
# Change logging of access logs to stderr
log_config["handlers"]["access"]["stream"] = log_config["handlers"]["default"]["stream"]
uvconfig = uvicorn.Config(self.app,
port=rest_port,
host=rest_ip,
use_colors=False,
log_config=log_config,
log_config=None,
access_log=True if verbosity != 'error' else False,
)
try:
self._server = UvicornServer(uvconfig)
self._server.run_in_thread()
if self._standalone:
self._server.run()
else:
self._server.run_in_thread()
except Exception:
logger.exception("Api server failed to start.")

46
freqtrade/rpc/fiat_convert.py
View File

@@ -5,7 +5,7 @@ e.g BTC to USD
import datetime
import logging
from typing import Dict
from typing import Dict, List
from cachetools.ttl import TTLCache
from pycoingecko import CoinGeckoAPI
@@ -25,8 +25,7 @@ class CryptoToFiatConverter:
"""
__instance = None
_coingekko: CoinGeckoAPI = None
_cryptomap: Dict = {}
_coinlistings: List[Dict] = []
_backoff: float = 0.0
def __new__(cls):
@@ -49,9 +48,8 @@ class CryptoToFiatConverter:
def _load_cryptomap(self) -> None:
try:
coinlistings = self._coingekko.get_coins_list()
# Create mapping table from symbol to coingekko_id
self._cryptomap = {x['symbol']: x['id'] for x in coinlistings}
# Use list-comprehension to ensure we get a list.
self._coinlistings = [x for x in self._coingekko.get_coins_list()]
except RequestException as request_exception:
if "429" in str(request_exception):
logger.warning(
@@ -62,13 +60,31 @@ class CryptoToFiatConverter:
# If the request is not a 429 error we want to raise the normal error
logger.error(
"Could not load FIAT Cryptocurrency map for the following problem: {}".format(
request_exception
request_exception
)
)
except (Exception) as exception:
logger.error(
f"Could not load FIAT Cryptocurrency map for the following problem: {exception}")
def _get_gekko_id(self, crypto_symbol):
if not self._coinlistings:
if self._backoff <= datetime.datetime.now().timestamp():
self._load_cryptomap()
# Still not loaded.
if not self._coinlistings:
return None
else:
return None
found = [x for x in self._coinlistings if x['symbol'] == crypto_symbol]
if len(found) == 1:
return found[0]['id']
if len(found) > 0:
# Wrong!
logger.warning(f"Found multiple mappings in goingekko for {crypto_symbol}.")
return None
def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float:
"""
Convert an amount of crypto-currency to fiat
@@ -102,7 +118,7 @@ class CryptoToFiatConverter:
inverse = True
symbol = f"{crypto_symbol}/{fiat_symbol}"
# Check if the fiat convertion you want is supported
# Check if the fiat conversion you want is supported
if not self._is_supported_fiat(fiat=fiat_symbol):
raise ValueError(f'The fiat {fiat_symbol} is not supported.')
@@ -135,7 +151,7 @@ class CryptoToFiatConverter:
:param fiat_symbol: FIAT currency you want to convert to (e.g usd)
:return: float, price of the crypto-currency in Fiat
"""
# Check if the fiat convertion you want is supported
# Check if the fiat conversion you want is supported
if not self._is_supported_fiat(fiat=fiat_symbol):
raise ValueError(f'The fiat {fiat_symbol} is not supported.')
@@ -143,22 +159,14 @@ class CryptoToFiatConverter:
if crypto_symbol == fiat_symbol:
return 1.0
if self._cryptomap == {}:
if self._backoff <= datetime.datetime.now().timestamp():
self._load_cryptomap()
# return 0.0 if we still dont have data to check, no reason to proceed
if self._cryptomap == {}:
return 0.0
else:
return 0.0
_gekko_id = self._get_gekko_id(crypto_symbol)
if crypto_symbol not in self._cryptomap:
if not _gekko_id:
# return 0 for unsupported stake currencies (fiat-convert should not break the bot)
logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol)
return 0.0
try:
_gekko_id = self._cryptomap[crypto_symbol]
return float(
self._coingekko.get_price(
ids=_gekko_id,

100
freqtrade/rpc/rpc.py
View File

@@ -8,6 +8,7 @@ from math import isnan
from typing import Any, Dict, List, Optional, Tuple, Union
import arrow
import psutil
from numpy import NAN, inf, int64, mean
from pandas import DataFrame
@@ -18,7 +19,7 @@ from freqtrade.enums import SellType, State
from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
from freqtrade.misc import shorten_date
from freqtrade.misc import decimals_per_coin, shorten_date
from freqtrade.persistence import PairLocks, Trade
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -104,7 +105,9 @@ class RPC:
val = {
'dry_run': config['dry_run'],
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'stake_amount': config['stake_amount'],
'available_capital': config.get('available_capital'),
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {},
@@ -117,9 +120,9 @@ class RPC:
'bot_name': config.get('bot_name', 'freqtrade'),
'timeframe': config.get('timeframe'),
'timeframe_ms': timeframe_to_msecs(config['timeframe']
) if 'timeframe' in config else '',
) if 'timeframe' in config else 0,
'timeframe_min': timeframe_to_minutes(config['timeframe']
) if 'timeframe' in config else '',
) if 'timeframe' in config else 0,
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),
@@ -152,7 +155,8 @@ class RPC:
# calculate profit and send message to user
if trade.is_open:
try:
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
except (ExchangeError, PricingError):
current_rate = NAN
else:
@@ -180,9 +184,9 @@ class RPC:
base_currency=self._freqtrade.config['stake_currency'],
close_profit=trade.close_profit if trade.close_profit is not None else None,
current_rate=current_rate,
current_profit=current_profit, # Deprectated
current_profit_pct=round(current_profit * 100, 2), # Deprectated
current_profit_abs=current_profit_abs, # Deprectated
current_profit=current_profit, # Deprecated
current_profit_pct=round(current_profit * 100, 2), # Deprecated
current_profit_abs=current_profit_abs, # Deprecated
profit_ratio=current_profit,
profit_pct=round(current_profit * 100, 2),
profit_abs=current_profit_abs,
@@ -211,7 +215,8 @@ class RPC:
for trade in trades:
# calculate profit and send message to user
try:
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
except (PricingError, ExchangeError):
current_rate = NAN
trade_percent = (100 * trade.calc_profit_ratio(current_rate))
@@ -270,10 +275,10 @@ class RPC:
'date': key,
'abs_profit': value["amount"],
'fiat_value': self._fiat_converter.convert_amount(
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
'trade_count': value["trades"],
}
for key, value in profit_days.items()
@@ -339,7 +344,9 @@ class RPC:
self, stake_currency: str, fiat_display_currency: str,
start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]:
""" Returns cumulative profit statistics """
trades = Trade.get_trades([Trade.open_date >= start_date]).order_by(Trade.id).all()
trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) |
Trade.is_open.is_(True))
trades = Trade.get_trades(trade_filter).order_by(Trade.id).all()
profit_all_coin = []
profit_all_ratio = []
@@ -368,7 +375,8 @@ class RPC:
else:
# Get current rate
try:
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
except (PricingError, ExchangeError):
current_rate = NAN
profit_ratio = trade.calc_profit_ratio(rate=current_rate)
@@ -378,7 +386,7 @@ class RPC:
)
profit_all_ratio.append(profit_ratio)
best_pair = Trade.get_best_pair()
best_pair = Trade.get_best_pair(start_date)
# Prepare data to display
profit_closed_coin_sum = round(sum(profit_closed_coin), 8)
@@ -393,7 +401,15 @@ class RPC:
profit_all_coin_sum = round(sum(profit_all_coin), 8)
profit_all_ratio_mean = float(mean(profit_all_ratio) if profit_all_ratio else 0.0)
# Doing the sum is not right - overall profit needs to be based on initial capital
profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0
starting_balance = self._freqtrade.wallets.get_starting_balance()
profit_closed_ratio_fromstart = 0
profit_all_ratio_fromstart = 0
if starting_balance:
profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance
profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance
profit_all_fiat = self._fiat_converter.convert_amount(
profit_all_coin_sum,
stake_currency,
@@ -409,12 +425,16 @@ class RPC:
'profit_closed_ratio_mean': profit_closed_ratio_mean,
'profit_closed_percent_sum': round(profit_closed_ratio_sum * 100, 2),
'profit_closed_ratio_sum': profit_closed_ratio_sum,
'profit_closed_ratio': profit_closed_ratio_fromstart,
'profit_closed_percent': round(profit_closed_ratio_fromstart * 100, 2),
'profit_closed_fiat': profit_closed_fiat,
'profit_all_coin': profit_all_coin_sum,
'profit_all_percent_mean': round(profit_all_ratio_mean * 100, 2),
'profit_all_ratio_mean': profit_all_ratio_mean,
'profit_all_percent_sum': round(profit_all_ratio_sum * 100, 2),
'profit_all_ratio_sum': profit_all_ratio_sum,
'profit_all_ratio': profit_all_ratio_fromstart,
'profit_all_percent': round(profit_all_ratio_fromstart * 100, 2),
'profit_all_fiat': profit_all_fiat,
'trade_count': len(trades),
'closed_trade_count': len([t for t in trades if not t.is_open]),
@@ -439,6 +459,9 @@ class RPC:
raise RPCException('Error getting current tickers.')
self._freqtrade.wallets.update(require_update=False)
starting_capital = self._freqtrade.wallets.get_starting_balance()
starting_cap_fiat = self._fiat_converter.convert_amount(
starting_capital, stake_currency, fiat_display_currency) if self._fiat_converter else 0
for coin, balance in self._freqtrade.wallets.get_all_balances().items():
if not balance.total:
@@ -474,15 +497,25 @@ class RPC:
else:
raise RPCException('All balances are zero.')
symbol = fiat_display_currency
value = self._fiat_converter.convert_amount(total, stake_currency,
symbol) if self._fiat_converter else 0
value = self._fiat_converter.convert_amount(
total, stake_currency, fiat_display_currency) if self._fiat_converter else 0
starting_capital_ratio = 0.0
starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0
starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0
return {
'currencies': output,
'total': total,
'symbol': symbol,
'symbol': fiat_display_currency,
'value': value,
'stake': stake_currency,
'starting_capital': starting_capital,
'starting_capital_ratio': starting_capital_ratio,
'starting_capital_pct': round(starting_capital_ratio * 100, 2),
'starting_capital_fiat': starting_cap_fiat,
'starting_capital_fiat_ratio': starting_cap_fiat_ratio,
'starting_capital_fiat_pct': round(starting_cap_fiat_ratio * 100, 2),
'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else ''
}
@@ -529,24 +562,25 @@ class RPC:
order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
if order['side'] == 'buy':
fully_canceled = self._freqtrade.handle_cancel_buy(
fully_canceled = self._freqtrade.handle_cancel_enter(
trade, order, CANCEL_REASON['FORCE_SELL'])
if order['side'] == 'sell':
# Cancel order - so it is placed anew with a fresh price.
self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL'])
self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_SELL'])
if not fully_canceled:
# Get current rate and execute sell
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
self._freqtrade.execute_sell(trade, current_rate, sell_reason)
self._freqtrade.execute_trade_exit(trade, current_rate, sell_reason)
# ---- EOF def _exec_forcesell ----
if self._freqtrade.state != State.RUNNING:
raise RPCException('trader is not running')
with self._freqtrade._sell_lock:
with self._freqtrade._exit_lock:
if trade_id == 'all':
# Execute sell for all open orders
for trade in Trade.get_open_trades():
@@ -596,7 +630,7 @@ class RPC:
stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
# execute buy
if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True):
if self._freqtrade.execute_entry(pair, stakeamount, price, forcebuy=True):
Trade.commit()
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
return trade
@@ -608,7 +642,7 @@ class RPC:
Handler for delete <id>.
Delete the given trade and close eventually existing open orders.
"""
with self._freqtrade._sell_lock:
with self._freqtrade._exit_lock:
c_count = 0
trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first()
if not trade:
@@ -758,8 +792,8 @@ class RPC:
sell_signals = 0
if has_content:
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
# Move open to seperate column when signal for easy plotting
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
# Move open to separate column when signal for easy plotting
if 'buy' in dataframe.columns:
buy_mask = (dataframe['buy'] == 1)
buy_signals = int(buy_mask.sum())
@@ -822,8 +856,11 @@ class RPC:
)
if pair not in _data:
raise RPCException(f"No data for {pair}, {timeframe} in {timerange} found.")
from freqtrade.data.dataprovider import DataProvider
from freqtrade.resolvers.strategy_resolver import StrategyResolver
strategy = StrategyResolver.load_strategy(config)
strategy.dp = DataProvider(config, exchange=None, pairlists=None)
df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair})
return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe,
@@ -834,3 +871,10 @@ class RPC:
'subplots' not in self._freqtrade.strategy.plot_config):
self._freqtrade.strategy.plot_config['subplots'] = {}
return self._freqtrade.strategy.plot_config
@staticmethod
def _rpc_sysinfo() -> Dict[str, Any]:
return {
"cpu_pct": psutil.cpu_percent(interval=1, percpu=True),
"ram_pct": psutil.virtual_memory().percent
}

12
freqtrade/rpc/rpc_manager.py
View File

@@ -1,5 +1,5 @@
"""
This module contains class to manage RPC communications (Telegram, Slack, ...)
This module contains class to manage RPC communications (Telegram, API, ...)
"""
import logging
from typing import Any, Dict, List
@@ -13,8 +13,9 @@ logger = logging.getLogger(__name__)
class RPCManager:
"""
Class to manage RPC objects (Telegram, Slack, ...)
Class to manage RPC objects (Telegram, API, ...)
"""
def __init__(self, freqtrade) -> None:
""" Initializes all enabled rpc modules """
self.registered_modules: List[RPCHandler] = []
@@ -36,15 +37,16 @@ class RPCManager:
if config.get('api_server', {}).get('enabled', False):
logger.info('Enabling rpc.api_server')
from freqtrade.rpc.api_server import ApiServer
self.registered_modules.append(ApiServer(self._rpc, config))
apiserver = ApiServer(config)
apiserver.add_rpc_handler(self._rpc)
self.registered_modules.append(apiserver)
def cleanup(self) -> None:
""" Stops all enabled rpc modules """
logger.info('Cleaning up rpc modules ...')
while self.registered_modules:
mod = self.registered_modules.pop()
logger.debug('Cleaning up rpc.%s ...', mod.name)
logger.info('Cleaning up rpc.%s ...', mod.name)
mod.cleanup()
del mod

301
freqtrade/rpc/telegram.py
View File

@@ -10,13 +10,13 @@ from datetime import date, datetime, timedelta
from html import escape
from itertools import chain
from math import isnan
from typing import Any, Callable, Dict, List, Optional, Union, cast
from typing import Any, Callable, Dict, List, Optional, Union
import arrow
from tabulate import tabulate
from telegram import (InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ParseMode,
ReplyKeyboardMarkup, Update)
from telegram.error import NetworkError, TelegramError
from telegram import (CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton,
ParseMode, ReplyKeyboardMarkup, Update)
from telegram.error import BadRequest, NetworkError, TelegramError
from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater
from telegram.utils.helpers import escape_markdown
@@ -24,7 +24,8 @@ from freqtrade.__init__ import __version__
from freqtrade.constants import DUST_PER_COIN
from freqtrade.enums import RPCMessageType
from freqtrade.exceptions import OperationalException
from freqtrade.misc import chunks, round_coin_value
from freqtrade.misc import chunks, plural, round_coin_value
from freqtrade.persistence import Trade
from freqtrade.rpc import RPC, RPCException, RPCHandler
@@ -47,16 +48,21 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]:
update = kwargs.get('update') or args[0]
# Reject unauthorized messages
chat_id = int(self._config['telegram']['chat_id'])
if update.callback_query:
cchat_id = int(update.callback_query.message.chat.id)
else:
cchat_id = int(update.message.chat_id)
if int(update.message.chat_id) != chat_id:
chat_id = int(self._config['telegram']['chat_id'])
if cchat_id != chat_id:
logger.info(
'Rejected unauthorized message from: %s',
update.message.chat_id
)
return wrapper
logger.info(
# Rollback session to avoid getting data stored in a transaction.
Trade.query.session.rollback()
logger.debug(
'Executing handler: %s for chat_id: %s',
command_handler.__name__,
chat_id
@@ -73,7 +79,6 @@ class Telegram(RPCHandler):
""" This class handles all telegram communication """
def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None:
"""
Init the Telegram call, and init the super class RPCHandler
:param rpc: instance of RPC Helper class
@@ -91,7 +96,7 @@ class Telegram(RPCHandler):
Validates the keyboard configuration from telegram config
section.
"""
self._keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = [
self._keyboard: List[List[Union[str, KeyboardButton]]] = [
['/daily', '/profit', '/balance'],
['/status', '/status table', '/performance'],
['/count', '/start', '/stop', '/help']
@@ -99,7 +104,7 @@ class Telegram(RPCHandler):
# do not allow commands with mandatory arguments and critical cmds
# like /forcesell and /forcebuy
# TODO: DRY! - its not good to list all valid cmds here. But otherwise
# this needs refacoring of the whole telegram module (same
# this needs refactoring of the whole telegram module (same
# problem in _help()).
valid_keys: List[str] = [r'/start$', r'/stop$', r'/status$', r'/status table$',
r'/trades$', r'/performance$', r'/daily$', r'/daily \d+$',
@@ -164,8 +169,21 @@ class Telegram(RPCHandler):
CommandHandler('help', self._help),
CommandHandler('version', self._version),
]
callbacks = [
CallbackQueryHandler(self._status_table, pattern='update_status_table'),
CallbackQueryHandler(self._daily, pattern='update_daily'),
CallbackQueryHandler(self._profit, pattern='update_profit'),
CallbackQueryHandler(self._balance, pattern='update_balance'),
CallbackQueryHandler(self._performance, pattern='update_performance'),
CallbackQueryHandler(self._count, pattern='update_count'),
CallbackQueryHandler(self._forcebuy_inline),
]
for handle in handles:
self._updater.dispatcher.add_handler(handle)
for callback in callbacks:
self._updater.dispatcher.add_handler(callback)
self._updater.start_polling(
bootstrap_retries=-1,
timeout=30,
@@ -177,11 +195,6 @@ class Telegram(RPCHandler):
[h.command for h in handles]
)
self._current_callback_query_handler: Optional[CallbackQueryHandler] = None
self._callback_query_handlers = {
'forcebuy': CallbackQueryHandler(self._forcebuy_inline)
}
def cleanup(self) -> None:
"""
Stops all running telegram threads.
@@ -196,15 +209,25 @@ class Telegram(RPCHandler):
else:
msg['stake_amount_fiat'] = 0
message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
f" (#{msg['trade_id']})\n"
f"*Amount:* `{msg['amount']:.8f}`\n"
f"*Open Rate:* `{msg['limit']:.8f}`\n"
f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
content = []
content.append(
f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
f" (#{msg['trade_id']})\n"
)
if msg.get('buy_tag', None):
content.append(f"*Buy Tag:* `{msg['buy_tag']}`\n")
content.append(f"*Amount:* `{msg['amount']:.8f}`\n")
content.append(f"*Open Rate:* `{msg['limit']:.8f}`\n")
content.append(f"*Current Rate:* `{msg['current_rate']:.8f}`\n")
content.append(
f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}"
)
if msg.get('fiat_currency', None):
message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
content.append(
f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
)
message = ''.join(content)
message += ")`"
return message
@@ -239,29 +262,7 @@ class Telegram(RPCHandler):
return message
def send_msg(self, msg: Dict[str, Any]) -> None:
""" Send a message to telegram channel """
default_noti = 'on'
msg_type = msg['type']
noti = ''
if msg_type == RPCMessageType.SELL:
sell_noti = self._config['telegram'] \
.get('notification_settings', {}).get(str(msg_type), {})
# For backward compatibility sell still can be string
if isinstance(sell_noti, str):
noti = sell_noti
else:
noti = sell_noti.get(str(msg['sell_reason']), default_noti)
else:
noti = self._config['telegram'] \
.get('notification_settings', {}).get(str(msg_type), default_noti)
if noti == 'off':
logger.info(f"Notification '{msg_type}' not sent.")
# Notification disabled
return
def compose_message(self, msg: Dict[str, Any], msg_type: RPCMessageType) -> str:
if msg_type == RPCMessageType.BUY:
message = self._format_buy_msg(msg)
@@ -282,7 +283,16 @@ class Telegram(RPCHandler):
"for {close_rate}.".format(**msg))
elif msg_type == RPCMessageType.SELL:
message = self._format_sell_msg(msg)
elif msg_type == RPCMessageType.PROTECTION_TRIGGER:
message = (
"*Protection* triggered due to {reason}. "
"`{pair}` will be locked until `{lock_end_time}`."
).format(**msg)
elif msg_type == RPCMessageType.PROTECTION_TRIGGER_GLOBAL:
message = (
"*Protection* triggered due to {reason}. "
"*All pairs* will be locked until `{lock_end_time}`."
).format(**msg)
elif msg_type == RPCMessageType.STATUS:
message = '*Status:* `{status}`'.format(**msg)
@@ -294,6 +304,33 @@ class Telegram(RPCHandler):
else:
raise NotImplementedError('Unknown message type: {}'.format(msg_type))
return message
def send_msg(self, msg: Dict[str, Any]) -> None:
""" Send a message to telegram channel """
default_noti = 'on'
msg_type = msg['type']
noti = ''
if msg_type == RPCMessageType.SELL:
sell_noti = self._config['telegram'] \
.get('notification_settings', {}).get(str(msg_type), {})
# For backward compatibility sell still can be string
if isinstance(sell_noti, str):
noti = sell_noti
else:
noti = sell_noti.get(str(msg['sell_reason']), default_noti)
else:
noti = self._config['telegram'] \
.get('notification_settings', {}).get(str(msg_type), default_noti)
if noti == 'off':
logger.info(f"Notification '{msg_type}' not sent.")
# Notification disabled
return
message = self.compose_message(msg, msg_type)
self._send_msg(message, disable_notification=(noti == 'silent'))
@@ -342,6 +379,7 @@ class Telegram(RPCHandler):
"*Trade ID:* `{trade_id}` `(since {open_date_hum})`",
"*Current Pair:* {pair}",
"*Amount:* `{amount} ({stake_amount} {base_currency})`",
"*Buy Tag:* `{buy_tag}`" if r['buy_tag'] else "",
"*Open Rate:* `{open_rate:.8f}`",
"*Close Rate:* `{close_rate}`" if r['close_rate'] else "",
"*Current Rate:* `{current_rate:.8f}`",
@@ -409,7 +447,9 @@ class Telegram(RPCHandler):
# insert separators line between Total
lines = message.split("\n")
message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]])
self._send_msg(f"<pre>{message}</pre>", parse_mode=ParseMode.HTML)
self._send_msg(f"<pre>{message}</pre>", parse_mode=ParseMode.HTML,
reload_able=True, callback_path="update_status_table",
query=update.callback_query)
except RPCException as e:
self._send_msg(str(e))
@@ -447,7 +487,8 @@ class Telegram(RPCHandler):
],
tablefmt='simple')
message = f'<b>Daily Profit over the last {timescale} days</b>:\n<pre>{stats_tab}</pre>'
self._send_msg(message, parse_mode=ParseMode.HTML)
self._send_msg(message, parse_mode=ParseMode.HTML, reload_able=True,
callback_path="update_daily", query=update.callback_query)
except RPCException as e:
self._send_msg(str(e))
@@ -467,7 +508,7 @@ class Telegram(RPCHandler):
timescale = None
try:
if context.args:
timescale = int(context.args[0])
timescale = int(context.args[0]) - 1
today_start = datetime.combine(date.today(), datetime.min.time())
start_date = today_start - timedelta(days=timescale)
except (TypeError, ValueError, IndexError):
@@ -479,11 +520,11 @@ class Telegram(RPCHandler):
start_date)
profit_closed_coin = stats['profit_closed_coin']
profit_closed_percent_mean = stats['profit_closed_percent_mean']
profit_closed_percent_sum = stats['profit_closed_percent_sum']
profit_closed_percent = stats['profit_closed_percent']
profit_closed_fiat = stats['profit_closed_fiat']
profit_all_coin = stats['profit_all_coin']
profit_all_percent_mean = stats['profit_all_percent_mean']
profit_all_percent_sum = stats['profit_all_percent_sum']
profit_all_percent = stats['profit_all_percent']
profit_all_fiat = stats['profit_all_fiat']
trade_count = stats['trade_count']
first_trade_date = stats['first_trade_date']
@@ -499,7 +540,7 @@ class Telegram(RPCHandler):
markdown_msg = ("*ROI:* Closed trades\n"
f"∙ `{round_coin_value(profit_closed_coin, stake_cur)} "
f"({profit_closed_percent_mean:.2f}%) "
f"({profit_closed_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"({profit_closed_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"∙ `{round_coin_value(profit_closed_fiat, fiat_disp_cur)}`\n")
else:
markdown_msg = "`No closed trade` \n"
@@ -508,18 +549,19 @@ class Telegram(RPCHandler):
f"*ROI:* All trades\n"
f"∙ `{round_coin_value(profit_all_coin, stake_cur)} "
f"({profit_all_percent_mean:.2f}%) "
f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"({profit_all_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n"
f"*Total Trade Count:* `{trade_count}`\n"
f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* "
f"`{first_trade_date}`\n"
f"*Latest Trade opened:* `{latest_trade_date}\n`"
f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`"
)
)
if stats['closed_trade_count'] > 0:
markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n"
f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`")
self._send_msg(markdown_msg)
self._send_msg(markdown_msg, reload_able=True, callback_path="update_profit",
query=update.callback_query)
@authorized_only
def _stats(self, update: Update, context: CallbackContext) -> None:
@@ -549,13 +591,14 @@ class Telegram(RPCHandler):
sell_reasons_msg = tabulate(
sell_reasons_tabulate,
headers=['Sell Reason', 'Sells', 'Wins', 'Losses']
)
)
durations = stats['durations']
duration_msg = tabulate([
['Wins', str(timedelta(seconds=durations['wins']))
if durations['wins'] != 'N/A' else 'N/A'],
['Losses', str(timedelta(seconds=durations['losses']))
if durations['losses'] != 'N/A' else 'N/A']
duration_msg = tabulate(
[
['Wins', str(timedelta(seconds=durations['wins']))
if durations['wins'] != 'N/A' else 'N/A'],
['Losses', str(timedelta(seconds=durations['losses']))
if durations['losses'] != 'N/A' else 'N/A']
],
headers=['', 'Avg. Duration']
)
@@ -576,13 +619,19 @@ class Telegram(RPCHandler):
output = ''
if self._config['dry_run']:
output += (
f"*Warning:* Simulated balances in Dry Mode.\n"
"This mode is still experimental!\n"
"Starting capital: "
f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n"
)
output += "*Warning:* Simulated balances in Dry Mode.\n"
output += ("Starting capital: "
f"`{result['starting_capital']}` {self._config['stake_currency']}"
)
output += (f" `{result['starting_capital_fiat']}` "
f"{self._config['fiat_display_currency']}.\n"
) if result['starting_capital_fiat'] > 0 else '.\n'
total_dust_balance = 0
total_dust_currencies = 0
for curr in result['currencies']:
curr_output = ''
if curr['est_stake'] > balance_dust_level:
curr_output = (
f"*{curr['currency']}:*\n"
@@ -591,22 +640,34 @@ class Telegram(RPCHandler):
f"\t`Pending: {curr['used']:.8f}`\n"
f"\t`Est. {curr['stake']}: "
f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n")
else:
curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} "
f"{curr['stake']} amount \n")
elif curr['est_stake'] <= balance_dust_level:
total_dust_balance += curr['est_stake']
total_dust_currencies += 1
# Handle overflowing messsage length
# Handle overflowing message length
if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH:
self._send_msg(output)
output = curr_output
else:
output += curr_output
if total_dust_balance > 0:
output += (
f"*{total_dust_currencies} Other "
f"{plural(total_dust_currencies, 'Currency', 'Currencies')} "
f"(< {balance_dust_level} {result['stake']}):*\n"
f"\t`Est. {result['stake']}: "
f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n")
output += ("\n*Estimated Value*:\n"
f"\t`{result['stake']}: {result['total']: .8f}`\n"
f"\t`{result['stake']}: "
f"{round_coin_value(result['total'], result['stake'], False)}`"
f" `({result['starting_capital_pct']}%)`\n"
f"\t`{result['symbol']}: "
f"{round_coin_value(result['value'], result['symbol'], False)}`\n")
self._send_msg(output)
f"{round_coin_value(result['value'], result['symbol'], False)}`"
f" `({result['starting_capital_fiat_pct']}%)`\n")
self._send_msg(output, reload_able=True, callback_path="update_balance",
query=update.callback_query)
except RPCException as e:
self._send_msg(str(e))
@@ -713,10 +774,10 @@ class Telegram(RPCHandler):
self._forcebuy_action(pair, price)
else:
whitelist = self._rpc._rpc_whitelist()['whitelist']
pairs = [InlineKeyboardButton(pair, callback_data=pair) for pair in whitelist]
self._send_inline_msg("Which pair?",
keyboard=self._layout_inline_keyboard(pairs),
callback_query_handler='forcebuy')
pairs = [InlineKeyboardButton(text=pair, callback_data=pair) for pair in whitelist]
self._send_msg(msg="Which pair?",
keyboard=self._layout_inline_keyboard(pairs))
@authorized_only
def _trades(self, update: Update, context: CallbackContext) -> None:
@@ -800,7 +861,9 @@ class Telegram(RPCHandler):
else:
output += stat_line
self._send_msg(output, parse_mode=ParseMode.HTML)
self._send_msg(output, parse_mode=ParseMode.HTML,
reload_able=True, callback_path="update_performance",
query=update.callback_query)
except RPCException as e:
self._send_msg(str(e))
@@ -820,7 +883,9 @@ class Telegram(RPCHandler):
tablefmt='simple')
message = "<pre>{}</pre>".format(message)
logger.debug(message)
self._send_msg(message, parse_mode=ParseMode.HTML)
self._send_msg(message, parse_mode=ParseMode.HTML,
reload_able=True, callback_path="update_count",
query=update.callback_query)
except RPCException as e:
self._send_msg(str(e))
@@ -968,7 +1033,8 @@ class Telegram(RPCHandler):
:return: None
"""
forcebuy_text = ("*/forcebuy <pair> [<rate>]:* `Instantly buys the given pair. "
"Optionally takes a rate at which to buy.` \n")
"Optionally takes a rate at which to buy "
"(only applies to limit orders).` \n")
message = ("*/start:* `Starts the trader`\n"
"*/stop:* `Stops the trader`\n"
"*/status <trade_id>|[table]:* `Lists all open trades`\n"
@@ -1052,29 +1118,42 @@ class Telegram(RPCHandler):
f"*Current state:* `{val['state']}`"
)
def _send_inline_msg(self, msg: str, callback_query_handler,
parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False,
keyboard: List[List[InlineKeyboardButton]] = None, ) -> None:
"""
Send given markdown message
:param msg: message
:param bot: alternative bot
:param parse_mode: telegram parse mode
:return: None
"""
if self._current_callback_query_handler:
self._updater.dispatcher.remove_handler(self._current_callback_query_handler)
self._current_callback_query_handler = self._callback_query_handlers[callback_query_handler]
self._updater.dispatcher.add_handler(self._current_callback_query_handler)
def _update_msg(self, query: CallbackQuery, msg: str, callback_path: str = "",
reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None:
if reload_able:
reply_markup = InlineKeyboardMarkup([
[InlineKeyboardButton("Refresh", callback_data=callback_path)],
])
else:
reply_markup = InlineKeyboardMarkup([[]])
msg += "\nUpdated: {}".format(datetime.now().ctime())
if not query.message:
return
chat_id = query.message.chat_id
message_id = query.message.message_id
self._send_msg(msg, parse_mode, disable_notification,
cast(List[List[Union[str, KeyboardButton, InlineKeyboardButton]]], keyboard),
reply_markup=InlineKeyboardMarkup)
try:
self._updater.bot.edit_message_text(
chat_id=chat_id,
message_id=message_id,
text=msg,
parse_mode=parse_mode,
reply_markup=reply_markup
)
except BadRequest as e:
if 'not modified' in e.message.lower():
pass
else:
logger.warning('TelegramError: %s', e.message)
except TelegramError as telegram_err:
logger.warning('TelegramError: %s! Giving up on that message.', telegram_err.message)
def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN,
disable_notification: bool = False,
keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None,
reply_markup=ReplyKeyboardMarkup) -> None:
keyboard: List[List[InlineKeyboardButton]] = None,
callback_path: str = "",
reload_able: bool = False,
query: Optional[CallbackQuery] = None) -> None:
"""
Send given markdown message
:param msg: message
@@ -1082,9 +1161,19 @@ class Telegram(RPCHandler):
:param parse_mode: telegram parse mode
:return: None
"""
if keyboard is None:
keyboard = self._keyboard
reply_markup = reply_markup(keyboard, resize_keyboard=True)
reply_markup: Union[InlineKeyboardMarkup, ReplyKeyboardMarkup]
if query:
self._update_msg(query=query, msg=msg, parse_mode=parse_mode,
callback_path=callback_path, reload_able=reload_able)
return
if reload_able and self._config['telegram'].get('reload', True):
reply_markup = InlineKeyboardMarkup([
[InlineKeyboardButton("Refresh", callback_data=callback_path)]])
else:
if keyboard is not None:
reply_markup = InlineKeyboardMarkup(keyboard, resize_keyboard=True)
else:
reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True)
try:
try:
self._updater.bot.send_message(

15
freqtrade/rpc/webhook.py
View File

@@ -77,14 +77,13 @@ class Webhook(RPCHandler):
def _send_msg(self, payload: dict) -> None:
"""do the actual call to the webhook"""
if self._format == 'form':
kwargs = {'data': payload}
elif self._format == 'json':
kwargs = {'json': payload}
else:
raise NotImplementedError('Unknown format: {}'.format(self._format))
try:
post(self._url, **kwargs)
if self._format == 'form':
post(self._url, data=payload)
elif self._format == 'json':
post(self._url, json=payload)
else:
raise NotImplementedError('Unknown format: {}'.format(self._format))
except RequestException as exc:
logger.warning("Could not call webhook url. Exception: %s", exc)

8
freqtrade/strategy/__init__.py
View File

@@ -1,7 +1,9 @@
# flake8: noqa: F401
from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date,
timeframe_to_prev_date, timeframe_to_seconds)
from freqtrade.strategy.hyper import (CategoricalParameter, DecimalParameter, IntParameter,
RealParameter)
from freqtrade.strategy.hyper import (BooleanParameter, CategoricalParameter, DecimalParameter,
IntParameter, RealParameter)
from freqtrade.strategy.informative_decorator import informative
from freqtrade.strategy.interface import IStrategy
from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open
from freqtrade.strategy.strategy_helper import (merge_informative_pair, stoploss_from_absolute,
stoploss_from_open)

110
freqtrade/strategy/hyper.py
View File

@@ -5,8 +5,10 @@ This module defines a base class for auto-hyperoptable strategies.
import logging
from abc import ABC, abstractmethod
from contextlib import suppress
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union
from freqtrade.misc import deep_merge_dicts, json_load
from freqtrade.optimize.hyperopt_tools import HyperoptTools
@@ -205,6 +207,21 @@ class DecimalParameter(NumericParameter):
return SKDecimal(low=self.low, high=self.high, decimals=self._decimals, name=name,
**self._space_params)
@property
def range(self):
"""
Get each value in this space as list.
Returns a List from low to high (inclusive) in Hyperopt mode.
Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid
calculating 100ds of indicators.
"""
if self.in_space and self.optimize:
low = int(self.low * pow(10, self._decimals))
high = int(self.high * pow(10, self._decimals)) + 1
return [round(n * pow(0.1, self._decimals), self._decimals) for n in range(low, high)]
else:
return [self.value]
class CategoricalParameter(BaseParameter):
default: Any
@@ -239,10 +256,45 @@ class CategoricalParameter(BaseParameter):
"""
return Categorical(self.opt_range, name=name, **self._space_params)
@property
def range(self):
"""
Get each value in this space as list.
Returns a List of categories in Hyperopt mode.
Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid
calculating 100ds of indicators.
"""
if self.in_space and self.optimize:
return self.opt_range
else:
return [self.value]
class BooleanParameter(CategoricalParameter):
def __init__(self, *, default: Optional[Any] = None,
space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
"""
Initialize hyperopt-optimizable Boolean Parameter.
It's a shortcut to `CategoricalParameter([True, False])`.
:param default: A default value. If not specified, first item from specified space will be
used.
:param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
parameter field
name is prefixed with 'buy_' or 'sell_'.
:param optimize: Include parameter in hyperopt optimizations.
:param load: Load parameter value from {space}_params.
:param kwargs: Extra parameters to skopt.space.Categorical.
"""
categories = [True, False]
super().__init__(categories=categories, default=default, space=space, optimize=optimize,
load=load, **kwargs)
class HyperStrategyMixin(object):
"""
A helper base class which allows HyperOptAuto class to reuse implementations of of buy/sell
A helper base class which allows HyperOptAuto class to reuse implementations of buy/sell
strategy logic.
"""
@@ -253,20 +305,22 @@ class HyperStrategyMixin(object):
self.config = config
self.ft_buy_params: List[BaseParameter] = []
self.ft_sell_params: List[BaseParameter] = []
self.ft_protection_params: List[BaseParameter] = []
self._load_hyper_params(config.get('runmode') == RunMode.HYPEROPT)
def enumerate_parameters(self, category: str = None) -> Iterator[Tuple[str, BaseParameter]]:
"""
Find all optimizeable parameters and return (name, attr) iterator.
Find all optimizable parameters and return (name, attr) iterator.
:param category:
:return:
"""
if category not in ('buy', 'sell', None):
raise OperationalException('Category must be one of: "buy", "sell", None.')
if category not in ('buy', 'sell', 'protection', None):
raise OperationalException(
'Category must be one of: "buy", "sell", "protection", None.')
if category is None:
params = self.ft_buy_params + self.ft_sell_params
params = self.ft_buy_params + self.ft_sell_params + self.ft_protection_params
else:
params = getattr(self, f"ft_{category}_params")
@@ -294,9 +348,10 @@ class HyperStrategyMixin(object):
params: Dict = {
'buy': list(cls.detect_parameters('buy')),
'sell': list(cls.detect_parameters('sell')),
'protection': list(cls.detect_parameters('protection')),
}
params.update({
'count': len(params['buy'] + params['sell'])
'count': len(params['buy'] + params['sell'] + params['protection'])
})
return params
@@ -305,12 +360,42 @@ class HyperStrategyMixin(object):
"""
Load Hyperoptable parameters
"""
self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt)
self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt)
params = self.load_params_from_file()
params = params.get('params', {})
self._ft_params_from_file = params
buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', {}))
sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', {}))
protection_params = deep_merge_dicts(params.get('protection', {}),
getattr(self, 'protection_params', {}))
def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None:
self._load_params(buy_params, 'buy', hyperopt)
self._load_params(sell_params, 'sell', hyperopt)
self._load_params(protection_params, 'protection', hyperopt)
def load_params_from_file(self) -> Dict:
filename_str = getattr(self, '__file__', '')
if not filename_str:
return {}
filename = Path(filename_str).with_suffix('.json')
if filename.is_file():
logger.info(f"Loading parameters from file {filename}")
try:
with filename.open('r') as f:
params = json_load(f)
if params.get('strategy_name') != self.__class__.__name__:
raise OperationalException('Invalid parameter file provided.')
return params
except ValueError:
logger.warning("Invalid parameter file format.")
return {}
logger.info("Found no parameter file.")
return {}
def _load_params(self, params: Dict, space: str, hyperopt: bool = False) -> None:
"""
Set optimizeable parameter values.
Set optimizable parameter values.
:param params: Dictionary with new parameter values.
"""
if not params:
@@ -335,13 +420,14 @@ class HyperStrategyMixin(object):
else:
logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}')
def get_params_dict(self):
def get_no_optimize_params(self):
"""
Returns list of Parameters that are not part of the current optimize job
"""
params = {
'buy': {},
'sell': {}
'sell': {},
'protection': {},
}
for name, p in self.enumerate_parameters():
if not p.optimize or not p.in_space:

128
freqtrade/strategy/informative_decorator.py Normal file
View File

@@ -0,0 +1,128 @@
from typing import Any, Callable, NamedTuple, Optional, Union
from pandas import DataFrame
from freqtrade.exceptions import OperationalException
from freqtrade.strategy.strategy_helper import merge_informative_pair
PopulateIndicators = Callable[[Any, DataFrame, dict], DataFrame]
class InformativeData(NamedTuple):
asset: Optional[str]
timeframe: str
fmt: Union[str, Callable[[Any], str], None]
ffill: bool
def informative(timeframe: str, asset: str = '',
fmt: Optional[Union[str, Callable[[Any], str]]] = None,
ffill: bool = True) -> Callable[[PopulateIndicators], PopulateIndicators]:
"""
A decorator for populate_indicators_Nn(self, dataframe, metadata), allowing these functions to
define informative indicators.
Example usage:
@informative('1h')
def populate_indicators_1h(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
return dataframe
:param timeframe: Informative timeframe. Must always be equal or higher than strategy timeframe.
:param asset: Informative asset, for example BTC, BTC/USDT, ETH/BTC. Do not specify to use
current pair.
:param fmt: Column format (str) or column formatter (callable(name, asset, timeframe)). When not
specified, defaults to:
* {base}_{quote}_{column}_{timeframe} if asset is specified.
* {column}_{timeframe} if asset is not specified.
Format string supports these format variables:
* {asset} - full name of the asset, for example 'BTC/USDT'.
* {base} - base currency in lower case, for example 'eth'.
* {BASE} - same as {base}, except in upper case.
* {quote} - quote currency in lower case, for example 'usdt'.
* {QUOTE} - same as {quote}, except in upper case.
* {column} - name of dataframe column.
* {timeframe} - timeframe of informative dataframe.
:param ffill: ffill dataframe after merging informative pair.
"""
_asset = asset
_timeframe = timeframe
_fmt = fmt
_ffill = ffill
def decorator(fn: PopulateIndicators):
informative_pairs = getattr(fn, '_ft_informative', [])
informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill))
setattr(fn, '_ft_informative', informative_pairs)
return fn
return decorator
def _format_pair_name(config, pair: str) -> str:
return pair.format(stake_currency=config['stake_currency'],
stake=config['stake_currency']).upper()
def _create_and_merge_informative_pair(strategy, dataframe: DataFrame, metadata: dict,
inf_data: InformativeData,
populate_indicators: PopulateIndicators):
asset = inf_data.asset or ''
timeframe = inf_data.timeframe
fmt = inf_data.fmt
config = strategy.config
if asset:
# Insert stake currency if needed.
asset = _format_pair_name(config, asset)
else:
# Not specifying an asset will define informative dataframe for current pair.
asset = metadata['pair']
if '/' in asset:
base, quote = asset.split('/')
else:
# When futures are supported this may need reevaluation.
# base, quote = asset, ''
raise OperationalException('Not implemented.')
# Default format. This optimizes for the common case: informative pairs using same stake
# currency. When quote currency matches stake currency, column name will omit base currency.
# This allows easily reconfiguring strategy to use different base currency. In a rare case
# where it is desired to keep quote currency in column name at all times user should specify
# fmt='{base}_{quote}_{column}_{timeframe}' format or similar.
if not fmt:
fmt = '{column}_{timeframe}' # Informatives of current pair
if inf_data.asset:
fmt = '{base}_{quote}_' + fmt # Informatives of other pairs
inf_metadata = {'pair': asset, 'timeframe': timeframe}
inf_dataframe = strategy.dp.get_pair_dataframe(asset, timeframe)
inf_dataframe = populate_indicators(strategy, inf_dataframe, inf_metadata)
formatter: Any = None
if callable(fmt):
formatter = fmt # A custom user-specified formatter function.
else:
formatter = fmt.format # A default string formatter.
fmt_args = {
'BASE': base.upper(),
'QUOTE': quote.upper(),
'base': base.lower(),
'quote': quote.lower(),
'asset': asset,
'timeframe': timeframe,
}
inf_dataframe.rename(columns=lambda column: formatter(column=column, **fmt_args),
inplace=True)
date_column = formatter(column='date', **fmt_args)
if date_column in dataframe.columns:
raise OperationalException(f'Duplicate column name {date_column} exists in '
f'dataframe! Ensure column names are unique!')
dataframe = merge_informative_pair(dataframe, inf_dataframe, strategy.timeframe, timeframe,
ffill=inf_data.ffill, append_timeframe=False,
date_column=date_column)
return dataframe

216
freqtrade/strategy/interface.py
View File

@@ -13,12 +13,15 @@ from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import SellType, SignalType
from freqtrade.enums import SellType, SignalTagType, SignalType
from freqtrade.exceptions import OperationalException, StrategyError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.exchange.exchange import timeframe_to_next_date
from freqtrade.persistence import PairLocks, Trade
from freqtrade.strategy.hyper import HyperStrategyMixin
from freqtrade.strategy.informative_decorator import (InformativeData, PopulateIndicators,
_create_and_merge_informative_pair,
_format_pair_name)
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
@@ -62,8 +65,9 @@ class IStrategy(ABC, HyperStrategyMixin):
_populate_fun_len: int = 0
_buy_fun_len: int = 0
_sell_fun_len: int = 0
_ft_params_from_file: Dict
# associated minimal roi
minimal_roi: Dict
minimal_roi: Dict = {}
# associated stoploss
stoploss: float
@@ -97,6 +101,11 @@ class IStrategy(ABC, HyperStrategyMixin):
# run "populate_indicators" only for new candle
process_only_new_candles: bool = False
use_sell_signal: bool
sell_profit_only: bool
sell_profit_offset: float
ignore_roi_if_buy_signal: bool
# Number of seconds after which the candle will no longer result in a buy on expired candles
ignore_buying_expired_candle_after: int = 0
@@ -107,13 +116,15 @@ class IStrategy(ABC, HyperStrategyMixin):
startup_candle_count: int = 0
# Protections
protections: List
protections: List = []
# Class level variables (intentional) containing
# the dataprovider (dp) (access to other candles, historic data, ...)
# and wallets - access to the current balance.
dp: Optional[DataProvider] = None
dp: Optional[DataProvider]
wallets: Optional[Wallets] = None
# Filled from configuration
stake_currency: str
# container variable for strategy source code
__source__: str = ''
@@ -126,6 +137,24 @@ class IStrategy(ABC, HyperStrategyMixin):
self._last_candle_seen_per_pair: Dict[str, datetime] = {}
super().__init__(config)
# Gather informative pairs from @informative-decorated methods.
self._ft_informative: List[Tuple[InformativeData, PopulateIndicators]] = []
for attr_name in dir(self.__class__):
cls_method = getattr(self.__class__, attr_name)
if not callable(cls_method):
continue
informative_data_list = getattr(cls_method, '_ft_informative', None)
if not isinstance(informative_data_list, list):
# Type check is required because mocker would return a mock object that evaluates to
# True, confusing this code.
continue
strategy_timeframe_minutes = timeframe_to_minutes(self.timeframe)
for informative_data in informative_data_list:
if timeframe_to_minutes(informative_data.timeframe) < strategy_timeframe_minutes:
raise OperationalException('Informative timeframe must be equal or higher than '
'strategy timeframe!')
self._ft_informative.append((informative_data, cls_method))
@abstractmethod
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
@@ -270,10 +299,47 @@ class IStrategy(ABC, HyperStrategyMixin):
:param current_rate: Rate, calculated based on pricing settings in ask_strategy.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New stoploss value, relative to the currentrate
:return float: New stoploss value, relative to the current_rate
"""
return self.stoploss
def custom_entry_price(self, pair: str, current_time: datetime, proposed_rate: float,
**kwargs) -> float:
"""
Custom entry price logic, returning the new entry price.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns None, orderbook is used to set entry price
:param pair: Pair that's currently analyzed
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in ask_strategy.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New entry price value if provided
"""
return proposed_rate
def custom_exit_price(self, pair: str, trade: Trade,
current_time: datetime, proposed_rate: float,
current_profit: float, **kwargs) -> float:
"""
Custom exit price logic, returning the new exit price.
For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/
When not implemented by a strategy, returns None, orderbook is used to set exit price
:param pair: Pair that's currently analyzed
:param trade: trade object.
:param current_time: datetime object, containing the current datetime
:param proposed_rate: Rate, calculated based on pricing settings in ask_strategy.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New exit price value if provided
"""
return proposed_rate
def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float,
current_profit: float, **kwargs) -> Optional[Union[str, bool]]:
"""
@@ -282,10 +348,10 @@ class IStrategy(ABC, HyperStrategyMixin):
time. This method is not called when sell signal is set.
This method should be overridden to create sell signals that depend on trade parameters. For
example you could implement a stoploss relative to candle when trade was opened, or a custom
1:2 risk-reward ROI.
example you could implement a sell relative to the candle when the trade was opened,
or a custom 1:2 risk-reward ROI.
Custom sell reason max length is 64. Exceeding this limit will raise OperationalException.
Custom sell reason max length is 64. Exceeding characters will be removed.
:param pair: Pair that's currently analyzed
:param trade: trade object.
@@ -298,10 +364,27 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
return None
def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float,
proposed_stake: float, min_stake: float, max_stake: float,
**kwargs) -> float:
"""
Customize stake size for each new trade. This method is not called when edge module is
enabled.
:param pair: Pair that's currently analyzed
:param current_time: datetime object, containing the current datetime
:param current_rate: Rate, calculated based on pricing settings in ask_strategy.
:param proposed_stake: A stake amount proposed by the bot.
:param min_stake: Minimal stake size allowed by exchange.
:param max_stake: Balance available for trading.
:return: A stake size, which is between min_stake and max_stake.
"""
return proposed_stake
def informative_pairs(self) -> ListPairsWithTimeframes:
"""
Define additional, informative pair/interval combinations to be cached from the exchange.
These pair/interval combinations are non-tradeable, unless they are part
These pair/interval combinations are non-tradable, unless they are part
of the whitelist as well.
For more information, please consult the documentation
:return: List of tuples in the format (pair, interval)
@@ -315,6 +398,23 @@ class IStrategy(ABC, HyperStrategyMixin):
# END - Intended to be overridden by strategy
###
def gather_informative_pairs(self) -> ListPairsWithTimeframes:
"""
Internal method which gathers all informative pairs (user or automatically defined).
"""
informative_pairs = self.informative_pairs()
for inf_data, _ in self._ft_informative:
if inf_data.asset:
pair_tf = (_format_pair_name(self.config, inf_data.asset), inf_data.timeframe)
informative_pairs.append(pair_tf)
else:
if not self.dp:
raise OperationalException('@informative decorator with unspecified asset '
'requires DataProvider instance.')
for pair in self.dp.current_whitelist():
informative_pairs.append((pair, inf_data.timeframe))
return list(set(informative_pairs))
def get_strategy_name(self) -> str:
"""
Returns strategy class name
@@ -343,13 +443,22 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
PairLocks.unlock_pair(pair, datetime.now(timezone.utc))
def unlock_reason(self, reason: str) -> None:
"""
Unlocks all pairs previously locked using lock_pair with specified reason.
Not used by freqtrade itself, but intended to be used if users lock pairs
manually from within the strategy, to allow an easy way to unlock pairs.
:param reason: Unlock pairs to allow trading again
"""
PairLocks.unlock_reason(reason, datetime.now(timezone.utc))
def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
"""
Checks if a pair is currently locked
The 2nd, optional parameter ensures that locks are applied until the new candle arrives,
and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap
of 2 seconds for a buy to happen on an old signal.
:param: pair: "Pair to check"
:param pair: "Pair to check"
:param candle_date: Date of the last candle. Optional, defaults to current date
:returns: locking state of the pair in question.
"""
@@ -399,6 +508,7 @@ class IStrategy(ABC, HyperStrategyMixin):
logger.debug("Skipping TA Analysis for already analyzed candle")
dataframe['buy'] = 0
dataframe['sell'] = 0
dataframe['buy_tag'] = None
# Other Defs in strategy that want to be called every loop here
# twitter_sell = self.watch_twitter_feed(dataframe, metadata)
@@ -453,20 +563,30 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
Ensure dataframe (length, last candle) was not modified, and has all elements we need.
"""
message_template = "Dataframe returned from strategy has mismatching {}."
message = ""
if df_len != len(dataframe):
message = "length"
if dataframe is None:
message = "No dataframe returned (return statement missing?)."
elif 'buy' not in dataframe:
message = "Buy column not set."
elif df_len != len(dataframe):
message = message_template.format("length")
elif df_close != dataframe["close"].iloc[-1]:
message = "last close price"
message = message_template.format("last close price")
elif df_date != dataframe["date"].iloc[-1]:
message = "last date"
message = message_template.format("last date")
if message:
if self.disable_dataframe_checks:
logger.warning(f"Dataframe returned from strategy has mismatching {message}.")
logger.warning(message)
else:
raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.")
raise StrategyError(message)
def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]:
def get_signal(
self,
pair: str,
timeframe: str,
dataframe: DataFrame
) -> Tuple[bool, bool, Optional[str]]:
"""
Calculates current signal based based on the buy / sell columns of the dataframe.
Used by Bot to get the signal to buy or sell
@@ -477,7 +597,7 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
return False, False
return False, False, None
latest_date = dataframe['date'].max()
latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
@@ -492,9 +612,16 @@ class IStrategy(ABC, HyperStrategyMixin):
'Outdated history for pair %s. Last tick is %s minutes old',
pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
)
return False, False
return False, False, None
buy = latest[SignalType.BUY.value] == 1
sell = False
if SignalType.SELL.value in latest:
sell = latest[SignalType.SELL.value] == 1
buy_tag = latest.get(SignalTagType.BUY_TAG.value, None)
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'], pair, str(buy), str(sell))
timeframe_seconds = timeframe_to_seconds(timeframe)
@@ -502,8 +629,8 @@ class IStrategy(ABC, HyperStrategyMixin):
current_time=datetime.now(timezone.utc),
timeframe_seconds=timeframe_seconds,
buy=buy):
return False, sell
return buy, sell
return False, sell, buy_tag
return buy, sell, buy_tag
def ignore_expired_candle(self, latest_date: datetime, current_time: datetime,
timeframe_seconds: int, buy: bool):
@@ -524,23 +651,21 @@ class IStrategy(ABC, HyperStrategyMixin):
:param force_stoploss: Externally provided stoploss
:return: True if trade should be sold, False otherwise
"""
# Set current rate to low for backtesting sell
current_rate = low or rate
current_rate = rate
current_profit = trade.calc_profit_ratio(current_rate)
trade.adjust_min_max_rates(high or current_rate)
trade.adjust_min_max_rates(high or current_rate, low or current_rate)
stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade,
current_time=date, current_profit=current_profit,
force_stoploss=force_stoploss, high=high)
force_stoploss=force_stoploss, low=low, high=high)
# Set current rate to high for backtesting sell
current_rate = high or rate
current_profit = trade.calc_profit_ratio(current_rate)
ask_strategy = self.config.get('ask_strategy', {})
# if buy signal and ignore_roi is set, we don't need to evaluate min_roi.
roi_reached = (not (buy and ask_strategy.get('ignore_roi_if_buy_signal', False))
roi_reached = (not (buy and self.ignore_roi_if_buy_signal)
and self.min_roi_reached(trade=trade, current_profit=current_profit,
current_time=date))
@@ -550,11 +675,10 @@ class IStrategy(ABC, HyperStrategyMixin):
current_rate = rate
current_profit = trade.calc_profit_ratio(current_rate)
if (ask_strategy.get('sell_profit_only', False)
and current_profit <= ask_strategy.get('sell_profit_offset', 0)):
if (self.sell_profit_only and current_profit <= self.sell_profit_offset):
# sell_profit_only and profit doesn't reach the offset - ignore sell signal
pass
elif ask_strategy.get('use_sell_signal', True) and not buy:
elif self.use_sell_signal and not buy:
if sell:
sell_signal = SellType.SELL_SIGNAL
else:
@@ -599,18 +723,21 @@ class IStrategy(ABC, HyperStrategyMixin):
def stop_loss_reached(self, current_rate: float, trade: Trade,
current_time: datetime, current_profit: float,
force_stoploss: float, high: float = None) -> SellCheckTuple:
force_stoploss: float, low: float = None,
high: float = None) -> SellCheckTuple:
"""
Based on current profit of the trade and configured (trailing) stoploss,
decides to sell or not
:param current_profit: current profit as ratio
:param low: Low value of this candle, only set in backtesting
:param high: High value of this candle, only set in backtesting
"""
stop_loss_value = force_stoploss if force_stoploss else self.stoploss
# Initiate stoploss with open_rate. Does nothing if stoploss is already set.
trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True)
if self.use_custom_stoploss:
if self.use_custom_stoploss and trade.stop_loss < (low or current_rate):
stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None
)(pair=trade.pair, trade=trade,
current_time=current_time,
@@ -623,7 +750,7 @@ class IStrategy(ABC, HyperStrategyMixin):
else:
logger.warning("CustomStoploss function did not return valid stoploss")
if self.trailing_stop:
if self.trailing_stop and trade.stop_loss < (low or current_rate):
# trailing stoploss handling
sl_offset = self.trailing_stop_positive_offset
@@ -643,7 +770,7 @@ class IStrategy(ABC, HyperStrategyMixin):
# evaluate if the stoploss was hit if stoploss is not on exchange
# in Dry-Run, this handles stoploss logic as well, as the logic will not be different to
# regular stoploss handling.
if ((trade.stop_loss >= current_rate) and
if ((trade.stop_loss >= (low or current_rate)) and
(not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])):
sell_type = SellType.STOP_LOSS
@@ -652,7 +779,7 @@ class IStrategy(ABC, HyperStrategyMixin):
if trade.initial_stop_loss != trade.stop_loss:
sell_type = SellType.TRAILING_STOP_LOSS
logger.debug(
f"{trade.pair} - HIT STOP: current price at {current_rate:.6f}, "
f"{trade.pair} - HIT STOP: current price at {(low or current_rate):.6f}, "
f"stoploss is {trade.stop_loss:.6f}, "
f"initial stoploss was at {trade.initial_stop_loss:.6f}, "
f"trade opened at {trade.open_rate:.6f}")
@@ -691,16 +818,17 @@ class IStrategy(ABC, HyperStrategyMixin):
else:
return current_profit > roi
def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
def advise_all_indicators(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]:
"""
Populates indicators for given candle (OHLCV) data (for multiple pairs)
Does not run advise_buy or advise_sell!
Used by optimize operations only, not during dry / live runs.
Using .copy() to get a fresh copy of the dataframe for every strategy run.
Also copy on output to avoid PerformanceWarnings pandas 1.3.0 started to show.
Has positive effects on memory usage for whatever reason - also when
using only one strategy.
"""
return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair})
return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}).copy()
for pair, pair_data in data.items()}
def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
@@ -712,6 +840,12 @@ class IStrategy(ABC, HyperStrategyMixin):
:return: a Dataframe with all mandatory indicators for the strategies
"""
logger.debug(f"Populating indicators for pair {metadata.get('pair')}.")
# call populate_indicators_Nm() which were tagged with @informative decorator.
for inf_data, populate_fn in self._ft_informative:
dataframe = _create_and_merge_informative_pair(
self, dataframe, metadata, inf_data, populate_fn)
if self._populate_fun_len == 2:
warnings.warn("deprecated - check out the Sample strategy to see "
"the current function headers!", DeprecationWarning)
@@ -724,7 +858,8 @@ class IStrategy(ABC, HyperStrategyMixin):
Based on TA indicators, populates the buy signal for the given dataframe
This method should not be overridden.
:param dataframe: DataFrame
:param pair: Additional information, like the currently traded pair
:param metadata: Additional information dictionary, with details like the
currently traded pair
:return: DataFrame with buy column
"""
logger.debug(f"Populating buy signals for pair {metadata.get('pair')}.")
@@ -741,7 +876,8 @@ class IStrategy(ABC, HyperStrategyMixin):
Based on TA indicators, populates the sell signal for the given dataframe
This method should not be overridden.
:param dataframe: DataFrame
:param pair: Additional information, like the currently traded pair
:param metadata: Additional information dictionary, with details like the
currently traded pair
:return: DataFrame with sell column
"""
logger.debug(f"Populating sell signals for pair {metadata.get('pair')}.")

47
freqtrade/strategy/strategy_helper.py
View File

@@ -4,7 +4,9 @@ from freqtrade.exchange import timeframe_to_minutes
def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame:
timeframe: str, timeframe_inf: str, ffill: bool = True,
append_timeframe: bool = True,
date_column: str = 'date') -> pd.DataFrame:
"""
Correctly merge informative samples to the original dataframe, avoiding lookahead bias.
@@ -24,6 +26,8 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
:param timeframe: Timeframe of the original pair sample.
:param timeframe_inf: Timeframe of the informative pair sample.
:param ffill: Forwardfill missing values - optional but usually required
:param append_timeframe: Rename columns by appending timeframe.
:param date_column: A custom date column name.
:return: Merged dataframe
:raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe
"""
@@ -32,25 +36,29 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame,
minutes = timeframe_to_minutes(timeframe)
if minutes == minutes_inf:
# No need to forwardshift if the timeframes are identical
informative['date_merge'] = informative["date"]
informative['date_merge'] = informative[date_column]
elif minutes < minutes_inf:
# Subtract "small" timeframe so merging is not delayed by 1 small candle
# Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073
informative['date_merge'] = (
informative["date"] + pd.to_timedelta(minutes_inf, 'm') - pd.to_timedelta(minutes, 'm')
)
informative[date_column] + pd.to_timedelta(minutes_inf, 'm') -
pd.to_timedelta(minutes, 'm')
)
else:
raise ValueError("Tried to merge a faster timeframe to a slower timeframe."
"This would create new rows, and can throw off your regular indicators.")
# Rename columns to be unique
informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
date_merge = 'date_merge'
if append_timeframe:
date_merge = f'date_merge_{timeframe_inf}'
informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns]
# Combine the 2 dataframes
# all indicators on the informative sample MUST be calculated before this point
dataframe = pd.merge(dataframe, informative, left_on='date',
right_on=f'date_merge_{timeframe_inf}', how='left')
dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1)
right_on=date_merge, how='left')
dataframe = dataframe.drop(date_merge, axis=1)
if ffill:
dataframe = dataframe.ffill()
@@ -83,3 +91,28 @@ def stoploss_from_open(open_relative_stop: float, current_profit: float) -> floa
# negative stoploss values indicate the requested stop price is higher than the current price
return max(stoploss, 0.0)
def stoploss_from_absolute(stop_rate: float, current_rate: float) -> float:
"""
Given current price and desired stop price, return a stop loss value that is relative to current
price.
The requested stop can be positive for a stop above the open price, or negative for
a stop below the open price. The return value is always >= 0.
Returns 0 if the resulting stop price would be above the current price.
:param stop_rate: Stop loss price.
:param current_rate: Current asset price.
:return: Positive stop loss value relative to current price
"""
# formula is undefined for current_rate 0, return maximum value
if current_rate == 0:
return 1
stoploss = 1 - (stop_rate / current_rate)
# negative stoploss values indicate the requested stop price is higher than the current price
return max(stoploss, 0.0)

19
freqtrade/templates/base_config.json.j2
View File

@@ -1,3 +1,10 @@
{%set volume_pairlist = '{
"method": "VolumePairList",
"number_assets": 20,
"sort_key": "quoteVolume",
"min_value": 0,
"refresh_period": 1800
}' %}
{
"max_open_trades": {{ max_open_trades }},
"stake_currency": "{{ stake_currency }}",
@@ -15,7 +22,7 @@
"bid_strategy": {
"price_side": "bid",
"ask_last_balance": 0.0,
"use_order_book": false,
"use_order_book": true,
"order_book_top": 1,
"check_depth_of_market": {
"enabled": false,
@@ -24,16 +31,12 @@
},
"ask_strategy": {
"price_side": "ask",
"use_order_book": false,
"order_book_min": 1,
"order_book_max": 1,
"use_sell_signal": true,
"sell_profit_only": false,
"ignore_roi_if_buy_signal": false
"use_order_book": true,
"order_book_top": 1
},
{{ exchange | indent(4) }},
"pairlists": [
{"method": "StaticPairList"}
{{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else volume_pairlist }}
],
"edge": {
"enabled": false,

Some files were not shown because too many files have changed in this diff Show More