adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into v3_fixes

Browse Source
This commit is contained in:
Robert Davey
2022-04-16 14:23:13 +01:00
committed by GitHub
parent 162e94455b 4019c05fee
commit afd3a32883
284 changed files with 44668 additions and 8606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
freqtrade/__init__.py
View File

@@ -1,27 +1,14 @@
""" Freqtrade bot """
__version__ = 'develop'
if __version__ == 'develop':
if 'dev' in __version__:
try:
import subprocess
__version__ = 'develop-' + subprocess.check_output(
__version__ = __version__ + '-' + subprocess.check_output(
['git', 'log', '--format="%h"', '-n 1'],
stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
# from datetime import datetime
# last_release = subprocess.check_output(
# ['git', 'tag']
# ).decode('utf-8').split()[-1].split(".")
# # Releases are in the format "2020.1" - we increment the latest version for dev.
# prefix = f"{last_release[0]}.{int(last_release[1]) + 1}"
# dev_version = int(datetime.now().timestamp() // 1000)
# __version__ = f"{prefix}.dev{dev_version}"
# subprocess.check_output(
# ['git', 'log', '--format="%h"', '-n 1'],
# stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
except Exception: # pragma: no cover
# git not available, ignore
try:

2
freqtrade/__main__.py
View File

@@ -3,7 +3,7 @@
__main__.py for Freqtrade
To launch Freqtrade as a module
> python -m freqtrade (with Python >= 3.7)
> python -m freqtrade (with Python >= 3.8)
"""
from freqtrade import main

7
freqtrade/commands/__init__.py
View File

@@ -8,15 +8,16 @@ Note: Be careful with file-scoped imports in these subfiles.
"""
from freqtrade.commands.arguments import Arguments
from freqtrade.commands.build_config_commands import start_new_config
from freqtrade.commands.data_commands import (start_convert_data, start_download_data,
start_list_data)
from freqtrade.commands.data_commands import (start_convert_data, start_convert_trades,
start_download_data, start_list_data)
from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui,
start_new_strategy)
from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show
from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets,
start_list_strategies, start_list_timeframes,
start_show_trades)
from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt
from freqtrade.commands.optimize_commands import (start_backtesting, start_backtesting_show,
start_edge, start_hyperopt)
from freqtrade.commands.pairlist_commands import start_test_pairlist
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
from freqtrade.commands.trade_commands import start_trading

72
freqtrade/commands/arguments.py
View File

@@ -23,7 +23,8 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
"enable_protections", "dry_run_wallet", "timeframe_detail",
"strategy_list", "export", "exportfilename"]
"strategy_list", "export", "exportfilename",
"backtest_breakdown", "backtest_cache"]
ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"position_stacking", "use_max_market_positions",
@@ -31,7 +32,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"epochs", "spaces", "print_all",
"print_colorized", "print_json", "hyperopt_jobs",
"hyperopt_random_state", "hyperopt_min_trades",
"hyperopt_loss", "disableparamexport"]
"hyperopt_loss", "disableparamexport",
"hyperopt_ignore_missing_space"]
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
@@ -39,15 +41,18 @@ ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized"]
ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"]
ARGS_BACKTEST_SHOW = ["exportfilename", "backtest_show_pair_list"]
ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"]
ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column",
"print_csv", "base_currencies", "quote_currencies", "list_pairs_all"]
"print_csv", "base_currencies", "quote_currencies", "list_pairs_all",
"trading_mode"]
ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column",
"list_pairs_print_json"]
"list_pairs_print_json", "exchange"]
ARGS_CREATE_USERDIR = ["user_data_dir", "reset"]
@@ -56,22 +61,26 @@ ARGS_BUILD_CONFIG = ["config"]
ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"]
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"]
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "exchange", "trading_mode",
"candle_types"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
"download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
"dataformat_trades"]
ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive",
"timerange", "download_trades", "exchange", "timeframes",
"erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
"db_url", "trade_source", "export", "exportfilename",
"timerange", "timeframe", "no_trades"]
ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
"trade_source", "timeframe", "plot_auto_open"]
"trade_source", "timeframe", "plot_auto_open", ]
ARGS_INSTALL_UI = ["erase_ui_only"]
ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version']
ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"]
@@ -86,12 +95,12 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
"disableparamexport"]
"disableparamexport", "backtest_breakdown"]
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies", "list-data",
"hyperopt-list", "hyperopt-show",
"plot-dataframe", "plot-profit", "show-trades"]
"hyperopt-list", "hyperopt-show", "backtest-filter",
"plot-dataframe", "plot-profit", "show-trades", "trades-to-ohlcv"]
NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"]
@@ -169,14 +178,15 @@ class Arguments:
self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot')
self._build_args(optionlist=['version'], parser=self.parser)
from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir,
start_download_data, start_edge, start_hyperopt,
start_hyperopt_list, start_hyperopt_show, start_install_ui,
start_list_data, start_list_exchanges, start_list_markets,
start_list_strategies, start_list_timeframes,
start_new_config, start_new_strategy, start_plot_dataframe,
start_plot_profit, start_show_trades, start_test_pairlist,
start_trading, start_webserver)
from freqtrade.commands import (start_backtesting, start_backtesting_show,
start_convert_data, start_convert_trades,
start_create_userdir, start_download_data, start_edge,
start_hyperopt, start_hyperopt_list, start_hyperopt_show,
start_install_ui, start_list_data, start_list_exchanges,
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config, start_new_strategy,
start_plot_dataframe, start_plot_profit, start_show_trades,
start_test_pairlist, start_trading, start_webserver)
subparsers = self.parser.add_subparsers(dest='command',
# Use custom message when no subhandler is added
@@ -236,6 +246,15 @@ class Arguments:
convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False))
self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd)
# Add trades-to-ohlcv subcommand
convert_trade_data_cmd = subparsers.add_parser(
'trades-to-ohlcv',
help='Convert trade data to OHLCV data.',
parents=[_common_parser],
)
convert_trade_data_cmd.set_defaults(func=start_convert_trades)
self._build_args(optionlist=ARGS_CONVERT_TRADES, parser=convert_trade_data_cmd)
# Add list-data subcommand
list_data_cmd = subparsers.add_parser(
'list-data',
@@ -251,6 +270,15 @@ class Arguments:
backtesting_cmd.set_defaults(func=start_backtesting)
self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd)
# Add backtesting-show subcommand
backtesting_show_cmd = subparsers.add_parser(
'backtesting-show',
help='Show past Backtest results',
parents=[_common_parser],
)
backtesting_show_cmd.set_defaults(func=start_backtesting_show)
self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd)
# Add edge subcommand
edge_cmd = subparsers.add_parser('edge', help='Edge module.',
parents=[_common_parser, _strategy_parser])

46
freqtrade/commands/build_config_commands.py
View File

@@ -76,18 +76,23 @@ def ask_user_config() -> Dict[str, Any]:
{
"type": "text",
"name": "max_open_trades",
"message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):",
"message": "Please insert max_open_trades (Integer or -1 for unlimited open trades):",
"default": "3",
"validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val),
"filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
if val == UNLIMITED_STAKE_AMOUNT
else val
"validate": lambda val: validate_is_int(val)
},
{
"type": "select",
"name": "timeframe_in_config",
"message": "Time",
"choices": ["Have the strategy define timeframe.", "Override in configuration."]
},
{
"type": "text",
"name": "timeframe",
"message": "Please insert your desired timeframe (e.g. 5m):",
"default": "5m",
"when": lambda x: x["timeframe_in_config"] == 'Override in configuration.'
},
{
"type": "text",
@@ -99,18 +104,28 @@ def ask_user_config() -> Dict[str, Any]:
"type": "select",
"name": "exchange_name",
"message": "Select exchange",
"choices": [
"choices": lambda x: [
"binance",
"binanceus",
"bittrex",
"kraken",
"ftx",
"kucoin",
"gateio",
Separator(),
"huobi",
"kraken",
"kucoin",
"okx",
Separator("------------------"),
"other",
],
},
{
"type": "confirm",
"name": "trading_mode",
"message": "Do you want to trade Perpetual Swaps (perpetual futures)?",
"default": False,
"filter": lambda val: 'futures' if val else 'spot',
"when": lambda x: x["exchange_name"] in ['binance', 'gateio', 'okx'],
},
{
"type": "autocomplete",
"name": "exchange_name",
@@ -134,7 +149,7 @@ def ask_user_config() -> Dict[str, Any]:
"type": "password",
"name": "exchange_key_password",
"message": "Insert Exchange API Key password",
"when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin'
"when": lambda x: not x['dry_run'] and x['exchange_name'] in ('kucoin', 'okx')
},
{
"type": "confirm",
@@ -163,7 +178,8 @@ def ask_user_config() -> Dict[str, Any]:
{
"type": "text",
"name": "api_server_listen_addr",
"message": "Insert Api server Listen Address (best left untouched default!)",
"message": ("Insert Api server Listen Address (0.0.0.0 for docker, "
"otherwise best left untouched)"),
"default": "127.0.0.1",
"when": lambda x: x['api_server']
},
@@ -186,7 +202,13 @@ def ask_user_config() -> Dict[str, Any]:
if not answers:
# Interrupted questionary sessions return an empty dict.
raise OperationalException("User interrupted interactive questions.")
# Ensure default is set for non-futures exchanges
answers['trading_mode'] = answers.get('trading_mode', "spot")
answers['margin_mode'] = (
'isolated'
if answers.get('trading_mode') == 'futures'
else ''
)
# Force JWT token to be a random string
answers['api_server_jwt_key'] = secrets.token_hex()

67
freqtrade/commands/cli_options.py
View File

@@ -5,6 +5,7 @@ from argparse import SUPPRESS, ArgumentTypeError
from freqtrade import __version__, constants
from freqtrade.constants import HYPEROPT_LOSS_BUILTIN
from freqtrade.enums import CandleType
def check_int_positive(value: str) -> int:
@@ -117,7 +118,7 @@ AVAILABLE_CLI_OPTIONS = {
),
# Optimize common
"timeframe": Arg(
'-i', '--timeframe', '--ticker-interval',
'-i', '--timeframe',
help='Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).',
),
"timerange": Arg(
@@ -152,6 +153,12 @@ AVAILABLE_CLI_OPTIONS = {
action='store_false',
default=True,
),
"backtest_show_pair_list": Arg(
'--show-pair-list',
help='Show backtesting pairlist sorted by profit.',
action='store_true',
default=False,
),
"enable_protections": Arg(
'--enable-protections', '--enableprotections',
help='Enable protections for backtesting.'
@@ -163,7 +170,7 @@ AVAILABLE_CLI_OPTIONS = {
"strategy_list": Arg(
'--strategy-list',
help='Provide a space-separated list of strategies to backtest. '
'Please note that ticker-interval needs to be set either in config '
'Please note that timeframe needs to be set either in config '
'or via command line. When using this together with `--export trades`, '
'the strategy-name is injected into the filename '
'(so `backtest-data.json` becomes `backtest-data-SampleStrategy.json`',
@@ -173,14 +180,14 @@ AVAILABLE_CLI_OPTIONS = {
'--export',
help='Export backtest results (default: trades).',
choices=constants.EXPORT_OPTIONS,
),
"exportfilename": Arg(
'--export-filename',
help='Save backtest results to the file with this filename. '
'Requires `--export` to be set as well. '
'Example: `--export-filename=user_data/backtest_results/backtest_today.json`',
metavar='PATH',
"--export-filename",
"--backtest-filename",
help="Use this filename for backtest results."
"Requires `--export` to be set as well. "
"Example: `--export-filename=user_data/backtest_results/backtest_today.json`",
metavar="PATH",
),
"disableparamexport": Arg(
'--disable-param-export',
@@ -193,6 +200,18 @@ AVAILABLE_CLI_OPTIONS = {
type=float,
metavar='FLOAT',
),
"backtest_breakdown": Arg(
'--breakdown',
help='Show backtesting breakdown per [day, week, month].',
nargs='+',
choices=constants.BACKTEST_BREAKDOWNS
),
"backtest_cache": Arg(
'--cache',
help='Load a cached backtest result no older than specified age (default: %(default)s).',
default=constants.BACKTEST_CACHE_DEFAULT,
choices=constants.BACKTEST_CACHE_AGE,
),
# Edge
"stoploss_range": Arg(
'--stoplosses',
@@ -337,6 +356,17 @@ AVAILABLE_CLI_OPTIONS = {
nargs='+',
metavar='BASE_CURRENCY',
),
"trading_mode": Arg(
'--trading-mode',
help='Select Trading mode',
choices=constants.TRADING_MODES,
),
"candle_types": Arg(
'--candle-types',
help='Select candle type to use',
choices=[c.value for c in CandleType],
nargs='+',
),
# Script options
"pairs": Arg(
'-p', '--pairs',
@@ -355,6 +385,11 @@ AVAILABLE_CLI_OPTIONS = {
type=check_int_positive,
metavar='INT',
),
"include_inactive": Arg(
'--include-inactive-pairs',
help='Also download data from inactive pairs.',
action='store_true',
),
"new_pairs_days": Arg(
'--new-pairs-days',
help='Download data of new pairs for given number of days. Default: `%(default)s`.',
@@ -381,12 +416,12 @@ AVAILABLE_CLI_OPTIONS = {
),
"dataformat_ohlcv": Arg(
'--data-format-ohlcv',
help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).',
help='Storage format for downloaded candle (OHLCV) data. (default: `json`).',
choices=constants.AVAILABLE_DATAHANDLERS,
),
"dataformat_trades": Arg(
'--data-format-trades',
help='Storage format for downloaded trades data. (default: `%(default)s`).',
help='Storage format for downloaded trades data. (default: `jsongz`).',
choices=constants.AVAILABLE_DATAHANDLERS,
),
"exchange": Arg(
@@ -414,6 +449,12 @@ AVAILABLE_CLI_OPTIONS = {
action='store_true',
default=False,
),
"ui_version": Arg(
'--ui-version',
help=('Specify a specific version of FreqUI to install. '
'Not specifying this installs the latest version.'),
type=str,
),
# Templating options
"template": Arg(
'--template',
@@ -552,4 +593,10 @@ AVAILABLE_CLI_OPTIONS = {
help='Do not print epoch details header.',
action='store_true',
),
"hyperopt_ignore_missing_space": Arg(
"--ignore-missing-spaces", "--ignore-unparameterized-spaces",
help=("Suppress errors for any requested Hyperopt spaces "
"that do not contain any parameters."),
action="store_true",
),
}

81
freqtrade/commands/data_commands.py
View File

@@ -8,9 +8,10 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration
from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
refresh_backtest_trades_data)
from freqtrade.enums import RunMode
from freqtrade.enums import CandleType, RunMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.exchange.exchange import market_is_active
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver
@@ -47,11 +48,13 @@ def start_download_data(args: Dict[str, Any]) -> None:
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
markets = [p for p, m in exchange.markets.items() if market_is_active(m)
or config.get('include_inactive')]
expanded_pairs = expand_pairlist(config['pairs'], markets)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
exchange.validate_pairs(expanded_pairs)
logger.info(f"About to download pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
@@ -61,6 +64,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
try:
if config.get('download_trades'):
if config.get('trading_mode') == 'futures':
raise OperationalException("Trade download not supported for futures.")
pairs_not_available = refresh_backtest_trades_data(
exchange, pairs=expanded_pairs, datadir=config['datadir'],
timerange=timerange, new_pairs_days=config['new_pairs_days'],
@@ -78,7 +83,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange,
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
trading_mode=config.get('trading_mode', 'spot'),
)
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")
@@ -89,15 +96,52 @@ def start_download_data(args: Dict[str, Any]) -> None:
f"on exchange {exchange.name}.")
def start_convert_trades(args: Dict[str, Any]) -> None:
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
timerange = TimeRange()
# Remove stake-currency to skip checks which are not relevant for datadownload
config['stake_currency'] = ''
if 'pairs' not in config:
raise OperationalException(
"Downloading data requires a list of pairs. "
"Please check the documentation on how to configure this.")
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to Convert pairs: {expanded_pairs}, "
f"intervals: {config['timeframes']} to {config['datadir']}")
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
)
def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
"""
Convert data from one format to another
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
if ohlcv:
convert_ohlcv_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'])
candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]
for candle_type in candle_types:
convert_ohlcv_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
erase=args['erase'], candle_type=candle_type)
else:
convert_trades_format(config,
convert_from=args['format_from'], convert_to=args['format_to'],
@@ -116,17 +160,26 @@ def start_list_data(args: Dict[str, Any]) -> None:
from freqtrade.data.history.idatahandler import get_datahandler
dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
paircombs = dhc.ohlcv_get_available_data(config['datadir'])
paircombs = dhc.ohlcv_get_available_data(
config['datadir'],
config.get('trading_mode', TradingMode.SPOT)
)
if args['pairs']:
paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
print(f"Found {len(paircombs)} pair / timeframe combinations.")
groupedpair = defaultdict(list)
for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))):
groupedpair[pair].append(timeframe)
for pair, timeframe, candle_type in sorted(
paircombs,
key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
):
groupedpair[(pair, candle_type)].append(timeframe)
if groupedpair:
print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()],
headers=("Pair", "Timeframe"),
tablefmt='psql', stralign='right'))
print(tabulate([
(pair, ', '.join(timeframes), candle_type)
for (pair, candle_type), timeframes in groupedpair.items()
],
headers=("Pair", "Timeframe", "Type"),
tablefmt='psql', stralign='right'))

16
freqtrade/commands/deploy_commands.py
View File

@@ -128,7 +128,7 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str):
f.write(version)
def get_ui_download_url() -> Tuple[str, str]:
def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]:
base_url = 'https://api.github.com/repos/freqtrade/frequi/'
# Get base UI Repo path
@@ -136,8 +136,16 @@ def get_ui_download_url() -> Tuple[str, str]:
resp.raise_for_status()
r = resp.json()
latest_version = r[0]['name']
assets = r[0].get('assets', [])
if version:
tmp = [x for x in r if x['name'] == version]
if tmp:
latest_version = tmp[0]['name']
assets = tmp[0].get('assets', [])
else:
raise ValueError("UI-Version not found.")
else:
latest_version = r[0]['name']
assets = r[0].get('assets', [])
dl_url = ''
if assets and len(assets) > 0:
dl_url = assets[0]['browser_download_url']
@@ -156,7 +164,7 @@ def start_install_ui(args: Dict[str, Any]) -> None:
dest_folder = Path(__file__).parents[1] / 'rpc/api_server/ui/installed/'
# First make sure the assets are removed.
dl_url, latest_version = get_ui_download_url()
dl_url, latest_version = get_ui_download_url(args.get('ui_version'))
curr_version = read_ui_version(dest_folder)
if curr_version == latest_version and not args.get('erase_ui_only'):

2
freqtrade/commands/hyperopt_commands.py
View File

@@ -96,7 +96,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
if 'strategy_name' in metrics:
strategy_name = metrics['strategy_name']
show_backtest_result(strategy_name, metrics,
metrics['stake_currency'])
metrics['stake_currency'], config.get('backtest_breakdown', []))
HyperoptTools.try_export_params(config, strategy_name, val)

22
freqtrade/commands/list_commands.py
View File

@@ -131,7 +131,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
try:
pairs = exchange.get_markets(base_currencies=base_currencies,
quote_currencies=quote_currencies,
pairs_only=pairs_only,
tradable_only=pairs_only,
active_only=active_only)
# Sort the pairs/markets by symbol
pairs = dict(sorted(pairs.items()))
@@ -151,15 +151,19 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
if quote_currencies else ""))
headers = ["Id", "Symbol", "Base", "Quote", "Active",
*(['Is pair'] if not pairs_only else [])]
"Spot", "Margin", "Future", "Leverage"]
tabular_data = []
for _, v in pairs.items():
tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'],
'Base': v['base'], 'Quote': v['quote'],
'Active': market_is_active(v),
**({'Is pair': exchange.market_is_tradable(v)}
if not pairs_only else {})})
tabular_data = [{
'Id': v['id'],
'Symbol': v['symbol'],
'Base': v['base'],
'Quote': v['quote'],
'Active': market_is_active(v),
'Spot': 'Spot' if exchange.market_is_spot(v) else '',
'Margin': 'Margin' if exchange.market_is_margin(v) else '',
'Future': 'Future' if exchange.market_is_future(v) else '',
'Leverage': exchange.get_max_leverage(v['symbol'], 20)
} for _, v in pairs.items()]
if (args.get('print_one_column', False) or
args.get('list_pairs_print_json', False) or

28
freqtrade/commands/optimize_commands.py
View File

@@ -25,12 +25,16 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[
RunMode.HYPEROPT: 'hyperoptimization',
}
if method in no_unlimited_runmodes.keys():
wallet_size = config['dry_run_wallet'] * config['tradable_balance_ratio']
# tradable_balance_ratio
if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT
and config['stake_amount'] > config['dry_run_wallet']):
wallet = round_coin_value(config['dry_run_wallet'], config['stake_currency'])
and config['stake_amount'] > wallet_size):
wallet = round_coin_value(wallet_size, config['stake_currency'])
stake = round_coin_value(config['stake_amount'], config['stake_currency'])
raise OperationalException(f"Starting balance ({wallet}) "
f"is smaller than stake_amount {stake}.")
raise OperationalException(
f"Starting balance ({wallet}) is smaller than stake_amount {stake}. "
f"Wallet is calculated as `dry_run_wallet * tradable_balance_ratio`."
)
return config
@@ -54,6 +58,22 @@ def start_backtesting(args: Dict[str, Any]) -> None:
backtesting.start()
def start_backtesting_show(args: Dict[str, Any]) -> None:
"""
Show previous backtest result
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
from freqtrade.data.btanalysis import load_backtest_stats
from freqtrade.optimize.optimize_reports import show_backtest_results, show_sorted_pairlist
results = load_backtest_stats(config['exportfilename'])
show_backtest_results(config, results)
show_sorted_pairlist(config, results)
def start_hyperopt(args: Dict[str, Any]) -> None:
"""
Start hyperopt script

4
freqtrade/configuration/PeriodicCache.py
View File

@@ -1,6 +1,6 @@
from datetime import datetime, timezone
from cachetools.ttl import TTLCache
from cachetools import TTLCache
class PeriodicCache(TTLCache):
@@ -16,4 +16,4 @@ class PeriodicCache(TTLCache):
return ts - offset
# Init with smlight offset
super().__init__(maxsize=maxsize, ttl=ttl-1e-5, timer=local_timer, getsizeof=getsizeof)
super().__init__(maxsize=maxsize, ttl=ttl - 1e-5, timer=local_timer, getsizeof=getsizeof)

148
freqtrade/configuration/config_validation.py
View File

@@ -6,7 +6,8 @@ from jsonschema import Draft4Validator, validators
from jsonschema.exceptions import ValidationError, best_match
from freqtrade import constants
from freqtrade.enums import RunMode
from freqtrade.configuration.deprecated_settings import process_deprecated_setting
from freqtrade.enums import RunMode, TradingMode
from freqtrade.exceptions import OperationalException
@@ -80,6 +81,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None:
_validate_protections(conf)
_validate_unlimited_amount(conf)
_validate_ask_orderbook(conf)
validate_migrated_strategy_settings(conf)
# validate configuration before returning
logger.info('Validating configuration ...')
@@ -92,8 +94,8 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None:
:raise: OperationalException if config validation failed
"""
if (not conf.get('edge', {}).get('enabled')
and conf.get('max_open_trades') == float('inf')
and conf.get('stake_amount') == constants.UNLIMITED_STAKE_AMOUNT):
and conf.get('max_open_trades') == float('inf')
and conf.get('stake_amount') == constants.UNLIMITED_STAKE_AMOUNT):
raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.")
@@ -101,13 +103,15 @@ def _validate_price_config(conf: Dict[str, Any]) -> None:
"""
When using market orders, price sides must be using the "other" side of the price
"""
if (conf.get('order_types', {}).get('buy') == 'market'
and conf.get('bid_strategy', {}).get('price_side') != 'ask'):
raise OperationalException('Market buy orders require bid_strategy.price_side = "ask".')
# TODO: The below could be an enforced setting when using market orders
if (conf.get('order_types', {}).get('entry') == 'market'
and conf.get('entry_pricing', {}).get('price_side') not in ('ask', 'other')):
raise OperationalException(
'Market entry orders require entry_pricing.price_side = "other".')
if (conf.get('order_types', {}).get('sell') == 'market'
and conf.get('ask_strategy', {}).get('price_side') != 'bid'):
raise OperationalException('Market sell orders require ask_strategy.price_side = "bid".')
if (conf.get('order_types', {}).get('exit') == 'market'
and conf.get('exit_pricing', {}).get('price_side') not in ('bid', 'other')):
raise OperationalException('Market exit orders require exit_pricing.price_side = "other".')
def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None:
@@ -150,9 +154,9 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
if not conf.get('edge', {}).get('enabled'):
return
if not conf.get('use_sell_signal', True):
if not conf.get('use_exit_signal', True):
raise OperationalException(
"Edge requires `use_sell_signal` to be True, otherwise no sells will happen."
"Edge requires `use_exit_signal` to be True, otherwise no sells will happen."
)
@@ -190,13 +194,13 @@ def _validate_protections(conf: Dict[str, Any]) -> None:
def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
ask_strategy = conf.get('ask_strategy', {})
ask_strategy = conf.get('exit_pricing', {})
ob_min = ask_strategy.get('order_book_min')
ob_max = ask_strategy.get('order_book_max')
if ob_min is not None and ob_max is not None and ask_strategy.get('use_order_book'):
if ob_min != ob_max:
raise OperationalException(
"Using order_book_max != order_book_min in ask_strategy is no longer supported."
"Using order_book_max != order_book_min in exit_pricing is no longer supported."
"Please pick one value and use `order_book_top` in the future."
)
else:
@@ -205,5 +209,121 @@ def _validate_ask_orderbook(conf: Dict[str, Any]) -> None:
logger.warning(
"DEPRECATED: "
"Please use `order_book_top` instead of `order_book_min` and `order_book_max` "
"for your `ask_strategy` configuration."
"for your `exit_pricing` configuration."
)
def validate_migrated_strategy_settings(conf: Dict[str, Any]) -> None:
_validate_time_in_force(conf)
_validate_order_types(conf)
_validate_unfilledtimeout(conf)
_validate_pricing_rules(conf)
_strategy_settings(conf)
def _validate_time_in_force(conf: Dict[str, Any]) -> None:
time_in_force = conf.get('order_time_in_force', {})
if 'buy' in time_in_force or 'sell' in time_in_force:
if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
raise OperationalException(
"Please migrate your time_in_force settings to use 'entry' and 'exit'.")
else:
logger.warning(
"DEPRECATED: Using 'buy' and 'sell' for time_in_force is deprecated."
"Please migrate your time_in_force settings to use 'entry' and 'exit'."
)
process_deprecated_setting(
conf, 'order_time_in_force', 'buy', 'order_time_in_force', 'entry')
process_deprecated_setting(
conf, 'order_time_in_force', 'sell', 'order_time_in_force', 'exit')
def _validate_order_types(conf: Dict[str, Any]) -> None:
order_types = conf.get('order_types', {})
old_order_types = ['buy', 'sell', 'emergencysell', 'forcebuy',
'forcesell', 'emergencyexit', 'forceexit', 'forceentry']
if any(x in order_types for x in old_order_types):
if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
raise OperationalException(
"Please migrate your order_types settings to use the new wording.")
else:
logger.warning(
"DEPRECATED: Using 'buy' and 'sell' for order_types is deprecated."
"Please migrate your order_types settings to use 'entry' and 'exit' wording."
)
for o, n in [
('buy', 'entry'),
('sell', 'exit'),
('emergencysell', 'emergency_exit'),
('forcesell', 'force_exit'),
('forcebuy', 'force_entry'),
('emergencyexit', 'emergency_exit'),
('forceexit', 'force_exit'),
('forceentry', 'force_entry'),
]:
process_deprecated_setting(conf, 'order_types', o, 'order_types', n)
def _validate_unfilledtimeout(conf: Dict[str, Any]) -> None:
unfilledtimeout = conf.get('unfilledtimeout', {})
if any(x in unfilledtimeout for x in ['buy', 'sell']):
if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
raise OperationalException(
"Please migrate your unfilledtimeout settings to use the new wording.")
else:
logger.warning(
"DEPRECATED: Using 'buy' and 'sell' for unfilledtimeout is deprecated."
"Please migrate your unfilledtimeout settings to use 'entry' and 'exit' wording."
)
for o, n in [
('buy', 'entry'),
('sell', 'exit'),
]:
process_deprecated_setting(conf, 'unfilledtimeout', o, 'unfilledtimeout', n)
def _validate_pricing_rules(conf: Dict[str, Any]) -> None:
if conf.get('ask_strategy') or conf.get('bid_strategy'):
if conf.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
raise OperationalException(
"Please migrate your pricing settings to use the new wording.")
else:
logger.warning(
"DEPRECATED: Using 'ask_strategy' and 'bid_strategy' is deprecated."
"Please migrate your settings to use 'entry_pricing' and 'exit_pricing'."
)
conf['entry_pricing'] = {}
for obj in list(conf.get('bid_strategy', {}).keys()):
if obj == 'ask_last_balance':
process_deprecated_setting(conf, 'bid_strategy', obj,
'entry_pricing', 'price_last_balance')
else:
process_deprecated_setting(conf, 'bid_strategy', obj, 'entry_pricing', obj)
del conf['bid_strategy']
conf['exit_pricing'] = {}
for obj in list(conf.get('ask_strategy', {}).keys()):
if obj == 'bid_last_balance':
process_deprecated_setting(conf, 'ask_strategy', obj,
'exit_pricing', 'price_last_balance')
else:
process_deprecated_setting(conf, 'ask_strategy', obj, 'exit_pricing', obj)
del conf['ask_strategy']
def _strategy_settings(conf: Dict[str, Any]) -> None:
process_deprecated_setting(conf, None, 'use_sell_signal', None, 'use_exit_signal')
process_deprecated_setting(conf, None, 'sell_profit_only', None, 'exit_profit_only')
process_deprecated_setting(conf, None, 'sell_profit_offset', None, 'exit_profit_offset')
process_deprecated_setting(conf, None, 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_entry_signal')

78
freqtrade/configuration/configuration.py
View File

@@ -12,8 +12,8 @@ from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.environment_vars import enironment_vars_to_dict
from freqtrade.configuration.load_config import load_config_file, load_file
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
from freqtrade.configuration.load_config import load_file, load_from_files
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, CandleType, RunMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging
@@ -55,47 +55,28 @@ class Configuration:
:param files: List of file paths
:return: configuration dictionary
"""
# Keep this method as staticmethod, so it can be used from interactive environments
c = Configuration({'config': files}, RunMode.OTHER)
return c.get_config()
def load_from_files(self, files: List[str]) -> Dict[str, Any]:
# Keep this method as staticmethod, so it can be used from interactive environments
config: Dict[str, Any] = {}
if not files:
return deepcopy(constants.MINIMAL_CONFIG)
# We expect here a list of config filenames
for path in files:
logger.info(f'Using config: {path} ...')
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:
config['internals'] = {}
if 'ask_strategy' not in config:
config['ask_strategy'] = {}
if 'pairlists' not in config:
config['pairlists'] = []
return config
def load_config(self) -> Dict[str, Any]:
"""
Extract information for sys.argv and load the bot configuration
:return: Configuration dictionary
"""
# Load all configs
config: Dict[str, Any] = self.load_from_files(self.args.get("config", []))
config: Dict[str, Any] = load_from_files(self.args.get("config", []))
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
# Normalize config
if 'internals' not in config:
config['internals'] = {}
if 'pairlists' not in config:
config['pairlists'] = []
# Keep a copy of the original configuration file
config['original_config'] = deepcopy(config)
@@ -166,8 +147,8 @@ class Configuration:
config.update({'db_url': self.args['db_url']})
logger.info('Parameter --db-url detected ...')
if config.get('forcebuy_enable', False):
logger.warning('`forcebuy` RPC message enabled.')
if config.get('force_entry_enable', False):
logger.warning('`force_entry_enable` RPC message enabled.')
# Support for sd_notify
if 'sd_notify' in self.args and self.args['sd_notify']:
@@ -245,6 +226,10 @@ class Configuration:
self._args_to_config(config, argname='timeframe_detail',
logstring='Parameter --timeframe-detail detected, '
'using {} for intra-candle backtesting ...')
self._args_to_config(config, argname='backtest_show_pair_list',
logstring='Parameter --show-pair-list detected.')
self._args_to_config(config, argname='stake_amount',
logstring='Parameter --stake-amount detected, '
'overriding stake_amount to: {} ...')
@@ -269,8 +254,15 @@ class Configuration:
self._args_to_config(config, argname='export',
logstring='Parameter --export detected: {} ...')
self._args_to_config(config, argname='backtest_breakdown',
logstring='Parameter --breakdown detected ...')
self._args_to_config(config, argname='backtest_cache',
logstring='Parameter --cache={} detected ...')
self._args_to_config(config, argname='disableparamexport',
logstring='Parameter --disableparamexport detected: {} ...')
# Edge section:
if 'stoploss_range' in self.args and self.args["stoploss_range"]:
txt_range = eval(self.args["stoploss_range"])
@@ -369,6 +361,9 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_show_no_header',
logstring='Parameter --no-header detected: {}')
self._args_to_config(config, argname="hyperopt_ignore_missing_space",
logstring="Paramter --ignore-missing-space detected: {}")
def _process_plot_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='pairs',
@@ -404,6 +399,9 @@ class Configuration:
self._args_to_config(config, argname='days',
logstring='Detected --days: {}')
self._args_to_config(config, argname='include_inactive',
logstring='Detected --include-inactive-pairs: {}')
self._args_to_config(config, argname='download_trades',
logstring='Detected --dl-trades: {}')
@@ -414,9 +412,15 @@ class Configuration:
logstring='Using "{}" to store trades data.')
def _process_data_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='new_pairs_days',
logstring='Detected --new-pairs-days: {}')
self._args_to_config(config, argname='trading_mode',
logstring='Detected --trading-mode: {}')
config['candle_type_def'] = CandleType.get_default(
config.get('trading_mode', 'spot') or 'spot')
config['trading_mode'] = TradingMode(config.get('trading_mode', 'spot') or 'spot')
self._args_to_config(config, argname='candle_types',
logstring='Detected --candle-types: {}')
def _process_runmode(self, config: Dict[str, Any]) -> None:

82
freqtrade/configuration/deprecated_settings.py
View File

@@ -12,14 +12,15 @@ logger = logging.getLogger(__name__)
def check_conflicting_settings(config: Dict[str, Any],
section_old: str, name_old: str,
section_old: Optional[str], name_old: str,
section_new: Optional[str], name_new: str) -> None:
section_new_config = config.get(section_new, {}) if section_new else config
section_old_config = config.get(section_old, {})
section_old_config = config.get(section_old, {}) if section_old else config
if name_new in section_new_config and name_old in section_old_config:
new_name = f"{section_new}.{name_new}" if section_new else f"{name_new}"
old_name = f"{section_old}.{name_old}" if section_old else f"{name_old}"
raise OperationalException(
f"Conflicting settings `{new_name}` and `{section_old}.{name_old}` "
f"Conflicting settings `{new_name}` and `{old_name}` "
"(DEPRECATED) detected in the configuration file. "
"This deprecated setting will be removed in the next versions of Freqtrade. "
f"Please delete it from your configuration and use the `{new_name}` "
@@ -47,23 +48,25 @@ def process_removed_setting(config: Dict[str, Any],
def process_deprecated_setting(config: Dict[str, Any],
section_old: str, name_old: str,
section_old: Optional[str], name_old: str,
section_new: Optional[str], name_new: str
) -> None:
check_conflicting_settings(config, section_old, name_old, section_new, name_new)
section_old_config = config.get(section_old, {})
section_old_config = config.get(section_old, {}) if section_old else config
if name_old in section_old_config:
section_1 = f"{section_old}.{name_old}" if section_old else f"{name_old}"
section_2 = f"{section_new}.{name_new}" if section_new else f"{name_new}"
logger.warning(
"DEPRECATED: "
f"The `{section_old}.{name_old}` setting is deprecated and "
f"The `{section_1}` setting is deprecated and "
"will be removed in the next versions of Freqtrade. "
f"Please use the `{section_2}` setting in your configuration instead."
)
section_new_config = config.get(section_new, {}) if section_new else config
section_new_config[name_new] = section_old_config[name_old]
del section_old_config[name_old]
def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
@@ -71,25 +74,51 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
# Kept for future deprecated / moved settings
# check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal',
# 'experimental', 'use_sell_signal')
process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal',
None, 'use_sell_signal')
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only',
None, 'sell_profit_only')
process_deprecated_setting(config, 'ask_strategy', 'sell_profit_offset',
None, 'sell_profit_offset')
process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_buy_signal')
process_deprecated_setting(config, 'ask_strategy', 'ignore_buying_expired_candle_after',
None, 'ignore_buying_expired_candle_after')
# Legacy way - having them in experimental ...
process_removed_setting(config, 'experimental', 'use_sell_signal',
None, 'use_sell_signal')
process_removed_setting(config, 'experimental', 'sell_profit_only',
None, 'sell_profit_only')
process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_buy_signal')
process_deprecated_setting(config, None, 'forcebuy_enable', None, 'force_entry_enable')
# New settings
if config.get('telegram'):
process_deprecated_setting(config['telegram'], 'notification_settings', 'sell',
'notification_settings', 'exit')
process_deprecated_setting(config['telegram'], 'notification_settings', 'sell_fill',
'notification_settings', 'exit_fill')
process_deprecated_setting(config['telegram'], 'notification_settings', 'sell_cancel',
'notification_settings', 'exit_cancel')
process_deprecated_setting(config['telegram'], 'notification_settings', 'buy',
'notification_settings', 'entry')
process_deprecated_setting(config['telegram'], 'notification_settings', 'buy_fill',
'notification_settings', 'entry_fill')
process_deprecated_setting(config['telegram'], 'notification_settings', 'buy_cancel',
'notification_settings', 'entry_cancel')
if config.get('webhook'):
process_deprecated_setting(config, 'webhook', 'webhookbuy', 'webhook', 'webhookentry')
process_deprecated_setting(config, 'webhook', 'webhookbuycancel',
'webhook', 'webhookentrycancel')
process_deprecated_setting(config, 'webhook', 'webhookbuyfill',
'webhook', 'webhookentryfill')
process_deprecated_setting(config, 'webhook', 'webhooksell', 'webhook', 'webhookexit')
process_deprecated_setting(config, 'webhook', 'webhooksellcancel',
'webhook', 'webhookexitcancel')
process_deprecated_setting(config, 'webhook', 'webhooksellfill',
'webhook', 'webhookexitfill')
# Legacy way - having them in experimental ...
process_removed_setting(config, 'experimental', 'use_sell_signal', None, 'use_exit_signal')
process_removed_setting(config, 'experimental', 'sell_profit_only', None, 'exit_profit_only')
process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_entry_signal')
process_removed_setting(config, 'ask_strategy', 'use_sell_signal', None, 'exit_sell_signal')
process_removed_setting(config, 'ask_strategy', 'sell_profit_only', None, 'exit_profit_only')
process_removed_setting(config, 'ask_strategy', 'sell_profit_offset',
None, 'exit_profit_offset')
process_removed_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal',
None, 'ignore_roi_if_entry_signal')
if (config.get('edge', {}).get('enabled', False)
and 'capital_available_percentage' in config.get('edge', {})):
raise OperationalException(
@@ -100,16 +129,11 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None:
"from the edge configuration."
)
if 'ticker_interval' in config:
logger.warning(
"DEPRECATED: "
raise OperationalException(
"DEPRECATED: 'ticker_interval' detected. "
"Please use 'timeframe' instead of 'ticker_interval."
)
if 'timeframe' in config:
raise OperationalException(
"Both 'timeframe' and 'ticker_interval' detected."
"Please remove 'ticker_interval' from your configuration to continue operating."
)
config['timeframe'] = config['ticker_interval']
if 'protections' in config:
logger.warning("DEPRECATED: Setting 'protections' in the configuration is deprecated.")

5
freqtrade/configuration/environment_vars.py
View File

@@ -32,6 +32,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
:param prefix: Prefix to consider (usually FREQTRADE__)
:return: Nested dict based on available and relevant variables.
"""
no_convert = ['CHAT_ID']
relevant_vars: Dict[str, Any] = {}
for env_var, val in sorted(env_dict.items()):
@@ -39,9 +40,9 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
logger.info(f"Loading variable '{env_var}'")
key = env_var.replace(prefix, '')
for k in reversed(key.split('__')):
val = {k.lower(): get_var_typed(val) if type(val) != dict else val}
val = {k.lower(): get_var_typed(val)
if type(val) != dict and k not in no_convert else val}
relevant_vars = deep_merge_dicts(val, relevant_vars)
return relevant_vars

47
freqtrade/configuration/load_config.py
View File

@@ -4,12 +4,15 @@ This module contain functions to load the configuration file
import logging
import re
import sys
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, List
import rapidjson
from freqtrade.constants import MINIMAL_CONFIG
from freqtrade.exceptions import OperationalException
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
@@ -28,7 +31,7 @@ def log_config_error_range(path: str, errmsg: str) -> str:
offset = int(offsetlist[0])
text = Path(path).read_text()
# Fetch an offset of 80 characters around the error line
subtext = text[offset-min(80, offset):offset+80]
subtext = text[offset - min(80, offset):offset + 80]
segments = subtext.split('\n')
if len(segments) > 3:
# Remove first and last lines, to avoid odd truncations
@@ -70,3 +73,43 @@ def load_config_file(path: str) -> Dict[str, Any]:
)
return config
def load_from_files(files: List[str], base_path: Path = None, level: int = 0) -> Dict[str, Any]:
"""
Recursively load configuration files if specified.
Sub-files are assumed to be relative to the initial config.
"""
config: Dict[str, Any] = {}
if level > 5:
raise OperationalException("Config loop detected.")
if not files:
return deepcopy(MINIMAL_CONFIG)
files_loaded = []
# We expect here a list of config filenames
for filename in files:
logger.info(f'Using config: {filename} ...')
if filename == '-':
# Immediately load stdin and return
return load_config_file(filename)
file = Path(filename)
if base_path:
# Prepend basepath to allow for relative assignments
file = base_path / file
config_tmp = load_config_file(str(file))
if 'add_config_files' in config_tmp:
config_sub = load_from_files(
config_tmp['add_config_files'], file.resolve().parent, level + 1)
files_loaded.extend(config_sub.get('config_files', []))
config_tmp = deep_merge_dicts(config_tmp, config_sub)
files_loaded.insert(0, str(file))
# Merge config options, overwriting prior values
config = deep_merge_dicts(config_tmp, config)
config['config_files'] = files_loaded
return config

160
freqtrade/constants.py
View File

@@ -3,7 +3,9 @@
"""
bot constants
"""
from typing import List, Tuple
from typing import List, Literal, Tuple
from freqtrade.enums import CandleType
DEFAULT_CONFIG = 'config.json'
@@ -17,20 +19,25 @@ DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
UNLIMITED_STAKE_AMOUNT = 'unlimited'
DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05
REQUIRED_ORDERTIF = ['buy', 'sell']
REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
ORDERBOOK_SIDES = ['ask', 'bid']
REQUIRED_ORDERTIF = ['entry', 'exit']
REQUIRED_ORDERTYPES = ['entry', 'exit', 'stoploss', 'stoploss_on_exchange']
PRICING_SIDES = ['ask', 'bid', 'same', 'other']
ORDERTYPE_POSSIBILITIES = ['limit', 'market']
ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc']
HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily',
'CalmarHyperOptLoss',
'MaxDrawDownHyperOptLoss', 'ProfitDrawDownHyperOptLoss']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
BACKTEST_BREAKDOWNS = ['day', 'week', 'month']
BACKTEST_CACHE_AGE = ['none', 'day', 'week', 'month']
BACKTEST_CACHE_DEFAULT = 'day'
DRY_RUN_WALLET = 1000
DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S'
MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons
@@ -38,6 +45,8 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
# Don't modify sequence of DEFAULT_TRADES_COLUMNS
# it has wide consequences for stored trades files
DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
TRADING_MODES = ['spot', 'margin', 'futures']
MARGIN_MODES = ['cross', 'isolated', '']
LAST_BT_RESULT_FN = '.last_result.json'
FTHYPT_FILEVERSION = 'fthypt_fileversion'
@@ -47,11 +56,12 @@ USERPATH_STRATEGIES = 'strategies'
USERPATH_NOTEBOOKS = 'notebooks'
TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent']
WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw']
ENV_VAR_PREFIX = 'FREQTRADE__'
NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired')
# Define decimals per coin for outputs
# Only used for outputs.
DECIMAL_PER_COIN_FALLBACK = 3 # Should be low to avoid listing all possible FIAT's
@@ -65,7 +75,6 @@ DUST_PER_COIN = {
'ETH': 0.01
}
# Source files with destination directories within user-directory
USER_DATA_FILES = {
'sample_strategy.py': USERPATH_STRATEGIES,
@@ -77,20 +86,19 @@ SUPPORTED_FIAT = [
"AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK",
"EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY",
"KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN",
"RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD",
"BTC", "ETH", "XRP", "LTC", "BCH"
"RUB", "UAH", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR",
"USD", "BTC", "ETH", "XRP", "LTC", "BCH"
]
MINIMAL_CONFIG = {
'stake_currency': '',
'dry_run': True,
'exchange': {
'name': '',
'key': '',
'secret': '',
'pair_whitelist': [],
'ccxt_async_config': {
'enableRateLimit': True,
"stake_currency": "",
"dry_run": True,
"exchange": {
"name": "",
"key": "",
"secret": "",
"pair_whitelist": [],
"ccxt_async_config": {
}
}
}
@@ -135,35 +143,43 @@ CONF_SCHEMA = {
'minProperties': 1
},
'amount_reserve_percent': {'type': 'number', 'minimum': 0.0, 'maximum': 0.5},
'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True},
'stoploss': {'type': 'number', 'maximum': 0, 'exclusiveMaximum': True, 'minimum': -1},
'trailing_stop': {'type': 'boolean'},
'trailing_stop_positive': {'type': 'number', 'minimum': 0, 'maximum': 1},
'trailing_stop_positive_offset': {'type': 'number', 'minimum': 0, 'maximum': 1},
'trailing_only_offset_is_reached': {'type': 'boolean'},
'use_sell_signal': {'type': 'boolean'},
'sell_profit_only': {'type': 'boolean'},
'sell_profit_offset': {'type': 'number'},
'ignore_roi_if_buy_signal': {'type': 'boolean'},
'use_exit_signal': {'type': 'boolean'},
'exit_profit_only': {'type': 'boolean'},
'exit_profit_offset': {'type': 'number'},
'ignore_roi_if_entry_signal': {'type': 'boolean'},
'ignore_buying_expired_candle_after': {'type': 'number'},
'trading_mode': {'type': 'string', 'enum': TRADING_MODES},
'margin_mode': {'type': 'string', 'enum': MARGIN_MODES},
'liquidation_buffer': {'type': 'number', 'minimum': 0.0, 'maximum': 0.99},
'backtest_breakdown': {
'type': 'array',
'items': {'type': 'string', 'enum': BACKTEST_BREAKDOWNS}
},
'bot_name': {'type': 'string'},
'unfilledtimeout': {
'type': 'object',
'properties': {
'buy': {'type': 'number', 'minimum': 1},
'sell': {'type': 'number', 'minimum': 1},
'entry': {'type': 'number', 'minimum': 1},
'exit': {'type': 'number', 'minimum': 1},
'exit_timeout_count': {'type': 'number', 'minimum': 0, 'default': 0},
'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'}
}
},
'bid_strategy': {
'entry_pricing': {
'type': 'object',
'properties': {
'ask_last_balance': {
'price_last_balance': {
'type': 'number',
'minimum': 0,
'maximum': 1,
'exclusiveMaximum': False,
},
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'},
'price_side': {'type': 'string', 'enum': PRICING_SIDES, 'default': 'same'},
'use_order_book': {'type': 'boolean'},
'order_book_top': {'type': 'integer', 'minimum': 1, 'maximum': 50, },
'check_depth_of_market': {
@@ -176,11 +192,11 @@ CONF_SCHEMA = {
},
'required': ['price_side']
},
'ask_strategy': {
'exit_pricing': {
'type': 'object',
'properties': {
'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'ask'},
'bid_last_balance': {
'price_side': {'type': 'string', 'enum': PRICING_SIDES, 'default': 'same'},
'price_last_balance': {
'type': 'number',
'minimum': 0,
'maximum': 1,
@@ -192,31 +208,34 @@ CONF_SCHEMA = {
'required': ['price_side']
},
'custom_price_max_distance_ratio': {
'type': 'number', 'minimum': 0.0
'type': 'number', 'minimum': 0.0
},
'order_types': {
'type': 'object',
'properties': {
'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'forcesell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'forcebuy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'emergencysell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'entry': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'exit': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'force_exit': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'force_entry': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'emergency_exit': {
'type': 'string',
'enum': ORDERTYPE_POSSIBILITIES,
'default': 'market'},
'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES},
'stoploss_on_exchange': {'type': 'boolean'},
'stoploss_on_exchange_interval': {'type': 'number'},
'stoploss_on_exchange_limit_ratio': {'type': 'number', 'minimum': 0.0,
'maximum': 1.0}
},
'required': ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']
'required': ['entry', 'exit', 'stoploss', 'stoploss_on_exchange']
},
'order_time_in_force': {
'type': 'object',
'properties': {
'buy': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES},
'sell': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES}
'entry': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES},
'exit': {'type': 'string', 'enum': ORDERTIF_POSSIBILITIES}
},
'required': ['buy', 'sell']
'required': REQUIRED_ORDERTIF
},
'exchange': {'$ref': '#/definitions/exchange'},
'edge': {'$ref': '#/definitions/edge'},
@@ -265,21 +284,21 @@ CONF_SCHEMA = {
'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy_fill': {'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
},
'sell': {
'entry': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'entry_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'entry_fill': {'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
},
'exit': {
'type': ['string', 'object'],
'additionalProperties': {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS
}
},
'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell_fill': {
'exit_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'exit_fill': {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
@@ -303,10 +322,16 @@ CONF_SCHEMA = {
'type': 'object',
'properties': {
'enabled': {'type': 'boolean'},
'webhookbuy': {'type': 'object'},
'webhookbuycancel': {'type': 'object'},
'webhooksell': {'type': 'object'},
'webhooksellcancel': {'type': 'object'},
'url': {'type': 'string'},
'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'},
'retries': {'type': 'integer', 'minimum': 0},
'retry_delay': {'type': 'number', 'minimum': 0},
'webhookentry': {'type': 'object'},
'webhookentrycancel': {'type': 'object'},
'webhookentryfill': {'type': 'object'},
'webhookexit': {'type': 'object'},
'webhookexitcancel': {'type': 'object'},
'webhookexitfill': {'type': 'object'},
'webhookstatus': {'type': 'object'},
},
},
@@ -332,7 +357,7 @@ CONF_SCHEMA = {
'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
'disableparamexport': {'type': 'boolean'},
'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
'forcebuy_enable': {'type': 'boolean'},
'force_entry_enable': {'type': 'boolean'},
'disable_dataframe_checks': {'type': 'boolean'},
'internals': {
'type': 'object',
@@ -345,14 +370,16 @@ CONF_SCHEMA = {
},
'dataformat_ohlcv': {
'type': 'string',
'enum': AVAILABLE_DATAHANDLERS,
'default': 'json'
'enum': AVAILABLE_DATAHANDLERS,
'default': 'json'
},
'dataformat_trades': {
'type': 'string',
'enum': AVAILABLE_DATAHANDLERS,
'default': 'jsongz'
}
'enum': AVAILABLE_DATAHANDLERS,
'default': 'jsongz'
},
'position_adjustment_enable': {'type': 'boolean'},
'max_entry_position_adjustment': {'type': ['integer', 'number'], 'minimum': -1},
},
'definitions': {
'exchange': {
@@ -378,6 +405,7 @@ CONF_SCHEMA = {
},
'uniqueItems': True
},
'unknown_fee_rate': {'type': 'number'},
'outdated_offset': {'type': 'integer', 'minimum': 1},
'markets_refresh_interval': {'type': 'integer'},
'ccxt_config': {'type': 'object'},
@@ -416,9 +444,8 @@ SCHEMA_TRADE_REQUIRED = [
'last_stake_amount_min_ratio',
'dry_run',
'dry_run_wallet',
'ask_strategy',
'bid_strategy',
'unfilledtimeout',
'exit_pricing',
'entry_pricing',
'stoploss',
'minimal_roi',
'internals',
@@ -450,12 +477,15 @@ CANCEL_REASON = {
"FULLY_CANCELLED": "fully cancelled",
"ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)",
"CANCELLED_ON_EXCHANGE": "cancelled on exchange",
"FORCE_SELL": "forcesold",
"FORCE_EXIT": "forcesold",
}
# List of pairs with their timeframes
PairWithTimeframe = Tuple[str, str]
PairWithTimeframe = Tuple[str, str, CandleType]
ListPairsWithTimeframes = List[PairWithTimeframe]
# Type for trades list
TradeList = List[List]
LongShort = Literal['long', 'short']
EntryExit = Literal['entry', 'exit']

217
freqtrade/data/btanalysis.py
View File

@@ -2,6 +2,8 @@
Helpers when analyzing backtest data
"""
import logging
from copy import copy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union
@@ -9,28 +11,22 @@ import numpy as np
import pandas as pd
from freqtrade.constants import LAST_BT_RESULT_FN
from freqtrade.misc import json_load
from freqtrade.exceptions import OperationalException
from freqtrade.misc import get_backtest_metadata_filename, json_load
from freqtrade.persistence import LocalTrade, Trade, init_db
logger = logging.getLogger(__name__)
# Old format - maybe remove?
BT_DATA_COLUMNS_OLD = ["pair", "profit_percent", "open_date", "close_date", "index",
"trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"]
# Mid-term format, created by BacktestResult Named Tuple
BT_DATA_COLUMNS_MID = ['pair', 'profit_percent', 'open_date', 'close_date', 'trade_duration',
'open_rate', 'close_rate', 'open_at_end', 'sell_reason', 'fee_open',
'fee_close', 'amount', 'profit_abs', 'profit_ratio']
# Newest format
BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date',
'open_rate', 'close_rate',
'fee_open', 'fee_close', 'trade_duration',
'profit_ratio', 'profit_abs', 'sell_reason',
'profit_ratio', 'profit_abs', 'exit_reason',
'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs',
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'buy_tag']
'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag',
'is_short'
]
def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str:
@@ -106,10 +102,30 @@ def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str =
if isinstance(directory, str):
directory = Path(directory)
if predef_filename:
if Path(predef_filename).is_absolute():
raise OperationalException(
"--hyperopt-filename expects only the filename, not an absolute path.")
return directory / predef_filename
return directory / get_latest_hyperopt_filename(directory)
def load_backtest_metadata(filename: Union[Path, str]) -> Dict[str, Any]:
"""
Read metadata dictionary from backtest results file without reading and deserializing entire
file.
:param filename: path to backtest results file.
:return: metadata dict or None if metadata is not present.
"""
filename = get_backtest_metadata_filename(filename)
try:
with filename.open() as fp:
return json_load(fp)
except FileNotFoundError:
return {}
except Exception as e:
raise OperationalException('Unexpected error while loading backtest metadata.') from e
def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
"""
Load backtest statistics file.
@@ -126,9 +142,104 @@ def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]:
with filename.open() as file:
data = json_load(file)
# Legacy list format does not contain metadata.
if isinstance(data, dict):
data['metadata'] = load_backtest_metadata(filename)
return data
def load_and_merge_backtest_result(strategy_name: str, filename: Path, results: Dict[str, Any]):
"""
Load one strategy from multi-strategy result
and merge it with results
:param strategy_name: Name of the strategy contained in the result
:param filename: Backtest-result-filename to load
:param results: dict to merge the result to.
"""
bt_data = load_backtest_stats(filename)
for k in ('metadata', 'strategy'):
results[k][strategy_name] = bt_data[k][strategy_name]
comparison = bt_data['strategy_comparison']
for i in range(len(comparison)):
if comparison[i]['key'] == strategy_name:
results['strategy_comparison'].append(comparison[i])
break
def _get_backtest_files(dirname: Path) -> List[Path]:
return list(reversed(sorted(dirname.glob('backtest-result-*-[0-9][0-9].json'))))
def get_backtest_resultlist(dirname: Path):
"""
Get list of backtest results read from metadata files
"""
results = []
for filename in _get_backtest_files(dirname):
metadata = load_backtest_metadata(filename)
if not metadata:
continue
for s, v in metadata.items():
results.append({
'filename': filename.name,
'strategy': s,
'run_id': v['run_id'],
'backtest_start_time': v['backtest_start_time'],
})
return results
def find_existing_backtest_stats(dirname: Union[Path, str], run_ids: Dict[str, str],
min_backtest_date: datetime = None) -> Dict[str, Any]:
"""
Find existing backtest stats that match specified run IDs and load them.
:param dirname: pathlib.Path object, or string pointing to the file.
:param run_ids: {strategy_name: id_string} dictionary.
:param min_backtest_date: do not load a backtest older than specified date.
:return: results dict.
"""
# Copy so we can modify this dict without affecting parent scope.
run_ids = copy(run_ids)
dirname = Path(dirname)
results: Dict[str, Any] = {
'metadata': {},
'strategy': {},
'strategy_comparison': [],
}
# Weird glob expression here avoids including .meta.json files.
for filename in _get_backtest_files(dirname):
metadata = load_backtest_metadata(filename)
if not metadata:
# Files are sorted from newest to oldest. When file without metadata is encountered it
# is safe to assume older files will also not have any metadata.
break
for strategy_name, run_id in list(run_ids.items()):
strategy_metadata = metadata.get(strategy_name, None)
if not strategy_metadata:
# This strategy is not present in analyzed backtest.
continue
if min_backtest_date is not None:
backtest_date = strategy_metadata['backtest_start_time']
backtest_date = datetime.fromtimestamp(backtest_date, tz=timezone.utc)
if backtest_date < min_backtest_date:
# Do not use a cached result for this strategy as first result is too old.
del run_ids[strategy_name]
continue
if strategy_metadata['run_id'] == run_id:
del run_ids[strategy_name]
load_and_merge_backtest_result(strategy_name, filename, results)
if len(run_ids) == 0:
break
return results
def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame:
"""
Load backtest data file.
@@ -165,25 +276,18 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
utc=True,
infer_datetime_format=True
)
# Compatibility support for pre short Columns
if 'is_short' not in df.columns:
df['is_short'] = 0
if 'enter_tag' not in df.columns:
df['enter_tag'] = df['buy_tag']
df = df.drop(['buy_tag'], axis=1)
else:
# old format - only with lists.
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
if not df.empty:
df['open_date'] = pd.to_datetime(df['open_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
# Create compatibility with new format
df['profit_abs'] = df['close_rate'] - df['open_rate']
raise OperationalException(
"Backtest-results with only trades data are no longer supported.")
if not df.empty:
if 'profit_ratio' not in df.columns:
df['profit_ratio'] = df['profit_percent']
df = df.sort_values("open_date").reset_index(drop=True)
return df
@@ -325,6 +429,7 @@ def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame],
:param column: Column in the original dataframes to use
:return: DataFrame with the column renamed to the dict key, and a column
named mean, containing the mean of all pairs.
:raise: ValueError if no data is provided.
"""
df_comb = pd.concat([data[pair].set_index('date').rename(
{column: pair}, axis=1)[pair] for pair in data], axis=1)
@@ -360,9 +465,19 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
return df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float]:
def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str
) -> pd.DataFrame:
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
max_drawdown_df['date'] = profit_results.loc[:, date_col]
return max_drawdown_df
def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_ratio'
):
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
@@ -375,10 +490,29 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = pd.DataFrame()
max_drawdown_df['cumulative'] = profit_results[value_col].cumsum()
max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax()
max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value']
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
return max_drawdown_df
def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date',
value_col: str = 'profit_abs', starting_balance: float = 0
) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float, float]:
"""
Calculate max drawdown and the corresponding close dates
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param date_col: Column in DataFrame to use for dates (defaults to 'close_date')
:param value_col: Column in DataFrame to use for values (defaults to 'profit_abs')
:param starting_balance: Portfolio starting balance - properly calculate relative drawdown.
:return: Tuple (float, highdate, lowdate, highvalue, lowvalue, relative_drawdown)
with absolute max drawdown, high and low time and high and low value,
and the relative account drawdown
:raise: ValueError if trade-dataframe was found empty.
"""
if len(trades) == 0:
raise ValueError("Trade dataframe empty.")
profit_results = trades.sort_values(date_col).reset_index(drop=True)
max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col)
idxmin = max_drawdown_df['drawdown'].idxmin()
if idxmin == 0:
@@ -388,7 +522,18 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date'
high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin]
['high_value'].idxmax(), 'cumulative']
low_val = max_drawdown_df.loc[idxmin, 'cumulative']
return abs(min(max_drawdown_df['drawdown'])), high_date, low_date, high_val, low_val
max_drawdown_rel = 0.0
if high_val + starting_balance != 0:
max_drawdown_rel = (high_val - low_val) / (high_val + starting_balance)
return (
abs(min(max_drawdown_df['drawdown'])),
high_date,
low_date,
high_val,
low_val,
max_drawdown_rel
)
def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]:

33
freqtrade/data/converter.py
View File

@@ -11,6 +11,7 @@ import pandas as pd
from pandas import DataFrame, to_datetime
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.enums import CandleType
logger = logging.getLogger(__name__)
@@ -113,7 +114,7 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
if len_before != len_after:
message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
f" - {round(pct_missing * 100, 2)}%")
f" - {pct_missing:.2%}")
if pct_missing > 0.01:
logger.info(message)
else:
@@ -261,13 +262,20 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to:
src.trades_purge(pair=pair)
def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool):
def convert_ohlcv_format(
config: Dict[str, Any],
convert_from: str,
convert_to: str,
erase: bool,
candle_type: CandleType
):
"""
Convert OHLCV from one format to another
:param config: Config dictionary
:param convert_from: Source format
:param convert_to: Target format
:param erase: Erase source data (does not apply if source and target format are identical)
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
from freqtrade.data.history.idatahandler import get_datahandler
src = get_datahandler(config['datadir'], convert_from)
@@ -279,8 +287,11 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
config['pairs'] = []
# Check timeframes or fall back to timeframe.
for timeframe in timeframes:
config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'],
timeframe))
config['pairs'].extend(src.ohlcv_get_pairs(
config['datadir'],
timeframe,
candle_type=candle_type
))
logger.info(f"Converting candle (OHLCV) data for {config['pairs']}")
for timeframe in timeframes:
@@ -289,10 +300,16 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to:
timerange=None,
fill_missing=False,
drop_incomplete=False,
startup_candles=0)
logger.info(f"Converting {len(data)} candles for {pair}")
startup_candles=0,
candle_type=candle_type)
logger.info(f"Converting {len(data)} {candle_type} candles for {pair}")
if len(data) > 0:
trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data)
trg.ohlcv_store(
pair=pair,
timeframe=timeframe,
data=data,
candle_type=candle_type
)
if erase and convert_from != convert_to:
logger.info(f"Deleting source data for {pair} / {timeframe}")
src.ohlcv_purge(pair=pair, timeframe=timeframe)
src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type)

67
freqtrade/data/dataprovider.py
View File

@@ -13,7 +13,7 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe
from freqtrade.data.history import load_pair_history
from freqtrade.enums import RunMode
from freqtrade.enums import CandleType, RunMode
from freqtrade.exceptions import ExchangeError, OperationalException
from freqtrade.exchange import Exchange, timeframe_to_seconds
@@ -41,7 +41,13 @@ class DataProvider:
"""
self.__slice_index = limit_index
def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
def _set_cached_df(
self,
pair: str,
timeframe: str,
dataframe: DataFrame,
candle_type: CandleType
) -> None:
"""
Store cached Dataframe.
Using private method as this should never be used by a user
@@ -49,8 +55,10 @@ class DataProvider:
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param dataframe: analyzed dataframe
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
self.__cached_pairs[(pair, timeframe)] = (dataframe, datetime.now(timezone.utc))
self.__cached_pairs[(pair, timeframe, candle_type)] = (
dataframe, datetime.now(timezone.utc))
def add_pairlisthandler(self, pairlists) -> None:
"""
@@ -58,13 +66,21 @@ class DataProvider:
"""
self._pairlists = pairlists
def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
def historic_ohlcv(
self,
pair: str,
timeframe: str = None,
candle_type: str = ''
) -> DataFrame:
"""
Get stored historical candle (OHLCV) data
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:param candle_type: '', mark, index, premiumIndex, or funding_rate
"""
saved_pair = (pair, str(timeframe))
_candle_type = CandleType.from_string(
candle_type) if candle_type != '' else self._config['candle_type_def']
saved_pair = (pair, str(timeframe), _candle_type)
if saved_pair not in self.__cached_pairs_backtesting:
timerange = TimeRange.parse_timerange(None if self._config.get(
'timerange') is None else str(self._config.get('timerange')))
@@ -77,26 +93,36 @@ class DataProvider:
timeframe=timeframe or self._config['timeframe'],
datadir=self._config['datadir'],
timerange=timerange,
data_format=self._config.get('dataformat_ohlcv', 'json')
data_format=self._config.get('dataformat_ohlcv', 'json'),
candle_type=_candle_type,
)
return self.__cached_pairs_backtesting[saved_pair].copy()
def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame:
def get_pair_dataframe(
self,
pair: str,
timeframe: str = None,
candle_type: str = ''
) -> DataFrame:
"""
Return pair candle (OHLCV) data, either live or cached historical -- depending
on the runmode.
Only combinations in the pairlist or which have been specified as informative pairs
will be available.
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:return: Dataframe for this pair
:param candle_type: '', mark, index, premiumIndex, or funding_rate
"""
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
# Get live OHLCV data.
data = self.ohlcv(pair=pair, timeframe=timeframe)
data = self.ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type)
else:
# Get historical OHLCV data (cached on disk).
data = self.historic_ohlcv(pair=pair, timeframe=timeframe)
data = self.historic_ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type)
if len(data) == 0:
logger.warning(f"No data found for ({pair}, {timeframe}).")
logger.warning(f"No data found for ({pair}, {timeframe}, {candle_type}).")
return data
def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
@@ -109,7 +135,7 @@ class DataProvider:
combination.
Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
"""
pair_key = (pair, timeframe)
pair_key = (pair, timeframe, self._config.get('candle_type_def', CandleType.SPOT))
if pair_key in self.__cached_pairs:
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
df, date = self.__cached_pairs[pair_key]
@@ -149,6 +175,8 @@ class DataProvider:
Clear pair dataframe cache.
"""
self.__cached_pairs = {}
self.__cached_pairs_backtesting = {}
self.__slice_index = 0
# Exchange functions
@@ -175,20 +203,31 @@ class DataProvider:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
return list(self._exchange._klines.keys())
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
def ohlcv(
self,
pair: str,
timeframe: str = None,
copy: bool = True,
candle_type: str = ''
) -> DataFrame:
"""
Get candle (OHLCV) data for the given pair as DataFrame
Please use the `available_pairs` method to verify which pairs are currently cached.
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param candle_type: '', mark, index, premiumIndex, or funding_rate
:param copy: copy dataframe before returning if True.
Use False only for read-only operations (where the dataframe is not modified)
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
return self._exchange.klines((pair, timeframe or self._config['timeframe']),
copy=copy)
_candle_type = CandleType.from_string(
candle_type) if candle_type != '' else self._config['candle_type_def']
return self._exchange.klines(
(pair, timeframe or self._config['timeframe'], _candle_type),
copy=copy
)
else:
return DataFrame()

126
freqtrade/data/history/hdf5datahandler.py
View File

@@ -6,10 +6,10 @@ from typing import List, Optional
import numpy as np
import pandas as pd
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS,
ListPairsWithTimeframes, TradeList)
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
@@ -22,52 +22,72 @@ class HDF5DataHandler(IDataHandler):
_columns = DEFAULT_DATAFRAME_COLUMNS
@classmethod
def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
"""
_tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name)
for p in datadir.glob("*.h5")]
return [(match[1].replace('_', '/'), match[2]) for match in _tmp
if match and len(match.groups()) > 1]
if trading_mode == TradingMode.FUTURES:
datadir = datadir.joinpath('futures')
_tmp = [
re.search(
cls._OHLCV_REGEX, p.name
) for p in datadir.glob("*.h5")
]
return [
(
cls.rebuild_pair_from_filename(match[1]),
match[2],
CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1]
@classmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]:
"""
Returns a list of all pairs with ohlcv data available in this datadir
for the specified timeframe
:param datadir: Directory to search for ohlcv files
:param timeframe: Timeframe to search pairs for
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: List of Pairs
"""
candle = ""
if candle_type != CandleType.SPOT:
datadir = datadir.joinpath('futures')
candle = f"-{candle_type}"
_tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name)
for p in datadir.glob(f"*{timeframe}.h5")]
_tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.h5)', p.name)
for p in datadir.glob(f"*{timeframe}{candle}.h5")]
# Check if regex found something and only return these results
return [match[0].replace('_', '/') for match in _tmp if match]
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None:
def ohlcv_store(
self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType) -> None:
"""
Store data in hdf5 file.
:param pair: Pair - used to generate filename
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: None
"""
key = self._pair_ohlcv_key(pair, timeframe)
_data = data.copy()
filename = self._pair_data_filename(self._datadir, pair, timeframe)
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
self.create_dir_if_needed(filename)
ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc')
ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date'])
ds.close()
_data.loc[:, self._columns].to_hdf(
filename, key, mode='a', complevel=9, complib='blosc',
format='table', data_columns=['date']
)
def _ohlcv_load(self, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None) -> pd.DataFrame:
timerange: Optional[TimeRange], candle_type: CandleType
) -> pd.DataFrame:
"""
Internal method used to load data for one pair from disk.
Implements the loading and conversion to a Pandas dataframe.
@@ -77,10 +97,16 @@ class HDF5DataHandler(IDataHandler):
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
key = self._pair_ohlcv_key(pair, timeframe)
filename = self._pair_data_filename(self._datadir, pair, timeframe)
filename = self._pair_data_filename(
self._datadir,
pair,
timeframe,
candle_type=candle_type
)
if not filename.exists():
return pd.DataFrame(columns=self._columns)
@@ -99,25 +125,19 @@ class HDF5DataHandler(IDataHandler):
'low': 'float', 'close': 'float', 'volume': 'float'})
return pairdata
def ohlcv_purge(self, pair: str, timeframe: str) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Timeframe (e.g. "5m")
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)
if filename.exists():
filename.unlink()
return True
return False
def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None:
def ohlcv_append(
self,
pair: str,
timeframe: str,
data: pd.DataFrame,
candle_type: CandleType
) -> None:
"""
Append data to existing data structures
:param pair: Pair
:param timeframe: Timeframe this ohlcv data is for
:param data: Data to append.
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
raise NotImplementedError()
@@ -131,7 +151,7 @@ class HDF5DataHandler(IDataHandler):
_tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name)
for p in datadir.glob("*trades.h5")]
# Check if regex found something and only return these results to avoid exceptions.
return [match[0].replace('_', '/') for match in _tmp if match]
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
def trades_store(self, pair: str, data: TradeList) -> None:
"""
@@ -142,11 +162,11 @@ class HDF5DataHandler(IDataHandler):
"""
key = self._pair_trades_key(pair)
ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair),
mode='a', complevel=9, complib='blosc')
ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS),
format='table', data_columns=['timestamp'])
ds.close()
pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf(
self._pair_trades_filename(self._datadir, pair), key,
mode='a', complevel=9, complib='blosc',
format='table', data_columns=['timestamp']
)
def trades_append(self, pair: str, data: TradeList):
"""
@@ -180,34 +200,16 @@ class HDF5DataHandler(IDataHandler):
trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
return trades.values.tolist()
def trades_purge(self, pair: str) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_trades_filename(self._datadir, pair)
if filename.exists():
filename.unlink()
return True
return False
@classmethod
def _get_file_extension(cls):
return "h5"
@classmethod
def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
return f"{pair}/ohlcv/tf_{timeframe}"
# Escape futures pairs to avoid warnings
pair_esc = pair.replace(':', '_')
return f"{pair_esc}/ohlcv/tf_{timeframe}"
@classmethod
def _pair_trades_key(cls, pair: str) -> str:
return f"{pair}/trades"
@classmethod
def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path:
pair_s = misc.pair_to_filename(pair)
filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5')
return filename
@classmethod
def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
pair_s = misc.pair_to_filename(pair)
filename = datadir.joinpath(f'{pair_s}-trades.h5')
return filename

105
freqtrade/data/history/history_utils.py
View File

@@ -5,13 +5,14 @@ from pathlib import Path
from typing import Dict, List, Optional, Tuple
import arrow
from pandas import DataFrame
from pandas import DataFrame, concat
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS
from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe,
trades_remove_duplicates, trades_to_ohlcv)
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.misc import format_ms_time
@@ -29,6 +30,7 @@ def load_pair_history(pair: str,
startup_candles: int = 0,
data_format: str = None,
data_handler: IDataHandler = None,
candle_type: CandleType = CandleType.SPOT
) -> DataFrame:
"""
Load cached ohlcv history for the given pair.
@@ -43,6 +45,7 @@ def load_pair_history(pair: str,
:param startup_candles: Additional candles to load at the start of the period
:param data_handler: Initialized data-handler to use.
Will be initialized from data_format if not set
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
data_handler = get_datahandler(datadir, data_format, data_handler)
@@ -53,6 +56,7 @@ def load_pair_history(pair: str,
fill_missing=fill_up_missing,
drop_incomplete=drop_incomplete,
startup_candles=startup_candles,
candle_type=candle_type
)
@@ -64,6 +68,7 @@ def load_data(datadir: Path,
startup_candles: int = 0,
fail_without_data: bool = False,
data_format: str = 'json',
candle_type: CandleType = CandleType.SPOT
) -> Dict[str, DataFrame]:
"""
Load ohlcv history data for a list of pairs.
@@ -76,6 +81,7 @@ def load_data(datadir: Path,
:param startup_candles: Additional candles to load at the start of the period
:param fail_without_data: Raise OperationalException if no data is found.
:param data_format: Data format which should be used. Defaults to json
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: dict(<pair>:<Dataframe>)
"""
result: Dict[str, DataFrame] = {}
@@ -89,7 +95,8 @@ def load_data(datadir: Path,
datadir=datadir, timerange=timerange,
fill_up_missing=fill_up_missing,
startup_candles=startup_candles,
data_handler=data_handler
data_handler=data_handler,
candle_type=candle_type
)
if not hist.empty:
result[pair] = hist
@@ -99,12 +106,13 @@ def load_data(datadir: Path,
return result
def refresh_data(datadir: Path,
def refresh_data(*, datadir: Path,
timeframe: str,
pairs: List[str],
exchange: Exchange,
data_format: str = None,
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
) -> None:
"""
Refresh ohlcv history data for a list of pairs.
@@ -115,17 +123,24 @@ def refresh_data(datadir: Path,
:param exchange: Exchange object
:param data_format: dataformat to use
:param timerange: Limit data to be loaded to this timerange
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
data_handler = get_datahandler(datadir, data_format)
for idx, pair in enumerate(pairs):
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
timeframe=timeframe, datadir=datadir,
timerange=timerange, exchange=exchange, data_handler=data_handler)
timerange=timerange, exchange=exchange, data_handler=data_handler,
candle_type=candle_type)
def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange],
data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]:
def _load_cached_data_for_updating(
pair: str,
timeframe: str,
timerange: Optional[TimeRange],
data_handler: IDataHandler,
candle_type: CandleType
) -> Tuple[DataFrame, Optional[int]]:
"""
Load cached data to download more data.
If timerange is passed in, checks whether data from an before the stored data will be
@@ -142,7 +157,8 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
# Intentionally don't pass timerange in - since we need to load the full dataset.
data = data_handler.ohlcv_load(pair, timeframe=timeframe,
timerange=None, fill_missing=False,
drop_incomplete=True, warn_no_data=False)
drop_incomplete=True, warn_no_data=False,
candle_type=candle_type)
if not data.empty:
if start and start < data.iloc[0]['date']:
# Earlier data than existing data requested, redownload all
@@ -161,7 +177,10 @@ def _download_pair_history(pair: str, *,
process: str = '',
new_pairs_days: int = 30,
data_handler: IDataHandler = None,
timerange: Optional[TimeRange] = None) -> bool:
timerange: Optional[TimeRange] = None,
candle_type: CandleType,
erase: bool = False,
) -> bool:
"""
Download latest candles from the exchange for the pair and timeframe passed in parameters
The data is downloaded starting from the last correct data that
@@ -173,19 +192,25 @@ def _download_pair_history(pair: str, *,
:param pair: pair to download
:param timeframe: Timeframe (e.g "5m")
:param timerange: range of time to download
:param candle_type: Any of the enum CandleType (must match trading mode!)
:param erase: Erase existing data
:return: bool with success state
"""
data_handler = get_datahandler(datadir, data_handler=data_handler)
try:
if erase:
if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type):
logger.info(f'Deleting existing data for pair {pair}, {timeframe}, {candle_type}.')
logger.info(
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe} '
f'and store in {datadir}.'
f'Download history data for pair: "{pair}" ({process}), timeframe: {timeframe}, '
f'candle type: {candle_type} and store in {datadir}.'
)
# data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange)
data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange,
data_handler=data_handler)
data_handler=data_handler,
candle_type=candle_type)
logger.debug("Current Start: %s",
f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
@@ -198,7 +223,8 @@ def _download_pair_history(pair: str, *,
since_ms=since_ms if since_ms else
arrow.utcnow().shift(
days=-new_pairs_days).int_timestamp * 1000,
is_new_pair=data.empty
is_new_pair=data.empty,
candle_type=candle_type,
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -208,7 +234,7 @@ def _download_pair_history(pair: str, *,
else:
# Run cleaning again to ensure there were no duplicate candles
# Especially between existing and new data.
data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair,
data = clean_ohlcv_dataframe(concat([data, new_dataframe], axis=0), timeframe, pair,
fill_missing=False, drop_incomplete=False)
logger.debug("New Start: %s",
@@ -216,7 +242,7 @@ def _download_pair_history(pair: str, *,
logger.debug("New End: %s",
f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None')
data_handler.ohlcv_store(pair, timeframe, data=data)
data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type)
return True
except Exception:
@@ -227,9 +253,11 @@ def _download_pair_history(pair: str, *,
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
datadir: Path, timerange: Optional[TimeRange] = None,
datadir: Path, trading_mode: str,
timerange: Optional[TimeRange] = None,
new_pairs_days: int = 30, erase: bool = False,
data_format: str = None) -> List[str]:
data_format: str = None,
) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
@@ -237,6 +265,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
"""
pairs_not_available = []
data_handler = get_datahandler(datadir, data_format)
candle_type = CandleType.get_default(trading_mode)
for idx, pair in enumerate(pairs, start=1):
if pair not in exchange.markets:
pairs_not_available.append(pair)
@@ -244,17 +273,29 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
continue
for timeframe in timeframes:
if erase:
if data_handler.ohlcv_purge(pair, timeframe):
logger.info(
f'Deleting existing data for pair {pair}, interval {timeframe}.')
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
process = f'{idx}/{len(pairs)}'
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(timeframe), new_pairs_days=new_pairs_days)
timeframe=str(timeframe), new_pairs_days=new_pairs_days,
candle_type=candle_type,
erase=erase)
if trading_mode == 'futures':
# Predefined candletype (and timeframe) depending on exchange
# Downloads what is necessary to backtest based on futures data.
tf_mark = exchange._ft_has['mark_ohlcv_timeframe']
fr_candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price'])
# All exchanges need FundingRate for futures trading.
# The timeframe is aligned to the mark-price timeframe.
for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type):
_download_pair_history(pair=pair, process=process,
datadir=datadir, exchange=exchange,
timerange=timerange, data_handler=data_handler,
timeframe=str(tf_mark), new_pairs_days=new_pairs_days,
candle_type=funding_candle_type,
erase=erase)
return pairs_not_available
@@ -353,10 +394,16 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
return pairs_not_available
def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
datadir: Path, timerange: TimeRange, erase: bool = False,
data_format_ohlcv: str = 'json',
data_format_trades: str = 'jsongz') -> None:
def convert_trades_to_ohlcv(
pairs: List[str],
timeframes: List[str],
datadir: Path,
timerange: TimeRange,
erase: bool = False,
data_format_ohlcv: str = 'json',
data_format_trades: str = 'jsongz',
candle_type: CandleType = CandleType.SPOT
) -> None:
"""
Convert stored trades data to ohlcv data
"""
@@ -367,12 +414,12 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
trades = data_handler_trades.trades_load(pair)
for timeframe in timeframes:
if erase:
if data_handler_ohlcv.ohlcv_purge(pair, timeframe):
if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.')
try:
ohlcv = trades_to_ohlcv(trades, timeframe)
# Store ohlcv
data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv)
data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
except ValueError:
logger.exception(f'Could not convert {pair} to OHLCV.')

128
freqtrade/data/history/idatahandler.py
View File

@@ -4,6 +4,7 @@ It's subclasses handle and storing data from disk.
"""
import logging
import re
from abc import ABC, abstractclassmethod, abstractmethod
from copy import deepcopy
from datetime import datetime, timezone
@@ -12,9 +13,11 @@ from typing import List, Optional, Type
from pandas import DataFrame
from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import ListPairsWithTimeframes, TradeList
from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe
from freqtrade.enums import CandleType, TradingMode
from freqtrade.exchange import timeframe_to_seconds
@@ -23,40 +26,54 @@ logger = logging.getLogger(__name__)
class IDataHandler(ABC):
_OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)'
def __init__(self, datadir: Path) -> None:
self._datadir = datadir
@classmethod
def _get_file_extension(cls) -> str:
"""
Get file extension for this particular datahandler
"""
raise NotImplementedError()
@abstractclassmethod
def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
"""
@abstractclassmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]:
"""
Returns a list of all pairs with ohlcv data available in this datadir
for the specified timeframe
:param datadir: Directory to search for ohlcv files
:param timeframe: Timeframe to search pairs for
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: List of Pairs
"""
@abstractmethod
def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None:
def ohlcv_store(
self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType) -> None:
"""
Store ohlcv data.
:param pair: Pair - used to generate filename
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: None
"""
@abstractmethod
def _ohlcv_load(self, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None,
def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange],
candle_type: CandleType
) -> DataFrame:
"""
Internal method used to load data for one pair from disk.
@@ -67,25 +84,38 @@ class IDataHandler(ABC):
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
@abstractmethod
def ohlcv_purge(self, pair: str, timeframe: str) -> bool:
def ohlcv_purge(self, pair: str, timeframe: str, candle_type: CandleType) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Timeframe (e.g. "5m")
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
if filename.exists():
filename.unlink()
return True
return False
@abstractmethod
def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None:
def ohlcv_append(
self,
pair: str,
timeframe: str,
data: DataFrame,
candle_type: CandleType
) -> None:
"""
Append data to existing data structures
:param pair: Pair
:param timeframe: Timeframe this ohlcv data is for
:param data: Data to append.
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
@abstractclassmethod
@@ -123,13 +153,17 @@ class IDataHandler(ABC):
:return: List of trades
"""
@abstractmethod
def trades_purge(self, pair: str) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_trades_filename(self._datadir, pair)
if filename.exists():
filename.unlink()
return True
return False
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
"""
@@ -141,12 +175,55 @@ class IDataHandler(ABC):
"""
return trades_remove_duplicates(self._trades_load(pair, timerange=timerange))
@classmethod
def create_dir_if_needed(cls, datadir: Path):
"""
Creates datadir if necessary
should only create directories for "futures" mode at the moment.
"""
if not datadir.parent.is_dir():
datadir.parent.mkdir()
@classmethod
def _pair_data_filename(
cls,
datadir: Path,
pair: str,
timeframe: str,
candle_type: CandleType
) -> Path:
pair_s = misc.pair_to_filename(pair)
candle = ""
if candle_type != CandleType.SPOT:
datadir = datadir.joinpath('futures')
candle = f"-{candle_type}"
filename = datadir.joinpath(
f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}')
return filename
@classmethod
def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
pair_s = misc.pair_to_filename(pair)
filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
return filename
@staticmethod
def rebuild_pair_from_filename(pair: str) -> str:
"""
Rebuild pair name from filename
Assumes a asset name of max. 7 length to also support BTC-PERP and BTC-PERP:USD names.
"""
res = re.sub(r'^(([A-Za-z]{1,10})|^([A-Za-z\-]{1,6}))(_)', r'\g<1>/', pair, 1)
res = re.sub('_', ':', res, 1)
return res
def ohlcv_load(self, pair, timeframe: str,
candle_type: CandleType,
timerange: Optional[TimeRange] = None,
fill_missing: bool = True,
drop_incomplete: bool = True,
startup_candles: int = 0,
warn_no_data: bool = True
warn_no_data: bool = True,
) -> DataFrame:
"""
Load cached candle (OHLCV) data for the given pair.
@@ -158,6 +235,7 @@ class IDataHandler(ABC):
:param drop_incomplete: Drop last candle assuming it may be incomplete.
:param startup_candles: Additional candles to load at the start of the period
:param warn_no_data: Log a warning message when no data is found
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
# Fix startup period
@@ -165,17 +243,21 @@ class IDataHandler(ABC):
if startup_candles > 0 and timerange_startup:
timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles)
pairdf = self._ohlcv_load(pair, timeframe,
timerange=timerange_startup)
if self._check_empty_df(pairdf, pair, timeframe, warn_no_data):
pairdf = self._ohlcv_load(
pair,
timeframe,
timerange=timerange_startup,
candle_type=candle_type
)
if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
return pairdf
else:
enddate = pairdf.iloc[-1]['date']
if timerange_startup:
self._validate_pairdata(pair, pairdf, timerange_startup)
self._validate_pairdata(pair, pairdf, timeframe, candle_type, timerange_startup)
pairdf = trim_dataframe(pairdf, timerange_startup)
if self._check_empty_df(pairdf, pair, timeframe, warn_no_data):
if self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data):
return pairdf
# incomplete candles should only be dropped if we didn't trim the end beforehand.
@@ -184,23 +266,25 @@ class IDataHandler(ABC):
fill_missing=fill_missing,
drop_incomplete=(drop_incomplete and
enddate == pairdf.iloc[-1]['date']))
self._check_empty_df(pairdf, pair, timeframe, warn_no_data)
self._check_empty_df(pairdf, pair, timeframe, candle_type, warn_no_data)
return pairdf
def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, warn_no_data: bool):
def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str,
candle_type: CandleType, warn_no_data: bool):
"""
Warn on empty dataframe
"""
if pairdf.empty:
if warn_no_data:
logger.warning(
f'No history data for pair: "{pair}", timeframe: {timeframe}. '
'Use `freqtrade download-data` to download the data'
f"No history for {pair}, {candle_type}, {timeframe} found. "
"Use `freqtrade download-data` to download the data"
)
return True
return False
def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange):
def _validate_pairdata(self, pair, pairdata: DataFrame, timeframe: str,
candle_type: CandleType, timerange: TimeRange):
"""
Validates pairdata for missing data at start end end and logs warnings.
:param pairdata: Dataframe to validate
@@ -210,12 +294,12 @@ class IDataHandler(ABC):
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
if pairdata.iloc[0]['date'] > start:
logger.warning(f"Missing data at start for pair {pair}, "
logger.warning(f"{pair}, {candle_type}, {timeframe}, "
f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}")
if timerange.stoptype == 'date':
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
if pairdata.iloc[-1]['date'] < stop:
logger.warning(f"Missing data at end for pair {pair}, "
logger.warning(f"{pair}, {candle_type}, {timeframe}, "
f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}")

94
freqtrade/data/history/jsondatahandler.py
View File

@@ -10,6 +10,7 @@ from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList
from freqtrade.data.converter import trades_dict_to_list
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
@@ -23,33 +24,49 @@ class JsonDataHandler(IDataHandler):
_columns = DEFAULT_DATAFRAME_COLUMNS
@classmethod
def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes:
def ohlcv_get_available_data(
cls, datadir: Path, trading_mode: TradingMode) -> ListPairsWithTimeframes:
"""
Returns a list of all pairs with ohlcv data available in this datadir
:param datadir: Directory to search for ohlcv files
:param trading_mode: trading-mode to be used
:return: List of Tuples of (pair, timeframe)
"""
_tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name)
for p in datadir.glob(f"*.{cls._get_file_extension()}")]
return [(match[1].replace('_', '/'), match[2]) for match in _tmp
if match and len(match.groups()) > 1]
if trading_mode == 'futures':
datadir = datadir.joinpath('futures')
_tmp = [
re.search(
cls._OHLCV_REGEX, p.name
) for p in datadir.glob(f"*.{cls._get_file_extension()}")]
return [
(
cls.rebuild_pair_from_filename(match[1]),
match[2],
CandleType.from_string(match[3])
) for match in _tmp if match and len(match.groups()) > 1]
@classmethod
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]:
def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]:
"""
Returns a list of all pairs with ohlcv data available in this datadir
for the specified timeframe
:param datadir: Directory to search for ohlcv files
:param timeframe: Timeframe to search pairs for
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: List of Pairs
"""
candle = ""
if candle_type != CandleType.SPOT:
datadir = datadir.joinpath('futures')
candle = f"-{candle_type}"
_tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name)
for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")]
_tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.json)', p.name)
for p in datadir.glob(f"*{timeframe}{candle}.{cls._get_file_extension()}")]
# Check if regex found something and only return these results
return [match[0].replace('_', '/') for match in _tmp if match]
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None:
def ohlcv_store(
self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType) -> None:
"""
Store data in json format "values".
format looks as follows:
@@ -57,9 +74,11 @@ class JsonDataHandler(IDataHandler):
:param pair: Pair - used to generate filename
:param timeframe: Timeframe - used to generate filename
:param data: Dataframe containing OHLCV data
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: None
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
self.create_dir_if_needed(filename)
_data = data.copy()
# Convert date to int
_data['date'] = _data['date'].view(np.int64) // 1000 // 1000
@@ -70,7 +89,7 @@ class JsonDataHandler(IDataHandler):
compression='gzip' if self._use_zip else None)
def _ohlcv_load(self, pair: str, timeframe: str,
timerange: Optional[TimeRange] = None,
timerange: Optional[TimeRange], candle_type: CandleType
) -> DataFrame:
"""
Internal method used to load data for one pair from disk.
@@ -81,9 +100,10 @@ class JsonDataHandler(IDataHandler):
:param timerange: Limit data to be loaded to this timerange.
Optionally implemented by subclasses to avoid loading
all data where possible.
:param candle_type: Any of the enum CandleType (must match trading mode!)
:return: DataFrame with ohlcv data, or empty DataFrame
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)
filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type)
if not filename.exists():
return DataFrame(columns=self._columns)
try:
@@ -100,25 +120,19 @@ class JsonDataHandler(IDataHandler):
infer_datetime_format=True)
return pairdata
def ohlcv_purge(self, pair: str, timeframe: str) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param timeframe: Timeframe (e.g. "5m")
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_data_filename(self._datadir, pair, timeframe)
if filename.exists():
filename.unlink()
return True
return False
def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None:
def ohlcv_append(
self,
pair: str,
timeframe: str,
data: DataFrame,
candle_type: CandleType
) -> None:
"""
Append data to existing data structures
:param pair: Pair
:param timeframe: Timeframe this ohlcv data is for
:param data: Data to append.
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
raise NotImplementedError()
@@ -132,7 +146,7 @@ class JsonDataHandler(IDataHandler):
_tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name)
for p in datadir.glob(f"*trades.{cls._get_file_extension()}")]
# Check if regex found something and only return these results to avoid exceptions.
return [match[0].replace('_', '/') for match in _tmp if match]
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
def trades_store(self, pair: str, data: TradeList) -> None:
"""
@@ -174,34 +188,10 @@ class JsonDataHandler(IDataHandler):
pass
return tradesdata
def trades_purge(self, pair: str) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_trades_filename(self._datadir, pair)
if filename.exists():
filename.unlink()
return True
return False
@classmethod
def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path:
pair_s = misc.pair_to_filename(pair)
filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}')
return filename
@classmethod
def _get_file_extension(cls):
return "json.gz" if cls._use_zip else "json"
@classmethod
def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
pair_s = misc.pair_to_filename(pair)
filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
return filename
class JsonGzDataHandler(JsonDataHandler):

42
freqtrade/edge/edge_positioning.py
View File

@@ -13,7 +13,7 @@ from pandas import DataFrame
from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.enums import RunMode, SellType
from freqtrade.enums import CandleType, ExitType, RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.exchange import timeframe_to_seconds
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -116,11 +116,12 @@ class Edge:
timeframe=self.strategy.timeframe,
timerange=timerange_startup,
data_format=self.config.get('dataformat_ohlcv', 'json'),
candle_type=self.config.get('candle_type_def', CandleType.SPOT),
)
# Download informative pairs too
res = defaultdict(list)
for p, t in self.strategy.gather_informative_pairs():
res[t].append(p)
for pair, timeframe, _ in self.strategy.gather_informative_pairs():
res[timeframe].append(pair)
for timeframe, inf_pairs in res.items():
timerange_startup = deepcopy(self._timerange)
timerange_startup.subtract_start(timeframe_to_seconds(
@@ -132,6 +133,7 @@ class Edge:
timeframe=timeframe,
timerange=timerange_startup,
data_format=self.config.get('dataformat_ohlcv', 'json'),
candle_type=self.config.get('candle_type_def', CandleType.SPOT),
)
data = load_data(
@@ -141,6 +143,7 @@ class Edge:
timerange=self._timerange,
startup_candles=self.strategy.startup_candle_count,
data_format=self.config.get('dataformat_ohlcv', 'json'),
candle_type=self.config.get('candle_type_def', CandleType.SPOT),
)
if not data:
@@ -159,7 +162,9 @@ class Edge:
logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']
# TODO: Should edge support shorts? needs to be investigated further
# * (add enter_short exit_short)
headers = ['date', 'open', 'high', 'low', 'close', 'enter_long', 'exit_long']
trades: list = []
for pair, pair_data in preprocessed.items():
@@ -167,8 +172,13 @@ class Edge:
pair_data = pair_data.sort_values(by=['date'])
pair_data = pair_data.reset_index(drop=True)
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
df_analyzed = self.strategy.advise_exit(
dataframe=self.strategy.advise_entry(
dataframe=pair_data,
metadata={'pair': pair}
),
metadata={'pair': pair}
)[headers].copy()
trades += self._find_trades_for_stoploss_range(df_analyzed, pair, self._stoploss_range)
@@ -219,9 +229,11 @@ class Edge:
"""
final = []
for pair, info in self._cached_pairs.items():
if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
pair in pairs:
if (
info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2))
and info.winrate > float(self.edge_config.get('minimum_winrate', 0.60))
and pair in pairs
):
final.append(pair)
if self._final_pairs != final:
@@ -246,8 +258,8 @@ class Edge:
"""
final = []
for pair, info in self._cached_pairs.items():
if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
if (info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and
info.winrate > float(self.edge_config.get('minimum_winrate', 0.60))):
final.append({
'Pair': pair,
'Winrate': info.winrate,
@@ -382,8 +394,8 @@ class Edge:
return final
def _find_trades_for_stoploss_range(self, df, pair, stoploss_range):
buy_column = df['buy'].values
sell_column = df['sell'].values
buy_column = df['enter_long'].values
sell_column = df['exit_long'].values
date_column = df['date'].values
ohlc_columns = df[['open', 'high', 'low', 'close']].values
@@ -448,7 +460,7 @@ class Edge:
if stop_index <= sell_index:
exit_index = open_trade_index + stop_index
exit_type = SellType.STOP_LOSS
exit_type = ExitType.STOP_LOSS
exit_price = stop_price
elif stop_index > sell_index:
# If exit is SELL then we exit at the next candle
@@ -458,7 +470,7 @@ class Edge:
if len(ohlc_columns) - 1 < exit_index:
break
exit_type = SellType.SELL_SIGNAL
exit_type = ExitType.EXIT_SIGNAL
exit_price = ohlc_columns[exit_index, 0]
trade = {'pair': pair,

9
freqtrade/enums/__init__.py
View File

@@ -1,7 +1,12 @@
# flake8: noqa: F401
from freqtrade.enums.backteststate import BacktestState
from freqtrade.enums.candletype import CandleType
from freqtrade.enums.exitchecktuple import ExitCheckTuple
from freqtrade.enums.exittype import ExitType
from freqtrade.enums.marginmode import MarginMode
from freqtrade.enums.ordertypevalue import OrderTypeValues
from freqtrade.enums.rpcmessagetype import RPCMessageType
from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
from freqtrade.enums.selltype import SellType
from freqtrade.enums.signaltype import SignalTagType, SignalType
from freqtrade.enums.signaltype import SignalDirection, SignalTagType, SignalType
from freqtrade.enums.state import State
from freqtrade.enums.tradingmode import TradingMode

27
freqtrade/enums/candletype.py Normal file
View File

@@ -0,0 +1,27 @@
from enum import Enum
class CandleType(str, Enum):
"""Enum to distinguish candle types"""
SPOT = "spot"
FUTURES = "futures"
MARK = "mark"
INDEX = "index"
PREMIUMINDEX = "premiumIndex"
# TODO: Could take up less memory if these weren't a CandleType
FUNDING_RATE = "funding_rate"
# BORROW_RATE = "borrow_rate" # * unimplemented
@staticmethod
def from_string(value: str) -> 'CandleType':
if not value:
# Default to spot
return CandleType.SPOT
return CandleType(value)
@staticmethod
def get_default(trading_mode: str) -> 'CandleType':
if trading_mode == 'futures':
return CandleType.FUTURES
return CandleType.SPOT

17
freqtrade/enums/exitchecktuple.py Normal file
View File

@@ -0,0 +1,17 @@
from freqtrade.enums.exittype import ExitType
class ExitCheckTuple:
"""
NamedTuple for Exit type + reason
"""
exit_type: ExitType
exit_reason: str = ''
def __init__(self, exit_type: ExitType, exit_reason: str = ''):
self.exit_type = exit_type
self.exit_reason = exit_reason or exit_type.value
@property
def exit_flag(self):
return self.exit_type != ExitType.NONE

12
freqtrade/enums/selltype.py → freqtrade/enums/exittype.py
View File

@@ -1,18 +1,18 @@
from enum import Enum
class SellType(Enum):
class ExitType(Enum):
"""
Enum to distinguish between sell reasons
Enum to distinguish between exit reasons
"""
ROI = "roi"
STOP_LOSS = "stop_loss"
STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange"
TRAILING_STOP_LOSS = "trailing_stop_loss"
SELL_SIGNAL = "sell_signal"
FORCE_SELL = "force_sell"
EMERGENCY_SELL = "emergency_sell"
CUSTOM_SELL = "custom_sell"
EXIT_SIGNAL = "exit_signal"
FORCE_EXIT = "force_exit"
EMERGENCY_EXIT = "emergency_exit"
CUSTOM_EXIT = "custom_exit"
NONE = ""
def __str__(self):

12
freqtrade/enums/marginmode.py Normal file
View File

@@ -0,0 +1,12 @@
from enum import Enum
class MarginMode(Enum):
"""
Enum to distinguish between
cross margin/futures margin_mode and
isolated margin/futures margin_mode
"""
CROSS = "cross"
ISOLATED = "isolated"
NONE = ''

6
freqtrade/enums/ordertypevalue.py Normal file
View File

@@ -0,0 +1,6 @@
from enum import Enum
class OrderTypeValues(str, Enum):
limit = 'limit'
market = 'market'

15
freqtrade/enums/rpcmessagetype.py
View File

@@ -5,12 +5,15 @@ class RPCMessageType(Enum):
STATUS = 'status'
WARNING = 'warning'
STARTUP = 'startup'
BUY = 'buy'
BUY_FILL = 'buy_fill'
BUY_CANCEL = 'buy_cancel'
SELL = 'sell'
SELL_FILL = 'sell_fill'
SELL_CANCEL = 'sell_cancel'
ENTRY = 'entry'
ENTRY_FILL = 'entry_fill'
ENTRY_CANCEL = 'entry_cancel'
EXIT = 'exit'
EXIT_FILL = 'exit_fill'
EXIT_CANCEL = 'exit_cancel'
PROTECTION_TRIGGER = 'protection_trigger'
PROTECTION_TRIGGER_GLOBAL = 'protection_trigger_global'

16
freqtrade/enums/signaltype.py
View File

@@ -3,14 +3,22 @@ from enum import Enum
class SignalType(Enum):
"""
Enum to distinguish between buy and sell signals
Enum to distinguish between enter and exit signals
"""
BUY = "buy"
SELL = "sell"
ENTER_LONG = "enter_long"
EXIT_LONG = "exit_long"
ENTER_SHORT = "enter_short"
EXIT_SHORT = "exit_short"
class SignalTagType(Enum):
"""
Enum for signal columns
"""
BUY_TAG = "buy_tag"
ENTER_TAG = "enter_tag"
EXIT_TAG = "exit_tag"
class SignalDirection(str, Enum):
LONG = 'long'
SHORT = 'short'

11
freqtrade/enums/tradingmode.py Normal file
View File

@@ -0,0 +1,11 @@
from enum import Enum
class TradingMode(str, Enum):
"""
Enum to distinguish between
spot, margin, futures or any other trading method
"""
SPOT = "spot"
MARGIN = "margin"
FUTURES = "futures"

2
freqtrade/exceptions.py
View File

@@ -1,5 +1,3 @@
class FreqtradeException(Exception):
"""
Freqtrade base exception. Handled at the outermost level.

3
freqtrade/exchange/__init__.py
View File

@@ -5,6 +5,7 @@ from freqtrade.exchange.exchange import Exchange
# isort: on
from freqtrade.exchange.bibox import Bibox
from freqtrade.exchange.binance import Binance
from freqtrade.exchange.bitpanda import Bitpanda
from freqtrade.exchange.bittrex import Bittrex
from freqtrade.exchange.bybit import Bybit
from freqtrade.exchange.coinbasepro import Coinbasepro
@@ -17,5 +18,7 @@ from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
from freqtrade.exchange.ftx import Ftx
from freqtrade.exchange.gateio import Gateio
from freqtrade.exchange.hitbtc import Hitbtc
from freqtrade.exchange.huobi import Huobi
from freqtrade.exchange.kraken import Kraken
from freqtrade.exchange.kucoin import Kucoin
from freqtrade.exchange.okx import Okx

7
freqtrade/exchange/bibox.py
View File

@@ -20,4 +20,9 @@ class Bibox(Exchange):
# fetchCurrencies API point requires authentication for Bibox,
# so switch it off for Freqtrade load_markets()
_ccxt_config: Dict = {"has": {"fetchCurrencies": False}}
@property
def _ccxt_config(self) -> Dict:
# Parameters to add directly to ccxt sync/async initialization.
config = {"has": {"fetchCurrencies": False}}
config.update(super()._ccxt_config)
return config

213
freqtrade/exchange/binance.py
View File

@@ -1,14 +1,18 @@
""" Binance exchange subclass """
import json
import logging
from typing import Dict, List
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Tuple
import arrow
import ccxt
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.enums import CandleType, MarginMode, TradingMode
from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
from freqtrade.misc import deep_merge_dicts
logger = logging.getLogger(__name__)
@@ -18,93 +22,186 @@ class Binance(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"stoploss_order_types": {"limit": "stop_loss_limit"},
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"time_in_force_parameter": "timeInForce",
"ohlcv_candle_limit": 1000,
"trades_pagination": "id",
"trades_pagination_arg": "fromId",
"l2_limit_range": [5, 10, 20, 50, 100, 500, 1000],
"ccxt_futures_name": "future"
}
_ft_has_futures: Dict = {
"stoploss_order_types": {"limit": "stop"},
"tickers_have_price": False,
}
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
_supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
# TradingMode.SPOT always supported and not required in this list
# (TradingMode.MARGIN, MarginMode.CROSS),
# (TradingMode.FUTURES, MarginMode.CROSS),
(TradingMode.FUTURES, MarginMode.ISOLATED)
]
def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
Returns True if adjustment is necessary.
:param side: "buy" or "sell"
"""
return order['type'] == 'stop_loss_limit' and stop_loss > float(order['info']['stopPrice'])
@retrier(retries=0)
def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict:
ordertype = 'stop' if self.trading_mode == TradingMode.FUTURES else 'stop_loss_limit'
return order['type'] == ordertype and (
(side == "sell" and stop_loss > float(order['info']['stopPrice'])) or
(side == "buy" and stop_loss < float(order['info']['stopPrice']))
)
def get_tickers(self, symbols: List[str] = None, cached: bool = False) -> Dict:
tickers = super().get_tickers(symbols=symbols, cached=cached)
if self.trading_mode == TradingMode.FUTURES:
# Binance's future result has no bid/ask values.
# Therefore we must fetch that from fetch_bids_asks and combine the two results.
bidsasks = self.fetch_bids_asks(symbols, cached)
tickers = deep_merge_dicts(bidsasks, tickers, allow_null_overrides=False)
return tickers
@retrier
def _set_leverage(
self,
leverage: float,
pair: Optional[str] = None,
trading_mode: Optional[TradingMode] = None
):
"""
creates a stoploss limit order.
this stoploss-limit is binance-specific.
It may work with a limited number of other exchanges, but this has not been tested yet.
Set's the leverage before making a trade, in order to not
have the same leverage on every trade
"""
# Limit price threshold: As limit price should always be below stop-price
limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
rate = stop_price * limit_price_pct
trading_mode = trading_mode or self.trading_mode
ordertype = "stop_loss_limit"
stop_price = self.price_to_precision(pair, stop_price)
# Ensure rate is less than stop price
if stop_price <= rate:
raise OperationalException(
'In stoploss limit order, stop price should be more than limit price')
if self._config['dry_run']:
dry_order = self.create_dry_run_order(
pair, ordertype, "sell", amount, stop_price)
return dry_order
if self._config['dry_run'] or trading_mode != TradingMode.FUTURES:
return
try:
params = self._params.copy()
params.update({'stopPrice': stop_price})
amount = self.amount_to_precision(pair, amount)
rate = self.price_to_precision(pair, rate)
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
amount=amount, price=rate, params=params)
logger.info('stoploss limit order added for %s. '
'stop price: %s. limit: %s', pair, stop_price, rate)
self._log_exchange_response('create_stoploss_order', order)
return order
except ccxt.InsufficientFunds as e:
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Tried to sell amount {amount} at rate {rate}. '
f'Message: {e}') from e
except ccxt.InvalidOrder as e:
# Errors:
# `binance Order would trigger immediately.`
raise InvalidOrderException(
f'Could not create {ordertype} sell order on market {pair}. '
f'Tried to sell amount {amount} at rate {rate}. '
f'Message: {e}') from e
self._api.set_leverage(symbol=pair, leverage=round(leverage))
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
async def _async_get_historic_ohlcv(self, pair: str, timeframe: str,
since_ms: int, is_new_pair: bool
) -> List:
since_ms: int, candle_type: CandleType,
is_new_pair: bool = False, raise_: bool = False,
) -> Tuple[str, str, str, List]:
"""
Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date
Does not work for other exchanges, which don't return the earliest data when called with "0"
:param candle_type: Any of the enum CandleType (must match trading mode!)
"""
if is_new_pair:
x = await self._async_get_candle_history(pair, timeframe, 0)
if x and x[2] and x[2][0] and x[2][0][0] > since_ms:
x = await self._async_get_candle_history(pair, timeframe, candle_type, 0)
if x and x[3] and x[3][0] and x[3][0][0] > since_ms:
# Set starting date to first available candle.
since_ms = x[2][0][0]
since_ms = x[3][0][0]
logger.info(f"Candle-data for {pair} available starting with "
f"{arrow.get(since_ms // 1000).isoformat()}.")
return await super()._async_get_historic_ohlcv(
pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair)
pair=pair,
timeframe=timeframe,
since_ms=since_ms,
is_new_pair=is_new_pair,
raise_=raise_,
candle_type=candle_type
)
def funding_fee_cutoff(self, open_date: datetime):
"""
:param open_date: The open date for a trade
:return: The cutoff open time for when a funding fee is charged
"""
return open_date.minute > 0 or (open_date.minute == 0 and open_date.second > 15)
def dry_run_liquidation_price(
self,
pair: str,
open_rate: float, # Entry price of position
is_short: bool,
position: float, # Absolute value of position size
wallet_balance: float, # Or margin balance
mm_ex_1: float = 0.0, # (Binance) Cross only
upnl_ex_1: float = 0.0, # (Binance) Cross only
) -> Optional[float]:
"""
MARGIN: https://www.binance.com/en/support/faq/f6b010588e55413aa58b7d63ee0125ed
PERPETUAL: https://www.binance.com/en/support/faq/b3c689c1f50a44cabb3a84e663b81d93
:param exchange_name:
:param open_rate: (EP1) Entry price of position
:param is_short: True if the trade is a short, false otherwise
:param position: Absolute value of position size (in base currency)
:param wallet_balance: (WB)
Cross-Margin Mode: crossWalletBalance
Isolated-Margin Mode: isolatedWalletBalance
:param maintenance_amt:
# * Only required for Cross
:param mm_ex_1: (TMM)
Cross-Margin Mode: Maintenance Margin of all other contracts, excluding Contract 1
Isolated-Margin Mode: 0
:param upnl_ex_1: (UPNL)
Cross-Margin Mode: Unrealized PNL of all other contracts, excluding Contract 1.
Isolated-Margin Mode: 0
"""
side_1 = -1 if is_short else 1
position = abs(position)
cross_vars = upnl_ex_1 - mm_ex_1 if self.margin_mode == MarginMode.CROSS else 0.0
# mm_ratio: Binance's formula specifies maintenance margin rate which is mm_ratio * 100%
# maintenance_amt: (CUM) Maintenance Amount of position
mm_ratio, maintenance_amt = self.get_maintenance_ratio_and_amt(pair, position)
if (maintenance_amt is None):
raise OperationalException(
"Parameter maintenance_amt is required by Binance.liquidation_price"
f"for {self.trading_mode.value}"
)
if self.trading_mode == TradingMode.FUTURES:
return (
(
(wallet_balance + cross_vars + maintenance_amt) -
(side_1 * position * open_rate)
) / (
(position * mm_ratio) - (side_1 * position)
)
)
else:
raise OperationalException(
"Freqtrade only supports isolated futures for leverage trading")
@retrier
def load_leverage_tiers(self) -> Dict[str, List[Dict]]:
if self.trading_mode == TradingMode.FUTURES:
if self._config['dry_run']:
leverage_tiers_path = (
Path(__file__).parent / 'binance_leverage_tiers.json'
)
with open(leverage_tiers_path) as json_file:
return json.load(json_file)
else:
try:
return self._api.fetch_leverage_tiers()
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(f'Could not fetch leverage amounts due to'
f'{e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
else:
return {}

16481
freqtrade/exchange/binance_leverage_tiers.json Normal file
View File

File diff suppressed because it is too large Load Diff

37
freqtrade/exchange/bitpanda.py Normal file
View File

@@ -0,0 +1,37 @@
""" Bitpanda exchange subclass """
import logging
from datetime import datetime, timezone
from typing import Dict, List, Optional
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Bitpanda(Exchange):
"""
Bitpanda exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
"""
def get_trades_for_order(self, order_id: str, pair: str, since: datetime,
params: Optional[Dict] = None) -> List:
"""
Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id.
The "since" argument passed in is coming from the database and is in UTC,
as timezone-native datetime object.
From the python documentation:
> Naive datetime instances are assumed to represent local time
Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the
transformation from local timezone to UTC.
This works for timezones UTC+ since then the result will contain trades from a few hours
instead of from the last 5 seconds, however fails for UTC- timezones,
since we're then asking for trades with a "since" argument in the future.
:param order_id order_id: Order-id as given when creating the order
:param pair: Pair the order is for
:param since: datetime object of the order creation time. Assumes object is in UTC.
"""
params = {'to': int(datetime.now(timezone.utc).timestamp() * 1000)}
return super().get_trades_for_order(order_id, pair, since, params)

10
freqtrade/exchange/bybit.py
View File

@@ -1,7 +1,8 @@
""" Bybit exchange subclass """
import logging
from typing import Dict
from typing import Dict, List, Tuple
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.exchange import Exchange
@@ -20,4 +21,11 @@ class Bybit(Exchange):
_ft_has: Dict = {
"ohlcv_candle_limit": 200,
"ccxt_futures_name": "linear"
}
_supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
# TradingMode.SPOT always supported and not required in this list
# (TradingMode.FUTURES, MarginMode.CROSS),
# (TradingMode.FUTURES, MarginMode.ISOLATED)
]

66
freqtrade/exchange/common.py
View File

@@ -4,9 +4,20 @@ import time
from functools import wraps
from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError
from freqtrade.mixins import LoggingMixin
logger = logging.getLogger(__name__)
__logging_mixin = None
def _get_logging_mixin():
# Logging-mixin to cache kucoin responses
# Only to be used in retrier
global __logging_mixin
if not __logging_mixin:
__logging_mixin = LoggingMixin(logger)
return __logging_mixin
# Maximum default retry count.
@@ -16,17 +27,27 @@ API_FETCH_ORDER_RETRY_COUNT = 5
BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "
"Details in https://github.com/freqtrade/freqtrade/issues/1983",
"phemex": "Does not provide history. ",
"phemex": "Does not provide history.",
"probit": "Requires additional, regular calls to `signIn()`.",
"poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
}
MAP_EXCHANGE_CHILDCLASS = {
'binanceus': 'binance',
'binanceje': 'binance',
'binanceusdm': 'binance',
'okex': 'okx',
}
SUPPORTED_EXCHANGES = [
'binance',
'bittrex',
'ftx',
'gateio',
'huobi',
'kraken',
'okx',
]
EXCHANGE_HAS_REQUIRED = [
# Required / private
@@ -44,10 +65,17 @@ EXCHANGE_HAS_REQUIRED = [
EXCHANGE_HAS_OPTIONAL = [
# Private
'fetchMyTrades', # Trades for order - fee detection
# 'setLeverage', # Margin/Futures trading
# 'setMarginMode', # Margin/Futures trading
# 'fetchFundingHistory', # Futures trading
# Public
'fetchOrderBook', 'fetchL2OrderBook', 'fetchTicker', # OR for pricing
'fetchTickers', # For volumepairlist?
'fetchTrades', # Downloading trades data
# 'fetchFundingRateHistory', # Futures trading
# 'fetchPositions', # Futures trading
# 'fetchLeverageTiers', # Futures initialization
# 'fetchMarketLeverageTiers', # Futures initialization
]
@@ -74,21 +102,33 @@ def calculate_backoff(retrycount, max_retries):
def retrier_async(f):
async def wrapper(*args, **kwargs):
count = kwargs.pop('count', API_RETRY_COUNT)
kucoin = args[0].name == "KuCoin" # Check if the exchange is KuCoin.
try:
return await f(*args, **kwargs)
except TemporaryError as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
msg = f'{f.__name__}() returned exception: "{ex}". '
if count > 0:
logger.warning('retrying %s() still for %s times', f.__name__, count)
msg += f'Retrying still for {count} times.'
count -= 1
kwargs.update({'count': count})
kwargs['count'] = count
if isinstance(ex, DDosProtection):
backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT)
logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
await asyncio.sleep(backoff_delay)
if kucoin and "429000" in str(ex):
# Temporary fix for 429000 error on kucoin
# see https://github.com/freqtrade/freqtrade/issues/5700 for details.
_get_logging_mixin().log_once(
f"Kucoin 429 error, avoid triggering DDosProtection backoff delay. "
f"{count} tries left before giving up", logmethod=logger.warning)
# Reset msg to avoid logging too many times.
msg = ''
else:
backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT)
logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")
await asyncio.sleep(backoff_delay)
if msg:
logger.warning(msg)
return await wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
logger.warning(msg + 'Giving up.')
raise ex
return wrapper
@@ -101,9 +141,9 @@ def retrier(_func=None, retries=API_RETRY_COUNT):
try:
return f(*args, **kwargs)
except (TemporaryError, RetryableOrderError) as ex:
logger.warning('%s() returned exception: "%s"', f.__name__, ex)
msg = f'{f.__name__}() returned exception: "{ex}". '
if count > 0:
logger.warning('retrying %s() still for %s times', f.__name__, count)
logger.warning(msg + f'Retrying still for {count} times.')
count -= 1
kwargs.update({'count': count})
if isinstance(ex, (DDosProtection, RetryableOrderError)):
@@ -113,7 +153,7 @@ def retrier(_func=None, retries=API_RETRY_COUNT):
time.sleep(backoff_delay)
return wrapper(*args, **kwargs)
else:
logger.warning('Giving up retrying: %s()', f.__name__)
logger.warning(msg + 'Giving up.')
raise ex
return wrapper
# Support both @retrier and @retrier(retries=2) syntax

1408
freqtrade/exchange/exchange.py
View File

File diff suppressed because it is too large Load Diff

67
freqtrade/exchange/ftx.py
View File

@@ -1,9 +1,10 @@
""" FTX exchange subclass """
import logging
from typing import Any, Dict
from typing import Any, Dict, List, Tuple
import ccxt
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange import Exchange
@@ -19,27 +20,30 @@ class Ftx(Exchange):
_ft_has: Dict = {
"stoploss_on_exchange": True,
"ohlcv_candle_limit": 1500,
"ohlcv_volume_currency": "quote",
"mark_ohlcv_price": "index",
"mark_ohlcv_timeframe": "1h",
}
def market_is_tradable(self, market: Dict[str, Any]) -> bool:
"""
Check if the market symbol is tradable by Freqtrade.
Default checks + check if pair is spot pair (no futures trading yet).
"""
parent_check = super().market_is_tradable(market)
_supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
# TradingMode.SPOT always supported and not required in this list
# (TradingMode.MARGIN, MarginMode.CROSS),
# (TradingMode.FUTURES, MarginMode.CROSS)
]
return (parent_check and
market.get('spot', False) is True)
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
Returns True if adjustment is necessary.
"""
return order['type'] == 'stop' and stop_loss > float(order['price'])
return order['type'] == 'stop' and (
side == "sell" and stop_loss > float(order['price']) or
side == "buy" and stop_loss < float(order['price'])
)
@retrier(retries=0)
def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict:
def stoploss(self, pair: str, amount: float, stop_price: float,
order_types: Dict, side: str, leverage: float) -> Dict:
"""
Creates a stoploss order.
depending on order_types.stoploss configuration, uses 'market' or limit order.
@@ -47,7 +51,10 @@ class Ftx(Exchange):
Limit orders are defined by having orderPrice set, otherwise a market order is used.
"""
limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
limit_rate = stop_price * limit_price_pct
if side == "sell":
limit_rate = stop_price * limit_price_pct
else:
limit_rate = stop_price * (2 - limit_price_pct)
ordertype = "stop"
@@ -55,7 +62,7 @@ class Ftx(Exchange):
if self._config['dry_run']:
dry_order = self.create_dry_run_order(
pair, ordertype, "sell", amount, stop_price)
pair, ordertype, side, amount, stop_price, leverage, stop_loss=True)
return dry_order
try:
@@ -63,11 +70,14 @@ class Ftx(Exchange):
if order_types.get('stoploss', 'market') == 'limit':
# set orderPrice to place limit order, otherwise it's a market order
params['orderPrice'] = limit_rate
if self.trading_mode == TradingMode.FUTURES:
params.update({'reduceOnly': True})
params['stopPrice'] = stop_price
amount = self.amount_to_precision(pair, amount)
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
self._lev_prep(pair, leverage, side)
order = self._api.create_order(symbol=pair, type=ordertype, side=side,
amount=amount, params=params)
self._log_exchange_response('create_stoploss_order', order)
logger.info('stoploss order added for %s. '
@@ -75,19 +85,19 @@ class Ftx(Exchange):
return order
except ccxt.InsufficientFunds as e:
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e
except ccxt.InvalidOrder as e:
raise InvalidOrderException(
f'Could not create {ordertype} sell order on market {pair}. '
f'Could not create {ordertype} {side} order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
@@ -105,15 +115,18 @@ class Ftx(Exchange):
if order[0].get('status') == 'closed':
# Trigger order was triggered ...
real_order_id = order[0].get('info', {}).get('orderId')
# OrderId may be None for stoploss-market orders
# But contains "average" in these cases.
if real_order_id:
order1 = self._api.fetch_order(real_order_id, pair)
self._log_exchange_response('fetch_stoploss_order1', order1)
# Fake type to stop - as this was really a stop order.
order1['id_stop'] = order1['id']
order1['id'] = order_id
order1['type'] = 'stop'
order1['status_stop'] = 'triggered'
return order1
order1 = self._api.fetch_order(real_order_id, pair)
self._log_exchange_response('fetch_stoploss_order1', order1)
# Fake type to stop - as this was really a stop order.
order1['id_stop'] = order1['id']
order1['id'] = order_id
order1['type'] = 'stop'
order1['status_stop'] = 'triggered'
return order1
return order[0]
else:
raise InvalidOrderException(f"Could not get stoploss order for id {order_id}")

73
freqtrade/exchange/gateio.py
View File

@@ -1,7 +1,10 @@
""" Gate.io exchange subclass """
import logging
from typing import Dict
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
@@ -20,6 +23,72 @@ class Gateio(Exchange):
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
"ohlcv_volume_currency": "quote",
"stoploss_order_types": {"limit": "limit"},
"stoploss_on_exchange": True,
}
_headers = {'X-Gate-Channel-Id': 'freqtrade'}
_ft_has_futures: Dict = {
"needs_trading_fees": True
}
_supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
# TradingMode.SPOT always supported and not required in this list
# (TradingMode.MARGIN, MarginMode.CROSS),
# (TradingMode.FUTURES, MarginMode.CROSS),
(TradingMode.FUTURES, MarginMode.ISOLATED)
]
def validate_ordertypes(self, order_types: Dict) -> None:
super().validate_ordertypes(order_types)
if self.trading_mode != TradingMode.FUTURES:
if any(v == 'market' for k, v in order_types.items()):
raise OperationalException(
f'Exchange {self.name} does not support market orders.')
def get_trades_for_order(self, order_id: str, pair: str, since: datetime,
params: Optional[Dict] = None) -> List:
trades = super().get_trades_for_order(order_id, pair, since, params)
if self.trading_mode == TradingMode.FUTURES:
# Futures usually don't contain fees in the response.
# As such, futures orders on gateio will not contain a fee, which causes
# a repeated "update fee" cycle and wrong calculations.
# Therefore we patch the response with fees if it's not available.
# An alternative also contianing fees would be
# privateFuturesGetSettleAccountBook({"settle": "usdt"})
pair_fees = self._trading_fees.get(pair, {})
if pair_fees:
for idx, trade in enumerate(trades):
if trade.get('fee', {}).get('cost') is None:
takerOrMaker = trade.get('takerOrMaker', 'taker')
if pair_fees.get(takerOrMaker) is not None:
trades[idx]['fee'] = {
'currency': self.get_pair_quote_currency(pair),
'cost': trade['cost'] * pair_fees[takerOrMaker],
'rate': pair_fees[takerOrMaker],
}
return trades
def fetch_stoploss_order(self, order_id: str, pair: str, params={}) -> Dict:
return self.fetch_order(
order_id=order_id,
pair=pair,
params={'stop': True}
)
def cancel_stoploss_order(self, order_id: str, pair: str, params={}) -> Dict:
return self.cancel_order(
order_id=order_id,
pair=pair,
params={'stop': True}
)
def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
Returns True if adjustment is necessary.
"""
return ((side == "sell" and stop_loss > float(order['stopPrice'])) or
(side == "buy" and stop_loss < float(order['stopPrice'])))

39
freqtrade/exchange/huobi.py Normal file
View File

@@ -0,0 +1,39 @@
""" Huobi exchange subclass """
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Huobi(Exchange):
"""
Huobi exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
"""
_ft_has: Dict = {
"stoploss_on_exchange": True,
"stoploss_order_types": {"limit": "stop-limit"},
"ohlcv_candle_limit": 1000,
"l2_limit_range": [5, 10, 20],
"l2_limit_range_required": False,
}
def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
Returns True if adjustment is necessary.
"""
return order['type'] == 'stop' and stop_loss > float(order['stopPrice'])
def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict:
params = self._params.copy()
params.update({
"stopPrice": stop_price,
"operator": "lte",
})
return params

111
freqtrade/exchange/kraken.py
View File

@@ -1,9 +1,12 @@
""" Kraken exchange subclass """
import logging
from typing import Any, Dict
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
import ccxt
from pandas import DataFrame
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException,
OperationalException, TemporaryError)
from freqtrade.exchange import Exchange
@@ -21,8 +24,15 @@ class Kraken(Exchange):
"ohlcv_candle_limit": 720,
"trades_pagination": "id",
"trades_pagination_arg": "since",
"mark_ohlcv_timeframe": "4h",
}
_supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
# TradingMode.SPOT always supported and not required in this list
# (TradingMode.MARGIN, MarginMode.CROSS),
# (TradingMode.FUTURES, MarginMode.CROSS)
]
def market_is_tradable(self, market: Dict[str, Any]) -> bool:
"""
Check if the market symbol is tradable by Freqtrade.
@@ -33,6 +43,12 @@ class Kraken(Exchange):
return (parent_check and
market.get('darkpool', False) is False)
def get_tickers(self, symbols: List[str] = None, cached: bool = False) -> Dict:
# Only fetch tickers for current stake currency
# Otherwise the request for kraken becomes too large.
symbols = list(self.get_markets(quote_currencies=[self._config['stake_currency']]))
return super().get_tickers(symbols=symbols, cached=cached)
@retrier
def get_balances(self) -> dict:
if self._config['dry_run']:
@@ -67,26 +83,36 @@ class Kraken(Exchange):
except ccxt.BaseError as e:
raise OperationalException(e) from e
def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool:
def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
Returns True if adjustment is necessary.
"""
return (order['type'] in ('stop-loss', 'stop-loss-limit')
and stop_loss > float(order['price']))
return (order['type'] in ('stop-loss', 'stop-loss-limit') and (
(side == "sell" and stop_loss > float(order['price'])) or
(side == "buy" and stop_loss < float(order['price']))
))
@retrier(retries=0)
def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict:
def stoploss(self, pair: str, amount: float, stop_price: float,
order_types: Dict, side: str, leverage: float) -> Dict:
"""
Creates a stoploss market order.
Stoploss market orders is the only stoploss type supported by kraken.
TODO: investigate if this can be combined with generic implementation
(careful, prices are reversed)
"""
params = self._params.copy()
if self.trading_mode == TradingMode.FUTURES:
params.update({'reduceOnly': True})
if order_types.get('stoploss', 'market') == 'limit':
ordertype = "stop-loss-limit"
limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99)
limit_rate = stop_price * limit_price_pct
if side == "sell":
limit_rate = stop_price * limit_price_pct
else:
limit_rate = stop_price * (2 - limit_price_pct)
params['price2'] = self.price_to_precision(pair, limit_rate)
else:
ordertype = "stop-loss"
@@ -95,13 +121,13 @@ class Kraken(Exchange):
if self._config['dry_run']:
dry_order = self.create_dry_run_order(
pair, ordertype, "sell", amount, stop_price)
pair, ordertype, side, amount, stop_price, leverage, stop_loss=True)
return dry_order
try:
amount = self.amount_to_precision(pair, amount)
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
order = self._api.create_order(symbol=pair, type=ordertype, side=side,
amount=amount, price=stop_price, params=params)
self._log_exchange_response('create_stoploss_order', order)
logger.info('stoploss order added for %s. '
@@ -109,18 +135,81 @@ class Kraken(Exchange):
return order
except ccxt.InsufficientFunds as e:
raise InsufficientFundsError(
f'Insufficient funds to create {ordertype} sell order on market {pair}. '
f'Insufficient funds to create {ordertype} {side} order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e
except ccxt.InvalidOrder as e:
raise InvalidOrderException(
f'Could not create {ordertype} sell order on market {pair}. '
f'Could not create {ordertype} {side} order on market {pair}. '
f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. '
f'Message: {e}') from e
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e
f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
def _set_leverage(
self,
leverage: float,
pair: Optional[str] = None,
trading_mode: Optional[TradingMode] = None
):
"""
Kraken set's the leverage as an option in the order object, so we need to
add it to params
"""
return
def _get_params(
self,
ordertype: str,
leverage: float,
reduceOnly: bool,
time_in_force: str = 'gtc'
) -> Dict:
params = super()._get_params(
ordertype=ordertype,
leverage=leverage,
reduceOnly=reduceOnly,
time_in_force=time_in_force,
)
if leverage > 1.0:
params['leverage'] = round(leverage)
return params
def calculate_funding_fees(
self,
df: DataFrame,
amount: float,
is_short: bool,
open_date: datetime,
close_date: Optional[datetime] = None,
time_in_ratio: Optional[float] = None
) -> float:
"""
# ! This method will always error when run by Freqtrade because time_in_ratio is never
# ! passed to _get_funding_fee. For kraken futures to work in dry run and backtesting
# ! functionality must be added that passes the parameter time_in_ratio to
# ! _get_funding_fee when using Kraken
calculates the sum of all funding fees that occurred for a pair during a futures trade
:param df: Dataframe containing combined funding and mark rates
as `open_fund` and `open_mark`.
:param amount: The quantity of the trade
:param is_short: trade direction
:param open_date: The date and time that the trade started
:param close_date: The date and time that the trade ended
:param time_in_ratio: Not used by most exchange classes
"""
if not time_in_ratio:
raise OperationalException(
f"time_in_ratio is required for {self.name}._get_funding_fee")
fees: float = 0
if not df.empty:
df = df[(df['date'] >= open_date) & (df['date'] <= close_date)]
fees = sum(df['open_fund'] * df['open_mark'] * amount * time_in_ratio)
return fees if is_short else -fees

27
freqtrade/exchange/kucoin.py
View File

@@ -1,4 +1,4 @@
""" Kucoin exchange subclass """
"""Kucoin exchange subclass."""
import logging
from typing import Dict
@@ -9,9 +9,9 @@ logger = logging.getLogger(__name__)
class Kucoin(Exchange):
"""
Kucoin exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
"""Kucoin exchange class.
Contains adjustments needed for Freqtrade to work with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
@@ -19,8 +19,27 @@ class Kucoin(Exchange):
"""
_ft_has: Dict = {
"stoploss_on_exchange": True,
"stoploss_order_types": {"limit": "limit", "market": "market"},
"l2_limit_range": [20, 100],
"l2_limit_range_required": False,
"order_time_in_force": ['gtc', 'fok', 'ioc'],
"time_in_force_parameter": "timeInForce",
"ohlcv_candle_limit": 1500,
}
def stoploss_adjust(self, stop_loss: float, order: Dict, side: str) -> bool:
"""
Verify stop_loss against stoploss-order value (limit or price)
Returns True if adjustment is necessary.
"""
return order['info'].get('stop') is not None and stop_loss > float(order['stopPrice'])
def _get_stop_params(self, ordertype: str, stop_price: float) -> Dict:
params = self._params.copy()
params.update({
'stopPrice': stop_price,
'stop': 'loss'
})
return params

88
freqtrade/exchange/okx.py Normal file
View File

@@ -0,0 +1,88 @@
import logging
from typing import Dict, List, Tuple
import ccxt
from freqtrade.enums import MarginMode, TradingMode
from freqtrade.exceptions import DDosProtection, OperationalException, TemporaryError
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import retrier
logger = logging.getLogger(__name__)
class Okx(Exchange):
"""Okx exchange class.
Contains adjustments needed for Freqtrade to work with this exchange.
"""
_ft_has: Dict = {
"ohlcv_candle_limit": 300,
"mark_ohlcv_timeframe": "4h",
"funding_fee_timeframe": "8h",
}
_ft_has_futures: Dict = {
"tickers_have_quoteVolume": False,
}
_supported_trading_mode_margin_pairs: List[Tuple[TradingMode, MarginMode]] = [
# TradingMode.SPOT always supported and not required in this list
# (TradingMode.MARGIN, MarginMode.CROSS),
# (TradingMode.FUTURES, MarginMode.CROSS),
(TradingMode.FUTURES, MarginMode.ISOLATED),
]
def _get_params(
self,
ordertype: str,
leverage: float,
reduceOnly: bool,
time_in_force: str = 'gtc',
) -> Dict:
params = super()._get_params(
ordertype=ordertype,
leverage=leverage,
reduceOnly=reduceOnly,
time_in_force=time_in_force,
)
if self.trading_mode == TradingMode.FUTURES and self.margin_mode:
params['tdMode'] = self.margin_mode.value
return params
@retrier
def _lev_prep(self, pair: str, leverage: float, side: str):
if self.trading_mode != TradingMode.SPOT and self.margin_mode is not None:
try:
# TODO-lev: Test me properly (check mgnMode passed)
self._api.set_leverage(
leverage=leverage,
symbol=pair,
params={
"mgnMode": self.margin_mode.value,
# "posSide": "net"",
})
except ccxt.DDoSProtection as e:
raise DDosProtection(e) from e
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
raise TemporaryError(
f'Could not set leverage due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
def get_max_pair_stake_amount(
self,
pair: str,
price: float,
leverage: float = 1.0
) -> float:
if self.trading_mode == TradingMode.SPOT:
return float('inf') # Not actually inf, but this probably won't matter for SPOT
if pair not in self._leverage_tiers:
return float('inf')
pair_tiers = self._leverage_tiers[pair]
return pair_tiers[-1]['max'] / leverage

988
freqtrade/freqtradebot.py
View File

File diff suppressed because it is too large Load Diff

2
freqtrade/leverage/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
# flake8: noqa: F401
from freqtrade.leverage.interest import interest

43
freqtrade/leverage/interest.py Normal file
View File

@@ -0,0 +1,43 @@
from decimal import Decimal
from math import ceil
from freqtrade.exceptions import OperationalException
one = Decimal(1.0)
four = Decimal(4.0)
twenty_four = Decimal(24.0)
def interest(
exchange_name: str,
borrowed: Decimal,
rate: Decimal,
hours: Decimal
) -> Decimal:
"""
Equation to calculate interest on margin trades
:param exchange_name: The exchanged being trading on
:param borrowed: The amount of currency being borrowed
:param rate: The rate of interest (i.e daily interest rate)
:param hours: The time in hours that the currency has been borrowed for
Raises:
OperationalException: Raised if freqtrade does
not support margin trading for this exchange
Returns: The amount of interest owed (currency matches borrowed)
"""
exchange_name = exchange_name.lower()
if exchange_name == "binance":
return borrowed * rate * ceil(hours) / twenty_four
elif exchange_name == "kraken":
# Rounded based on https://kraken-fees-calculator.github.io/
return borrowed * rate * (one + ceil(hours / four))
elif exchange_name == "ftx":
# As Explained under #Interest rates section in
# https://help.ftx.com/hc/en-us/articles/360053007671-Spot-Margin-Trading-Explainer
return borrowed * rate * ceil(hours) / twenty_four
else:
raise OperationalException(f"Leverage not available on {exchange_name} with freqtrade")

16
freqtrade/loggers.py
View File

@@ -7,11 +7,25 @@ from typing import Any, Dict
from freqtrade.exceptions import OperationalException
class FTBufferingHandler(BufferingHandler):
def flush(self):
"""
Override Flush behaviour - we keep half of the configured capacity
otherwise, we have moments with "empty" logs.
"""
self.acquire()
try:
# Keep half of the records in buffer.
self.buffer = self.buffer[-int(self.capacity / 2):]
finally:
self.release()
logger = logging.getLogger(__name__)
LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
# Initialize bufferhandler - will be used for /log endpoints
bufferHandler = BufferingHandler(1000)
bufferHandler = FTBufferingHandler(1000)
bufferHandler.setFormatter(Formatter(LOGFORMAT))

4
freqtrade/main.py
View File

@@ -9,8 +9,8 @@ from typing import Any, List
# check min. python version
if sys.version_info < (3, 7): # pragma: no cover
sys.exit("Freqtrade requires Python version >= 3.7")
if sys.version_info < (3, 8): # pragma: no cover
sys.exit("Freqtrade requires Python version >= 3.8")
from freqtrade.commands import Arguments
from freqtrade.exceptions import FreqtradeException, OperationalException

58
freqtrade/misc.py
View File

@@ -2,11 +2,13 @@
Various tool function for Freqtrade and scripts
"""
import gzip
import hashlib
import logging
import re
from copy import deepcopy
from datetime import datetime
from pathlib import Path
from typing import Any, Iterator, List
from typing import Any, Iterator, List, Union
from typing.io import IO
from urllib.parse import urlparse
@@ -27,18 +29,23 @@ def decimals_per_coin(coin: str):
return DECIMALS_PER_COIN.get(coin, DECIMAL_PER_COIN_FALLBACK)
def round_coin_value(value: float, coin: str, show_coin_name=True) -> str:
def round_coin_value(
value: float, coin: str, show_coin_name=True, keep_trailing_zeros=False) -> str:
"""
Get price value for this coin
:param value: Value to be printed
:param coin: Which coin are we printing the price / value for
:param show_coin_name: Return string in format: "222.22 USDT" or "222.22"
:param keep_trailing_zeros: Keep trailing zeros "222.200" vs. "222.2"
:return: Formatted / rounded value (with or without coin name)
"""
val = f"{value:.{decimals_per_coin(coin)}f}"
if not keep_trailing_zeros:
val = val.rstrip('0').rstrip('.')
if show_coin_name:
return f"{value:.{decimals_per_coin(coin)}f} {coin}"
else:
return f"{value:.{decimals_per_coin(coin)}f}"
val = f"{val} {coin}"
return val
def shorten_date(_date: str) -> str:
@@ -109,7 +116,7 @@ def file_load_json(file):
def pair_to_filename(pair: str) -> str:
for ch in ['/', '-', ' ', '.', '@', '$', '+', ':']:
for ch in ['/', ' ', '.', '@', '$', '+', ':']:
pair = pair.replace(ch, '_')
return pair
@@ -119,10 +126,10 @@ def format_ms_time(date: int) -> str:
convert MS date to readable format.
: epoch-string in ms
"""
return datetime.fromtimestamp(date/1000.0).strftime('%Y-%m-%dT%H:%M:%S')
return datetime.fromtimestamp(date / 1000.0).strftime('%Y-%m-%dT%H:%M:%S')
def deep_merge_dicts(source, destination):
def deep_merge_dicts(source, destination, allow_null_overrides: bool = True):
"""
Values from Source override destination, destination is returned (and modified!!)
Sample:
@@ -135,8 +142,8 @@ def deep_merge_dicts(source, destination):
if isinstance(value, dict):
# get node or create one
node = destination.setdefault(key, {})
deep_merge_dicts(value, node)
else:
deep_merge_dicts(value, node, allow_null_overrides)
elif value is not None or allow_null_overrides:
destination[key] = value
return destination
@@ -228,3 +235,34 @@ def parse_db_uri_for_logging(uri: str):
return uri
pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
def get_strategy_run_id(strategy) -> str:
"""
Generate unique identification hash for a backtest run. Identical config and strategy file will
always return an identical hash.
:param strategy: strategy object.
:return: hex string id.
"""
digest = hashlib.sha1()
config = deepcopy(strategy.config)
# Options that have no impact on results of individual backtest.
not_important_keys = ('strategy_list', 'original_config', 'telegram', 'api_server')
for k in not_important_keys:
if k in config:
del config[k]
# Explicitly allow NaN values (e.g. max_open_trades).
# as it does not matter for getting the hash.
digest.update(rapidjson.dumps(config, default=str,
number_mode=rapidjson.NM_NAN).encode('utf-8'))
with open(strategy.__file__, 'rb') as fp:
digest.update(fp.read())
return digest.hexdigest().lower()
def get_backtest_metadata_filename(filename: Union[Path, str]) -> Path:
"""Return metadata filename for specified backtest results file."""
filename = Path(filename)
return filename.parent / Path(f'{filename.stem}.meta{filename.suffix}')

903
freqtrade/optimize/backtesting.py
View File

File diff suppressed because it is too large Load Diff

2
freqtrade/optimize/bt_progress.py
View File

@@ -12,7 +12,7 @@ class BTProgress:
def init_step(self, action: BacktestState, max_steps: float):
self._action = action
self._max_steps = max_steps
self._proress = 0
self._progress = 0
def set_new_value(self, new_value: float):
self._progress = new_value

2
freqtrade/optimize/edge_cli.py
View File

@@ -34,7 +34,7 @@ class EdgeCli:
self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
self.strategy = StrategyResolver.load_strategy(self.config)
self.strategy.dp = DataProvider(config, None)
self.strategy.dp = DataProvider(config, self.exchange)
validate_config_consistency(self.config)

16
freqtrade/optimize/hyperopt.py
View File

@@ -45,7 +45,7 @@ progressbar.streams.wrap_stdout()
logger = logging.getLogger(__name__)
INITIAL_POINTS = 5
INITIAL_POINTS = 30
# Keep no more than SKOPT_MODEL_QUEUE_SIZE models
# in the skopt model queue, to optimize memory consumption
@@ -76,6 +76,7 @@ class Hyperopt:
self.config = config
self.backtesting = Backtesting(self.config)
self.pairlist = self.backtesting.pairlists.whitelist
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
@@ -113,10 +114,8 @@ class Hyperopt:
self.position_stacking = self.config.get('position_stacking', False)
if HyperoptTools.has_space(self.config, 'sell'):
# Make sure use_sell_signal is enabled
if 'ask_strategy' not in self.config:
self.config['ask_strategy'] = {}
self.config['ask_strategy']['use_sell_signal'] = True
# Make sure use_exit_signal is enabled
self.config['use_exit_signal'] = True
self.print_all = self.config.get('print_all', False)
self.hyperopt_table_header = 0
@@ -258,6 +257,7 @@ class Hyperopt:
if HyperoptTools.has_space(self.config, 'trailing'):
logger.debug("Hyperopt has 'trailing' space")
self.trailing_space = self.custom_hyperopt.trailing_space()
self.dimensions = (self.buy_space + self.sell_space + self.protection_space
+ self.roi_space + self.stoploss_space + self.trailing_space)
@@ -331,7 +331,7 @@ class Hyperopt:
params_details = self._get_params_details(params_dict)
strat_stats = generate_strategy_stats(
processed, self.backtesting.strategy.get_strategy_name(),
self.pairlist, self.backtesting.strategy.get_strategy_name(),
backtesting_results, min_date, max_date, market_change=0
)
results_explanation = HyperoptTools.format_results_explanation_string(
@@ -365,7 +365,7 @@ class Hyperopt:
}
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
estimator = self.custom_hyperopt.generate_estimator()
estimator = self.custom_hyperopt.generate_estimator(dimensions=dimensions)
acq_optimizer = "sampling"
if isinstance(estimator, str):
@@ -394,6 +394,7 @@ class Hyperopt:
def prepare_hyperopt_data(self) -> None:
data, timerange = self.backtesting.load_bt_data()
self.backtesting.load_bt_data_detail()
logger.info("Dataload complete. Calculating indicators")
preprocessed = self.backtesting.strategy.advise_all_indicators(data)
@@ -421,6 +422,7 @@ class Hyperopt:
self.backtesting.exchange.close()
self.backtesting.exchange._api = None # type: ignore
self.backtesting.exchange._api_async = None # type: ignore
self.backtesting.exchange.loop = None # type: ignore
# self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore

31
freqtrade/optimize/hyperopt_auto.py
View File

@@ -3,6 +3,7 @@ HyperOptAuto class.
This module implements a convenience auto-hyperopt class, which can be used together with strategies
that implement IHyperStrategy interface.
"""
import logging
from contextlib import suppress
from typing import Callable, Dict, List
@@ -15,12 +16,19 @@ with suppress(ImportError):
from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
def _format_exception_message(space: str) -> str:
raise OperationalException(
f"The '{space}' space is included into the hyperoptimization "
f"but no parameter for this space was not found in your Strategy. "
f"Please make sure to have parameters for this space enabled for optimization "
f"or remove the '{space}' space from hyperoptimization.")
logger = logging.getLogger(__name__)
def _format_exception_message(space: str, ignore_missing_space: bool) -> None:
msg = (f"The '{space}' space is included into the hyperoptimization "
f"but no parameter for this space was not found in your Strategy. "
)
if ignore_missing_space:
logger.warning(msg + "This space will be ignored.")
else:
raise OperationalException(
msg + f"Please make sure to have parameters for this space enabled for optimization "
f"or remove the '{space}' space from hyperoptimization.")
class HyperOptAuto(IHyperOpt):
@@ -48,13 +56,16 @@ class HyperOptAuto(IHyperOpt):
if attr.optimize:
yield attr.get_space(attr_name)
def _get_indicator_space(self, category):
def _get_indicator_space(self, category) -> List:
# TODO: is this necessary, or can we call "generate_space" directly?
indicator_space = list(self._generate_indicator_space(category))
if len(indicator_space) > 0:
return indicator_space
else:
_format_exception_message(category)
_format_exception_message(
category,
self.config.get("hyperopt_ignore_missing_space", False))
return []
def buy_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy')
@@ -80,5 +91,5 @@ class HyperOptAuto(IHyperOpt):
def trailing_space(self) -> List['Dimension']:
return self._get_func('trailing_space')()
def generate_estimator(self) -> EstimatorType:
return self._get_func('generate_estimator')()
def generate_estimator(self, dimensions: List['Dimension'], **kwargs) -> EstimatorType:
return self._get_func('generate_estimator')(dimensions=dimensions, **kwargs)

9
freqtrade/optimize/hyperopt_interface.py
View File

@@ -29,18 +29,16 @@ class IHyperOpt(ABC):
Class attributes you can use:
timeframe -> int: value of the timeframe to use for the strategy
"""
ticker_interval: str # DEPRECATED
timeframe: str
strategy: IStrategy
def __init__(self, config: dict) -> None:
self.config = config
# Assign ticker_interval to be used in hyperopt
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
# Assign timeframe to be used in hyperopt
IHyperOpt.timeframe = str(config['timeframe'])
def generate_estimator(self) -> EstimatorType:
def generate_estimator(self, dimensions: List[Dimension], **kwargs) -> EstimatorType:
"""
Return base_estimator.
Can be any of "GP", "RF", "ET", "GBRT" or an instance of a class
@@ -192,7 +190,7 @@ class IHyperOpt(ABC):
Categorical([True, False], name='trailing_only_offset_is_reached'),
]
# This is needed for proper unpickling the class attribute ticker_interval
# This is needed for proper unpickling the class attribute timeframe
# which is set to the actual value by the resolver.
# Why do I still need such shamanic mantras in modern python?
def __getstate__(self):
@@ -202,5 +200,4 @@ class IHyperOpt(ABC):
def __setstate__(self, state):
self.__dict__.update(state)
IHyperOpt.ticker_interval = state['timeframe']
IHyperOpt.timeframe = state['timeframe']

63
freqtrade/optimize/hyperopt_loss_calmar.py Normal file
View File

@@ -0,0 +1,63 @@
"""
CalmarHyperOptLoss
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
from datetime import datetime
from math import sqrt as msqrt
from typing import Any, Dict
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss
class CalmarHyperOptLoss(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation uses the Calmar Ratio calculation.
"""
@staticmethod
def hyperopt_loss_function(
results: DataFrame,
trade_count: int,
min_date: datetime,
max_date: datetime,
config: Dict,
processed: Dict[str, DataFrame],
backtest_stats: Dict[str, Any],
*args,
**kwargs
) -> float:
"""
Objective function, returns smaller number for more optimal results.
Uses Calmar Ratio calculation.
"""
total_profit = backtest_stats["profit_total"]
days_period = (max_date - min_date).days
# adding slippage of 0.1% per trade
total_profit = total_profit - 0.0005
expected_returns_mean = total_profit.sum() / days_period * 100
# calculate max drawdown
try:
_, _, _, _, _, max_drawdown = calculate_max_drawdown(
results, value_col="profit_abs"
)
except ValueError:
max_drawdown = 0
if max_drawdown != 0:
calmar_ratio = expected_returns_mean / max_drawdown * msqrt(365)
else:
# Define high (negative) calmar ratio to be clear that this is NOT optimal.
calmar_ratio = -20.0
# print(expected_returns_mean, max_drawdown, calmar_ratio)
return -calmar_ratio

41
freqtrade/optimize/hyperopt_loss_max_drawdown.py Normal file
View File

@@ -0,0 +1,41 @@
"""
MaxDrawDownHyperOptLoss
This module defines the alternative HyperOptLoss class which can be used for
Hyperoptimization.
"""
from datetime import datetime
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss
class MaxDrawDownHyperOptLoss(IHyperOptLoss):
"""
Defines the loss function for hyperopt.
This implementation optimizes for max draw down and profit
Less max drawdown more profit -> Lower return value
"""
@staticmethod
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
*args, **kwargs) -> float:
"""
Objective function.
Uses profit ratio weighted max_drawdown when drawdown is available.
Otherwise directly optimizes profit ratio.
"""
total_profit = results['profit_abs'].sum()
try:
max_drawdown = calculate_max_drawdown(results, value_col='profit_abs')
except ValueError:
# No losing trade, therefore no drawdown.
return -total_profit
return -total_profit / max_drawdown[0]

30
freqtrade/optimize/hyperopt_loss_profit_drawdown.py Normal file
View File

@@ -0,0 +1,30 @@
"""
ProfitDrawDownHyperOptLoss
This module defines the alternative HyperOptLoss class based on Profit &
Drawdown objective which can be used for Hyperoptimization.
Possible to change `DRAWDOWN_MULT` to penalize drawdown objective for
individual needs.
"""
from pandas import DataFrame
from freqtrade.data.btanalysis import calculate_max_drawdown
from freqtrade.optimize.hyperopt import IHyperOptLoss
# higher numbers penalize drawdowns more severely
DRAWDOWN_MULT = 0.075
class ProfitDrawDownHyperOptLoss(IHyperOptLoss):
@staticmethod
def hyperopt_loss_function(results: DataFrame, trade_count: int, *args, **kwargs) -> float:
total_profit = results["profit_abs"].sum()
try:
max_drawdown_abs = calculate_max_drawdown(results, value_col="profit_abs")[5]
except ValueError:
max_drawdown_abs = 0
return -1 * (total_profit * (1 - max_drawdown_abs * DRAWDOWN_MULT))

111
freqtrade/optimize/hyperopt_tools.py
View File

@@ -1,4 +1,3 @@
import io
import logging
from copy import deepcopy
@@ -64,10 +63,11 @@ class HyperoptTools():
'export_time': datetime.now(timezone.utc),
}
logger.info(f"Dumping parameters to {filename}")
rapidjson.dump(final_params, filename.open('w'), indent=2,
default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
)
with filename.open('w') as f:
rapidjson.dump(final_params, f, indent=2,
default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
)
@staticmethod
def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict):
@@ -137,6 +137,7 @@ class HyperoptTools():
}
if not HyperoptTools._test_hyperopt_results_exist(results_file):
# No file found.
logger.warning(f"Hyperopt file {results_file} not found.")
return [], 0
epochs = []
@@ -284,10 +285,10 @@ class HyperoptTools():
return (f"{results_metrics['total_trades']:6d} trades. "
f"{results_metrics['wins']}/{results_metrics['draws']}"
f"/{results_metrics['losses']} Wins/Draws/Losses. "
f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. "
f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. "
f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} "
f"({results_metrics['profit_total'] * 100: 7.2f}%). "
f"Avg profit {results_metrics['profit_mean']:7.2%}. "
f"Median profit {results_metrics['profit_median']:7.2%}. "
f"Total profit {results_metrics['profit_total_abs']:11.8f} {stake_currency} "
f"({results_metrics['profit_total']:8.2%}). "
f"Avg duration {results_metrics['holding_avg']} min."
)
@@ -299,8 +300,7 @@ class HyperoptTools():
f"Objective: {results['loss']:.5f}")
@staticmethod
def prepare_trials_columns(trials: pd.DataFrame, legacy_mode: bool,
has_drawdown: bool) -> pd.DataFrame:
def prepare_trials_columns(trials: pd.DataFrame, has_drawdown: bool) -> pd.DataFrame:
trials['Best'] = ''
if 'results_metrics.winsdrawslosses' not in trials.columns:
@@ -309,33 +309,26 @@ class HyperoptTools():
if not has_drawdown:
# Ensure compatibility with older versions of hyperopt results
trials['results_metrics.max_drawdown_abs'] = None
trials['results_metrics.max_drawdown'] = None
trials['results_metrics.max_drawdown_account'] = None
if not legacy_mode:
# New mode, using backtest result for metrics
trials['results_metrics.winsdrawslosses'] = trials.apply(
lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
f"{x['results_metrics.losses']:>4}", axis=1)
trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.winsdrawslosses',
'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
'results_metrics.profit_total', 'results_metrics.holding_avg',
'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs',
'loss', 'is_initial_point', 'is_best']]
# New mode, using backtest result for metrics
trials['results_metrics.winsdrawslosses'] = trials.apply(
lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
f"{x['results_metrics.losses']:>4}", axis=1)
else:
# Legacy mode
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.winsdrawslosses', 'results_metrics.avg_profit',
'results_metrics.total_profit', 'results_metrics.profit',
'results_metrics.duration', 'results_metrics.max_drawdown',
'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point',
'is_best']]
trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.winsdrawslosses',
'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
'results_metrics.profit_total', 'results_metrics.holding_avg',
'results_metrics.max_drawdown',
'results_metrics.max_drawdown_account', 'results_metrics.max_drawdown_abs',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'Max Drawdown',
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best']
trials.columns = [
'Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'max_drawdown', 'max_drawdown_account',
'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'
]
return trials
@@ -351,10 +344,9 @@ class HyperoptTools():
tabulate.PRESERVE_WHITESPACE = True
trials = json_normalize(results, max_level=1)
legacy_mode = 'results_metrics.total_trades' not in trials
has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns
has_account_drawdown = 'results_metrics.max_drawdown_account' in trials.columns
trials = HyperoptTools.prepare_trials_columns(trials, legacy_mode, has_drawdown)
trials = HyperoptTools.prepare_trials_columns(trials, has_account_drawdown)
trials['is_profit'] = False
trials.loc[trials['is_initial_point'], 'Best'] = '* '
@@ -362,12 +354,12 @@ class HyperoptTools():
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
trials['Trades'] = trials['Trades'].astype(str)
perc_multi = 1 if legacy_mode else 100
# perc_multi = 1 if legacy_mode else 100
trials['Epoch'] = trials['Epoch'].apply(
lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
)
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: f'{x * perc_multi:,.2f}%'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
lambda x: f'{x:,.2%}'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
)
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}"
@@ -379,26 +371,27 @@ class HyperoptTools():
stake_currency = config['stake_currency']
if has_drawdown:
trials['Max Drawdown'] = trials.apply(
lambda x: '{} {}'.format(
round_coin_value(x['max_drawdown_abs'], stake_currency),
'({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ')
).rjust(25 + len(stake_currency))
if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)),
axis=1
)
else:
trials = trials.drop(columns=['Max Drawdown'])
trials[f"Max Drawdown{' (Acct)' if has_account_drawdown else ''}"] = trials.apply(
lambda x: "{} {}".format(
round_coin_value(x['max_drawdown_abs'], stake_currency, keep_trailing_zeros=True),
(f"({x['max_drawdown_account']:,.2%})"
if has_account_drawdown
else f"({x['max_drawdown']:,.2%})"
).rjust(10, ' ')
).rjust(25 + len(stake_currency))
if x['max_drawdown'] != 0.0 or x['max_drawdown_account'] != 0.0
else '--'.rjust(25 + len(stake_currency)),
axis=1
)
trials = trials.drop(columns=['max_drawdown_abs'])
trials = trials.drop(columns=['max_drawdown_abs', 'max_drawdown', 'max_drawdown_account'])
trials['Profit'] = trials.apply(
lambda x: '{} {}'.format(
round_coin_value(x['Total profit'], stake_currency),
'({:,.2f}%)'.format(x['Profit'] * perc_multi).rjust(10, ' ')
).rjust(25+len(stake_currency))
if x['Total profit'] != 0.0 else '--'.rjust(25+len(stake_currency)),
round_coin_value(x['Total profit'], stake_currency, keep_trailing_zeros=True),
f"({x['Profit']:,.2%})".rjust(10, ' ')
).rjust(25 + len(stake_currency))
if x['Total profit'] != 0.0 else '--'.rjust(25 + len(stake_currency)),
axis=1
)
trials = trials.drop(columns=['Total profit'])
@@ -406,11 +399,11 @@ class HyperoptTools():
if print_colorized:
for i in range(len(trials)):
if trials.loc[i]['is_profit']:
for j in range(len(trials.loc[i])-3):
for j in range(len(trials.loc[i]) - 3):
trials.iat[i, j] = "{}{}{}".format(Fore.GREEN,
str(trials.loc[i][j]), Fore.RESET)
if trials.loc[i]['is_best'] and highlight_best:
for j in range(len(trials.loc[i])-3):
for j in range(len(trials.loc[i]) - 3):
trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT,
str(trials.loc[i][j]), Style.RESET_ALL)
@@ -466,7 +459,7 @@ class HyperoptTools():
'loss', 'is_initial_point', 'is_best']
perc_multi = 100
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
param_metrics = [("params_dict." + param) for param in results[0]['params_dict'].keys()]
trials = trials[base_metrics + param_metrics]
base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Median profit', 'Total profit',

365
freqtrade/optimize/optimize_reports.py
View File

@@ -1,16 +1,18 @@
import logging
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Union
from numpy import int64
from pandas import DataFrame
from pandas import DataFrame, to_datetime
from tabulate import tabulate
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change,
calculate_max_drawdown)
from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value
from freqtrade.misc import (decimals_per_coin, file_dump_json, get_backtest_metadata_filename,
round_coin_value)
logger = logging.getLogger(__name__)
@@ -32,6 +34,11 @@ def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> N
recordfilename.parent,
f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
).with_suffix(recordfilename.suffix)
# Store metadata separately.
file_dump_json(get_backtest_metadata_filename(filename), stats['metadata'])
del stats['metadata']
file_dump_json(filename, stats)
latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN)
@@ -46,11 +53,11 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
'.2f', 'd', 's', 's']
def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
def _get_line_header(first_column: str, stake_currency: str, direction: str = 'Buys') -> List[str]:
"""
Generate header lines (goes in line with _generate_result_line())
"""
return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %',
return [first_column, direction, 'Avg Profit %', 'Cum Profit %',
f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration',
'Win Draw Loss Win%']
@@ -98,11 +105,11 @@ def _generate_result_line(result: DataFrame, starting_balance: int, first_column
}
def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_balance: int,
def generate_pair_metrics(pairlist: List[str], stake_currency: str, starting_balance: int,
results: DataFrame, skip_nan: bool = False) -> List[Dict]:
"""
Generates and returns a list for the given backtest data and the results dataframe
:param data: Dict of <pair: dataframe> containing data that was used during backtesting.
:param pairlist: Pairlist used
:param stake_currency: stake-currency - used to correctly name headers
:param starting_balance: Starting balance
:param results: Dataframe containing the backtest results
@@ -112,7 +119,7 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_b
tabular_data = []
for pair in data:
for pair in pairlist:
result = results[results['pair'] == pair]
if skip_nan and result['profit_abs'].isnull().all():
continue
@@ -127,7 +134,39 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_b
return tabular_data
def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List[Dict]:
def generate_tag_metrics(tag_type: str,
starting_balance: int,
results: DataFrame,
skip_nan: bool = False) -> List[Dict]:
"""
Generates and returns a list of metrics for the given tag trades and the results dataframe
:param starting_balance: Starting balance
:param results: Dataframe containing the backtest results
:param skip_nan: Print "left open" open trades
:return: List of Dicts containing the metrics per pair
"""
tabular_data = []
if tag_type in results.columns:
for tag, count in results[tag_type].value_counts().iteritems():
result = results[results[tag_type] == tag]
if skip_nan and result['profit_abs'].isnull().all():
continue
tabular_data.append(_generate_result_line(result, starting_balance, tag))
# Sort by total profit %:
tabular_data = sorted(tabular_data, key=lambda k: k['profit_total_abs'], reverse=True)
# Append Total
tabular_data.append(_generate_result_line(results, starting_balance, 'TOTAL'))
return tabular_data
else:
return []
def generate_exit_reason_stats(max_open_trades: int, results: DataFrame) -> List[Dict]:
"""
Generate small table outlining Backtest results
:param max_open_trades: Max_open_trades parameter
@@ -136,8 +175,8 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
"""
tabular_data = []
for reason, count in results['sell_reason'].value_counts().iteritems():
result = results.loc[results['sell_reason'] == reason]
for reason, count in results['exit_reason'].value_counts().iteritems():
result = results.loc[results['exit_reason'] == reason]
profit_mean = result['profit_ratio'].mean()
profit_sum = result['profit_ratio'].sum()
@@ -145,7 +184,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
tabular_data.append(
{
'sell_reason': reason,
'exit_reason': reason,
'trades': count,
'wins': len(result[result['profit_abs'] > 0]),
'draws': len(result[result['profit_abs'] == 0]),
@@ -162,34 +201,25 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
return tabular_data
def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
def generate_strategy_comparison(bt_stats: Dict) -> List[Dict]:
"""
Generate summary per strategy
:param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
:param bt_stats: Dict of <Strategyname: DataFrame> containing results for all strategies
:return: List of Dicts containing the metrics per Strategy
"""
tabular_data = []
for strategy, results in all_results.items():
tabular_data.append(_generate_result_line(
results['results'], results['config']['dry_run_wallet'], strategy)
)
try:
max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_ratio')
max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_abs')
except ValueError:
max_drawdown_per = 0
max_drawdown_abs = 0
tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2)
tabular_data[-1]['max_drawdown_abs'] = \
round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False)
for strategy, result in bt_stats.items():
tabular_data.append(deepcopy(result['results_per_pair'][-1]))
# Update "key" to strategy (results_per_pair has it as "Total").
tabular_data[-1]['key'] = strategy
tabular_data[-1]['max_drawdown_account'] = result['max_drawdown_account']
tabular_data[-1]['max_drawdown_abs'] = round_coin_value(
result['max_drawdown_abs'], result['stake_currency'], False)
return tabular_data
def generate_edge_table(results: dict) -> str:
floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', 'd', 'd')
tabular_data = []
headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio',
@@ -214,6 +244,41 @@ def generate_edge_table(results: dict) -> str:
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def _get_resample_from_period(period: str) -> str:
if period == 'day':
return '1d'
if period == 'week':
return '1w'
if period == 'month':
return '1M'
raise ValueError(f"Period {period} is not supported.")
def generate_periodic_breakdown_stats(trade_list: List, period: str) -> List[Dict[str, Any]]:
results = DataFrame.from_records(trade_list)
if len(results) == 0:
return []
results['close_date'] = to_datetime(results['close_date'], utc=True)
resample_period = _get_resample_from_period(period)
resampled = results.resample(resample_period, on='close_date')
stats = []
for name, day in resampled:
profit_abs = day['profit_abs'].sum().round(10)
wins = sum(day['profit_abs'] > 0)
draws = sum(day['profit_abs'] == 0)
loses = sum(day['profit_abs'] < 0)
stats.append(
{
'date': name.strftime('%d/%m/%Y'),
'profit_abs': profit_abs,
'wins': wins,
'draws': draws,
'loses': loses
}
)
return stats
def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
""" Generate overall trade statistics """
if len(results) == 0:
@@ -286,14 +351,14 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
}
def generate_strategy_stats(btdata: Dict[str, DataFrame],
def generate_strategy_stats(pairlist: List[str],
strategy: str,
content: Dict[str, Any],
min_date: datetime, max_date: datetime,
market_change: float
) -> Dict[str, Any]:
"""
:param btdata: Backtest data
:param pairlist: List of pairs to backtest
:param strategy: Strategy name
:param content: Backtest result data in the format:
{'results: results, 'config: config}}.
@@ -306,17 +371,21 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
if not isinstance(results, DataFrame):
return {}
config = content['config']
max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
starting_balance = config['dry_run_wallet']
max_open_trades = min(config['max_open_trades'], len(pairlist))
start_balance = config['dry_run_wallet']
stake_currency = config['stake_currency']
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
starting_balance=starting_balance,
pair_results = generate_pair_metrics(pairlist, stake_currency=stake_currency,
starting_balance=start_balance,
results=results, skip_nan=False)
sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
enter_tag_results = generate_tag_metrics("enter_tag", starting_balance=start_balance,
results=results, skip_nan=False)
exit_reason_stats = generate_exit_reason_stats(max_open_trades=max_open_trades,
results=results)
left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
starting_balance=starting_balance,
left_open_results = generate_pair_metrics(pairlist, stake_currency=stake_currency,
starting_balance=start_balance,
results=results.loc[results['is_open']],
skip_nan=True)
daily_stats = generate_daily_stats(results)
@@ -329,22 +398,31 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
results['open_timestamp'] = results['open_date'].view(int64) // 1e6
results['close_timestamp'] = results['close_date'].view(int64) // 1e6
backtest_days = (max_date - min_date).days
backtest_days = (max_date - min_date).days or 1
strat_stats = {
'trades': results.to_dict(orient='records'),
'locks': [lock.to_json() for lock in content['locks']],
'best_pair': best_pair,
'worst_pair': worst_pair,
'results_per_pair': pair_results,
'sell_reason_summary': sell_reason_stats,
'results_per_enter_tag': enter_tag_results,
'exit_reason_summary': exit_reason_stats,
'left_open_trades': left_open_results,
# 'days_breakdown_stats': days_breakdown_stats,
'total_trades': len(results),
'trade_count_long': len(results.loc[~results['is_short']]),
'trade_count_short': len(results.loc[results['is_short']]),
'total_volume': float(results['stake_amount'].sum()),
'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
'profit_total': results['profit_abs'].sum() / starting_balance,
'profit_total': results['profit_abs'].sum() / start_balance,
'profit_total_long': results.loc[~results['is_short'], 'profit_abs'].sum() / start_balance,
'profit_total_short': results.loc[results['is_short'], 'profit_abs'].sum() / start_balance,
'profit_total_abs': results['profit_abs'].sum(),
'profit_total_long_abs': results.loc[~results['is_short'], 'profit_abs'].sum(),
'profit_total_short_abs': results.loc[results['is_short'], 'profit_abs'].sum(),
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
@@ -354,16 +432,18 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'backtest_run_start_ts': content['backtest_start_time'],
'backtest_run_end_ts': content['backtest_end_time'],
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
'trades_per_day': round(len(results) / backtest_days, 2),
'market_change': market_change,
'pairlist': list(btdata.keys()),
'pairlist': pairlist,
'stake_amount': config['stake_amount'],
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'starting_balance': starting_balance,
'dry_run_wallet': starting_balance,
'starting_balance': start_balance,
'dry_run_wallet': start_balance,
'final_balance': content['final_balance'],
'rejected_signals': content['rejected_signals'],
'timedout_entry_orders': content['timedout_entry_orders'],
'timedout_exit_orders': content['timedout_exit_orders'],
'max_open_trades': max_open_trades,
'max_open_trades_setting': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
@@ -380,21 +460,23 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
'use_custom_stoploss': config.get('use_custom_stoploss', False),
'minimal_roi': config['minimal_roi'],
'use_sell_signal': config['use_sell_signal'],
'sell_profit_only': config['sell_profit_only'],
'sell_profit_offset': config['sell_profit_offset'],
'ignore_roi_if_buy_signal': config['ignore_roi_if_buy_signal'],
'use_exit_signal': config['use_exit_signal'],
'exit_profit_only': config['exit_profit_only'],
'exit_profit_offset': config['exit_profit_offset'],
'ignore_roi_if_entry_signal': config['ignore_roi_if_entry_signal'],
**daily_stats,
**trade_stats
}
try:
max_drawdown, _, _, _, _ = calculate_max_drawdown(
max_drawdown_legacy, _, _, _, _, _ = calculate_max_drawdown(
results, value_col='profit_ratio')
drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
results, value_col='profit_abs')
(drawdown_abs, drawdown_start, drawdown_end, high_val, low_val,
max_drawdown) = calculate_max_drawdown(
results, value_col='profit_abs', starting_balance=start_balance)
strat_stats.update({
'max_drawdown': max_drawdown,
'max_drawdown': max_drawdown_legacy, # Deprecated - do not use
'max_drawdown_account': max_drawdown,
'max_drawdown_abs': drawdown_abs,
'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
@@ -405,7 +487,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
'max_drawdown_high': high_val,
})
csum_min, csum_max = calculate_csum(results, starting_balance)
csum_min, csum_max = calculate_csum(results, start_balance)
strat_stats.update({
'csum_min': csum_min,
'csum_max': csum_max
@@ -414,6 +496,7 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
except ValueError:
strat_stats.update({
'max_drawdown': 0.0,
'max_drawdown_account': 0.0,
'max_drawdown_abs': 0.0,
'max_drawdown_low': 0.0,
'max_drawdown_high': 0.0,
@@ -440,16 +523,26 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
:param max_date: Backtest end date
:return: Dictionary containing results per strategy and a strategy summary.
"""
result: Dict[str, Any] = {'strategy': {}}
result: Dict[str, Any] = {
'metadata': {},
'strategy': {},
'strategy_comparison': [],
}
market_change = calculate_market_change(btdata, 'close')
metadata = {}
pairlist = list(btdata.keys())
for strategy, content in all_results.items():
strat_stats = generate_strategy_stats(btdata, strategy, content,
strat_stats = generate_strategy_stats(pairlist, strategy, content,
min_date, max_date, market_change=market_change)
metadata[strategy] = {
'run_id': content['run_id'],
'backtest_start_time': content['backtest_start_time'],
}
result['strategy'][strategy] = strat_stats
strategy_results = generate_strategy_comparison(all_results=all_results)
strategy_results = generate_strategy_comparison(bt_stats=result['strategy'])
result['metadata'] = metadata
result['strategy_comparison'] = strategy_results
return result
@@ -479,16 +572,16 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_currency: str) -> str:
def text_table_exit_reason(exit_reason_stats: List[Dict[str, Any]], stake_currency: str) -> str:
"""
Generate small table outlining Backtest results
:param sell_reason_stats: Sell reason metrics
:param sell_reason_stats: Exit reason metrics
:param stake_currency: Stakecurrency used
:return: pretty printed table with tabulate as string
"""
headers = [
'Sell Reason',
'Sells',
'Exit Reason',
'Exits',
'Win Draws Loss Win%',
'Avg Profit %',
'Cum Profit %',
@@ -497,12 +590,65 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
]
output = [[
t['sell_reason'], t['trades'],
t.get('exit_reason', t.get('sell_reason')), t['trades'],
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
t['profit_mean_pct'], t['profit_sum_pct'],
round_coin_value(t['profit_total_abs'], stake_currency, False),
t['profit_total_pct'],
] for t in sell_reason_stats]
] for t in exit_reason_stats]
return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
def text_table_tags(tag_type: str, tag_results: List[Dict[str, Any]], stake_currency: str) -> str:
"""
Generates and returns a text table for the given backtest data and the results dataframe
:param pair_results: List of Dictionaries - one entry per pair + final TOTAL row
:param stake_currency: stake-currency - used to correctly name headers
:return: pretty printed table with tabulate as string
"""
if(tag_type == "enter_tag"):
headers = _get_line_header("TAG", stake_currency)
else:
headers = _get_line_header("TAG", stake_currency, 'Sells')
floatfmt = _get_line_floatfmt(stake_currency)
output = [
[
t['key'] if t['key'] is not None and len(
t['key']) > 0 else "OTHER",
t['trades'],
t['profit_mean_pct'],
t['profit_sum_pct'],
t['profit_total_abs'],
t['profit_total_pct'],
t['duration_avg'],
_generate_wins_draws_losses(
t['wins'],
t['draws'],
t['losses'])] for t in tag_results]
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(output, headers=headers,
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
def text_table_periodic_breakdown(days_breakdown_stats: List[Dict[str, Any]],
stake_currency: str, period: str) -> str:
"""
Generate small table with Backtest results by days
:param days_breakdown_stats: Days breakdown metrics
:param stake_currency: Stakecurrency used
:return: pretty printed table with tabulate as string
"""
headers = [
period.capitalize(),
f'Tot Profit {stake_currency}',
'Wins',
'Draws',
'Losses',
]
output = [[
d['date'], round_coin_value(d['profit_abs'], stake_currency, False),
d['wins'], d['draws'], d['loses'],
] for d in days_breakdown_stats]
return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right")
@@ -520,7 +666,12 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
headers.append('Drawdown')
# Align drawdown string on the center two space separator.
drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
if 'max_drawdown_account' in strategy_results[0]:
drawdown = [f'{t["max_drawdown_account"] * 100:.2f}' for t in strategy_results]
else:
# Support for prior backtest results
drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results])
dd_pad_per = max([len(dd) for dd in drawdown])
drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%'
@@ -541,6 +692,19 @@ def text_table_add_metrics(strat_results: Dict) -> str:
best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio'])
worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio'])
short_metrics = [
('', ''), # Empty line to improve readability
('Long / Short',
f"{strat_results.get('trade_count_long', 'total_trades')} / "
f"{strat_results.get('trade_count_short', 0)}"),
('Total profit Long %', f"{strat_results['profit_total_long']:.2%}"),
('Total profit Short %', f"{strat_results['profit_total_short']:.2%}"),
('Absolute profit Long', round_coin_value(strat_results['profit_total_long_abs'],
strat_results['stake_currency'])),
('Absolute profit Short', round_coin_value(strat_results['profit_total_short_abs'],
strat_results['stake_currency'])),
] if strat_results.get('trade_count_short', 0) > 0 else []
# Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
# command stores these results and newer version of freqtrade must be able to handle old
# results with missing new fields.
@@ -551,25 +715,30 @@ def text_table_add_metrics(strat_results: Dict) -> str:
('', ''), # Empty line to improve readability
('Total/Daily Avg Trades',
f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"),
('Starting balance', round_coin_value(strat_results['starting_balance'],
strat_results['stake_currency'])),
('Final balance', round_coin_value(strat_results['final_balance'],
strat_results['stake_currency'])),
('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
strat_results['stake_currency'])),
('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"),
('Total profit %', f"{strat_results['profit_total']:.2%}"),
('Trades per day', strat_results['trades_per_day']),
('Avg. daily profit %',
f"{(strat_results['profit_total'] / strat_results['backtest_days']):.2%}"),
('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'],
strat_results['stake_currency'])),
('Total trade volume', round_coin_value(strat_results['total_volume'],
strat_results['stake_currency'])),
*short_metrics,
('', ''), # Empty line to improve readability
('Best Pair', f"{strat_results['best_pair']['key']} "
f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"),
f"{strat_results['best_pair']['profit_sum']:.2%}"),
('Worst Pair', f"{strat_results['worst_pair']['key']} "
f"{round(strat_results['worst_pair']['profit_sum_pct'], 2)}%"),
('Best trade', f"{best_trade['pair']} {round(best_trade['profit_ratio'] * 100, 2)}%"),
f"{strat_results['worst_pair']['profit_sum']:.2%}"),
('Best trade', f"{best_trade['pair']} {best_trade['profit_ratio']:.2%}"),
('Worst trade', f"{worst_trade['pair']} "
f"{round(worst_trade['profit_ratio'] * 100, 2)}%"),
f"{worst_trade['profit_ratio']:.2%}"),
('Best day', round_coin_value(strat_results['backtest_best_day_abs'],
strat_results['stake_currency'])),
@@ -579,7 +748,10 @@ def text_table_add_metrics(strat_results: Dict) -> str:
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
('Rejected Entry signals', strat_results.get('rejected_signals', 'N/A')),
('Entry/Exit Timeouts',
f"{strat_results.get('timedout_entry_orders', 'N/A')} / "
f"{strat_results.get('timedout_exit_orders', 'N/A')}"),
('', ''), # Empty line to improve readability
('Min balance', round_coin_value(strat_results['csum_min'],
@@ -587,7 +759,10 @@ def text_table_add_metrics(strat_results: Dict) -> str:
('Max balance', round_coin_value(strat_results['csum_max'],
strat_results['stake_currency'])),
('Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"),
# Compatibility to show old hyperopt results
('Drawdown (Account)', f"{strat_results['max_drawdown_account']:.2%}")
if 'max_drawdown_account' in strat_results else (
'Drawdown', f"{strat_results['max_drawdown']:.2%}"),
('Drawdown', round_coin_value(strat_results['max_drawdown_abs'],
strat_results['stake_currency'])),
('Drawdown high', round_coin_value(strat_results['max_drawdown_high'],
@@ -596,7 +771,7 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])),
('Drawdown Start', strat_results['drawdown_start']),
('Drawdown End', strat_results['drawdown_end']),
('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
('Market change', f"{strat_results['market_change']:.2%}"),
]
return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl")
@@ -614,7 +789,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
return message
def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str):
def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str,
backtest_breakdown=[]):
"""
Print results for one strategy
"""
@@ -625,10 +801,23 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
if (results.get('results_per_enter_tag') is not None
or results.get('results_per_buy_tag') is not None):
# results_per_buy_tag is deprecated and should be removed 2 versions after short golive.
table = text_table_tags(
"enter_tag",
results.get('results_per_enter_tag', results.get('results_per_buy_tag')),
stake_currency=stake_currency)
if isinstance(table, str) and len(table) > 0:
print(' ENTER TAG STATS '.center(len(table.splitlines()[0]), '='))
print(table)
exit_reasons = results.get('exit_reason_summary', results.get('sell_reason_summary'))
table = text_table_exit_reason(exit_reason_stats=exit_reasons,
stake_currency=stake_currency)
if isinstance(table, str) and len(table) > 0:
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(' EXIT REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
@@ -636,6 +825,15 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
for period in backtest_breakdown:
days_breakdown_stats = generate_periodic_breakdown_stats(
trade_list=results['trades'], period=period)
table = text_table_periodic_breakdown(days_breakdown_stats=days_breakdown_stats,
stake_currency=stake_currency, period=period)
if isinstance(table, str) and len(table) > 0:
print(f' {period.upper()} BREAKDOWN '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_add_metrics(results)
if isinstance(table, str) and len(table) > 0:
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
@@ -643,6 +841,7 @@ def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency:
if isinstance(table, str) and len(table) > 0:
print('=' * len(table.splitlines()[0]))
print()
@@ -650,7 +849,9 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
stake_currency = config['stake_currency']
for strategy, results in backtest_stats['strategy'].items():
show_backtest_result(strategy, results, stake_currency)
show_backtest_result(
strategy, results, stake_currency,
config.get('backtest_breakdown', []))
if len(backtest_stats['strategy']) > 1:
# Print Strategy summary table
@@ -662,3 +863,13 @@ def show_backtest_results(config: Dict, backtest_stats: Dict):
print(table)
print('=' * len(table.splitlines()[0]))
print('\nFor more details, please look at the detail tables above')
def show_sorted_pairlist(config: Dict, backtest_stats: Dict):
if config.get('backtest_show_pair_list', False):
for strategy, results in backtest_stats['strategy'].items():
print(f"Pairs for Strategy {strategy}: \n[")
for result in results['results_per_pair']:
if result["key"] != 'TOTAL':
print(f'"{result["key"]}", // {result["profit_mean"]:.2%}')
print("]")

18
freqtrade/optimize/space/decimalspace.py
View File

@@ -7,11 +7,15 @@ class SKDecimal(Integer):
def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None,
name=None, dtype=np.int64):
self.decimals = decimals
_low = int(low * pow(10, self.decimals))
_high = int(high * pow(10, self.decimals))
self.pow_dot_one = pow(0.1, self.decimals)
self.pow_ten = pow(10, self.decimals)
_low = int(low * self.pow_ten)
_high = int(high * self.pow_ten)
# trunc to precision to avoid points out of space
self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals)
self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals)
self.low_orig = round(_low * self.pow_dot_one, self.decimals)
self.high_orig = round(_high * self.pow_dot_one, self.decimals)
super().__init__(_low, _high, prior, base, transform, name, dtype)
@@ -25,9 +29,9 @@ class SKDecimal(Integer):
return self.low_orig <= point <= self.high_orig
def transform(self, Xt):
aa = [int(x * pow(10, self.decimals)) for x in Xt]
return super().transform(aa)
return super().transform([int(v * self.pow_ten) for v in Xt])
def inverse_transform(self, Xt):
res = super().inverse_transform(Xt)
return [round(x * pow(0.1, self.decimals), self.decimals) for x in res]
# equivalent to [round(x * pow(0.1, self.decimals), self.decimals) for x in res]
return [int(v) / self.pow_ten for v in res]

193
freqtrade/persistence/migrations.py
View File

@@ -3,6 +3,8 @@ from typing import List
from sqlalchemy import inspect, text
from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
@@ -28,7 +30,38 @@ def get_backup_name(tabs, backup_prefix: str):
return table_back_name
def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List):
def get_last_sequence_ids(engine, trade_back_name, order_back_name):
order_id: int = None
trade_id: int = None
if engine.name == 'postgresql':
with engine.begin() as connection:
trade_id = connection.execute(text("select nextval('trades_id_seq')")).fetchone()[0]
order_id = connection.execute(text("select nextval('orders_id_seq')")).fetchone()[0]
with engine.begin() as connection:
connection.execute(text(
f"ALTER SEQUENCE orders_id_seq rename to {order_back_name}_id_seq_bak"))
connection.execute(text(
f"ALTER SEQUENCE trades_id_seq rename to {trade_back_name}_id_seq_bak"))
return order_id, trade_id
def set_sequence_ids(engine, order_id, trade_id):
if engine.name == 'postgresql':
with engine.begin() as connection:
if order_id:
connection.execute(text(f"ALTER SEQUENCE orders_id_seq RESTART WITH {order_id}"))
if trade_id:
connection.execute(text(f"ALTER SEQUENCE trades_id_seq RESTART WITH {trade_id}"))
def migrate_trades_and_orders_table(
decl_base, inspector, engine,
trade_back_name: str, cols: List,
order_back_name: str, cols_order: List):
base_currency = get_column_def(cols, 'base_currency', 'null')
stake_currency = get_column_def(cols, 'stake_currency', 'null')
fee_open = get_column_def(cols, 'fee_open', 'fee')
fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null')
fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null')
@@ -45,9 +78,25 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null')
max_rate = get_column_def(cols, 'max_rate', '0.0')
min_rate = get_column_def(cols, 'min_rate', 'null')
sell_reason = get_column_def(cols, 'sell_reason', 'null')
exit_reason = get_column_def(cols, 'sell_reason', get_column_def(cols, 'exit_reason', 'null'))
strategy = get_column_def(cols, 'strategy', 'null')
buy_tag = get_column_def(cols, 'buy_tag', 'null')
enter_tag = get_column_def(cols, 'buy_tag', get_column_def(cols, 'enter_tag', 'null'))
trading_mode = get_column_def(cols, 'trading_mode', 'null')
# Leverage Properties
leverage = get_column_def(cols, 'leverage', '1.0')
liquidation_price = get_column_def(cols, 'liquidation_price',
get_column_def(cols, 'isolated_liq', 'null'))
# sqlite does not support literals for booleans
is_short = get_column_def(cols, 'is_short', '0')
# Margin Properties
interest_rate = get_column_def(cols, 'interest_rate', '0.0')
# Futures properties
funding_fees = get_column_def(cols, 'funding_fees', '0.0')
# If ticker-interval existed use that, else null.
if has_column(cols, 'ticker_interval'):
timeframe = get_column_def(cols, 'timeframe', 'ticker_interval')
@@ -59,33 +108,46 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
close_profit_abs = get_column_def(
cols, 'close_profit_abs',
f"(amount * close_rate * (1 - {fee_close})) - {open_trade_value}")
sell_order_status = get_column_def(cols, 'sell_order_status', 'null')
exit_order_status = get_column_def(cols, 'exit_order_status',
get_column_def(cols, 'sell_order_status', 'null'))
amount_requested = get_column_def(cols, 'amount_requested', 'amount')
# Schema migration necessary
with engine.begin() as connection:
connection.execute(text(f"alter table trades rename to {table_back_name}"))
connection.execute(text(f"alter table trades rename to {trade_back_name}"))
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_back_name):
connection.execute(text(f"drop index {index['name']}"))
for index in inspector.get_indexes(trade_back_name):
if engine.name == 'mysql':
connection.execute(text(f"drop index {index['name']} on {trade_back_name}"))
else:
connection.execute(text(f"drop index {index['name']}"))
order_id, trade_id = get_last_sequence_ids(engine, trade_back_name, order_back_name)
drop_orders_table(engine, order_back_name)
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
# Copy data back - following the correct schema
with engine.begin() as connection:
connection.execute(text(f"""insert into trades
(id, exchange, pair, is_open,
(id, exchange, pair, base_currency, stake_currency, is_open,
fee_open, fee_open_cost, fee_open_currency,
fee_close, fee_close_cost, fee_close_currency, open_rate,
open_rate_requested, close_rate, close_rate_requested, close_profit,
stake_amount, amount, amount_requested, open_date, close_date, open_order_id,
stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct,
stoploss_order_id, stoploss_last_update,
max_rate, min_rate, sell_reason, sell_order_status, strategy, buy_tag,
timeframe, open_trade_value, close_profit_abs
max_rate, min_rate, exit_reason, exit_order_status, strategy, enter_tag,
timeframe, open_trade_value, close_profit_abs,
trading_mode, leverage, liquidation_price, is_short,
interest_rate, funding_fees
)
select id, lower(exchange), pair,
select id, lower(exchange), pair, {base_currency} base_currency,
{stake_currency} stake_currency,
is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost,
{fee_open_currency} fee_open_currency, {fee_close} fee_close,
{fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency,
@@ -96,83 +158,88 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col
{initial_stop_loss} initial_stop_loss,
{initial_stop_loss_pct} initial_stop_loss_pct,
{stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update,
{max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason,
{sell_order_status} sell_order_status,
{strategy} strategy, {buy_tag} buy_tag, {timeframe} timeframe,
{open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs
from {table_back_name}
{max_rate} max_rate, {min_rate} min_rate,
case when {exit_reason} = 'sell_signal' then 'exit_signal'
when {exit_reason} = 'custom_sell' then 'custom_exit'
when {exit_reason} = 'force_sell' then 'force_exit'
when {exit_reason} = 'emergency_sell' then 'emergency_exit'
else {exit_reason}
end exit_reason,
{exit_order_status} exit_order_status,
{strategy} strategy, {enter_tag} enter_tag, {timeframe} timeframe,
{open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs,
{trading_mode} trading_mode, {leverage} leverage, {liquidation_price} liquidation_price,
{is_short} is_short, {interest_rate} interest_rate,
{funding_fees} funding_fees
from {trade_back_name}
"""))
def migrate_open_orders_to_trades(engine):
with engine.begin() as connection:
connection.execute(text("""
insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open)
select id ft_trade_id, pair ft_pair, open_order_id,
case when close_rate_requested is null then 'buy'
else 'sell' end ft_order_side, 1 ft_is_open
from trades
where open_order_id is not null
union all
select id ft_trade_id, pair ft_pair, stoploss_order_id order_id,
'stoploss' ft_order_side, 1 ft_is_open
from trades
where stoploss_order_id is not null
"""))
migrate_orders_table(engine, order_back_name, cols_order)
set_sequence_ids(engine, order_id, trade_id)
def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List):
# Schema migration necessary
def drop_orders_table(engine, table_back_name: str):
# Drop and recreate orders table as backup
# This drops foreign keys, too.
with engine.begin() as connection:
connection.execute(text(f"alter table orders rename to {table_back_name}"))
connection.execute(text(f"create table {table_back_name} as select * from orders"))
connection.execute(text("drop table orders"))
with engine.begin() as connection:
# drop indexes on backup table in new session
for index in inspector.get_indexes(table_back_name):
connection.execute(text(f"drop index {index['name']}"))
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
def migrate_orders_table(engine, table_back_name: str, cols_order: List):
ft_fee_base = get_column_def(cols_order, 'ft_fee_base', 'null')
average = get_column_def(cols_order, 'average', 'null')
# sqlite does not support literals for booleans
with engine.begin() as connection:
connection.execute(text(f"""
insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
insert into orders (id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
status, symbol, order_type, side, price, amount, filled, average, remaining, cost,
order_date, order_filled_date, order_update_date)
order_date, order_filled_date, order_update_date, ft_fee_base)
select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id,
status, symbol, order_type, side, price, amount, filled, null average, remaining, cost,
order_date, order_filled_date, order_update_date
status, symbol, order_type, side, price, amount, filled, {average} average, remaining,
cost, order_date, order_filled_date, order_update_date, {ft_fee_base} ft_fee_base
from {table_back_name}
"""))
def set_sqlite_to_wal(engine):
if engine.name == 'sqlite' and str(engine.url) != 'sqlite://':
# Set Mode to
with engine.begin() as connection:
connection.execute(text("PRAGMA journal_mode=wal"))
def check_migrate(engine, decl_base, previous_tables) -> None:
"""
Checks if migration is necessary and migrates if necessary
"""
inspector = inspect(engine)
cols = inspector.get_columns('trades')
cols_trades = inspector.get_columns('trades')
cols_orders = inspector.get_columns('orders')
tabs = get_table_names_for_table(inspector, 'trades')
table_back_name = get_backup_name(tabs, 'trades_bak')
order_tabs = get_table_names_for_table(inspector, 'orders')
order_table_bak_name = get_backup_name(order_tabs, 'orders_bak')
# Check for latest column
if not has_column(cols, 'buy_tag'):
logger.info(f'Running database migration for trades - backup: {table_back_name}')
migrate_trades_table(decl_base, inspector, engine, table_back_name, cols)
# Reread columns - the above recreated the table!
inspector = inspect(engine)
cols = inspector.get_columns('trades')
# Check if migration necessary
# Migrates both trades and orders table!
# if ('orders' not in previous_tables
# or not has_column(cols_orders, 'leverage')):
if not has_column(cols_trades, 'base_currency'):
logger.info(f"Running database migration for trades - "
f"backup: {table_back_name}, {order_table_bak_name}")
migrate_trades_and_orders_table(
decl_base, inspector, engine, table_back_name, cols_trades,
order_table_bak_name, cols_orders)
if 'orders' not in previous_tables and 'trades' in previous_tables:
logger.info('Moving open orders to Orders table.')
migrate_open_orders_to_trades(engine)
else:
cols_order = inspector.get_columns('orders')
raise OperationalException(
"Your database seems to be very old. "
"Please update to freqtrade 2022.3 to migrate this database or "
"start with a fresh database.")
if not has_column(cols_order, 'average'):
tabs = get_table_names_for_table(inspector, 'orders')
# Empty for now - as there is only one iteration of the orders table so far.
table_back_name = get_backup_name(tabs, 'orders_bak')
migrate_orders_table(decl_base, inspector, engine, table_back_name, cols)
set_sqlite_to_wal(engine)

713
freqtrade/persistence/models.py
View File

File diff suppressed because it is too large Load Diff

34
freqtrade/persistence/pairlock_middleware.py
View File

@@ -103,6 +103,36 @@ class PairLocks():
if PairLocks.use_db:
PairLock.query.session.commit()
@staticmethod
def unlock_reason(reason: str, now: Optional[datetime] = None) -> None:
"""
Release all locks for this reason.
:param reason: Which reason to unlock
:param now: Datetime object (generated via datetime.now(timezone.utc)).
defaults to datetime.now(timezone.utc)
"""
if not now:
now = datetime.now(timezone.utc)
if PairLocks.use_db:
# used in live modes
logger.info(f"Releasing all locks with reason '{reason}':")
filters = [PairLock.lock_end_time > now,
PairLock.active.is_(True),
PairLock.reason == reason
]
locks = PairLock.query.filter(*filters)
for lock in locks:
logger.info(f"Releasing lock for {lock.pair} with reason '{reason}'.")
lock.active = False
PairLock.query.session.commit()
else:
# used in backtesting mode; don't show log messages for speed
locks = PairLocks.get_pair_locks(None)
for lock in locks:
if lock.reason == reason:
lock.active = False
@staticmethod
def is_global_lock(now: Optional[datetime] = None) -> bool:
"""
@@ -128,7 +158,9 @@ class PairLocks():
@staticmethod
def get_all_locks() -> List[PairLock]:
"""
Return all locks, also locks with expired end date
"""
if PairLocks.use_db:
return PairLock.query.all()
else:

165
freqtrade/plot/plotting.py
View File

@@ -1,15 +1,17 @@
import logging
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.data.btanalysis import (calculate_max_drawdown, combine_dataframes_with_mean,
from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown,
calculate_underwater, combine_dataframes_with_mean,
create_cum_profit, extract_trades_of_period, load_trades)
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import get_timerange, load_data
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds
from freqtrade.misc import pair_to_filename
@@ -51,6 +53,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
timerange=timerange,
startup_candles=startup_candles,
data_format=config.get('dataformat_ohlcv', 'json'),
candle_type=config.get('candle_type_def', CandleType.SPOT)
)
if startup_candles and data:
@@ -60,8 +63,8 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
startup_candles, min_date)
no_trades = False
filename = config.get('exportfilename')
if config.get('no_trades', False):
filename = config.get("exportfilename")
if config.get("no_trades", False):
no_trades = True
elif config['trade_source'] == 'file':
if not filename.is_dir() and not filename.is_file():
@@ -160,7 +163,7 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame,
Add scatter points indicating max drawdown
"""
try:
max_drawdown, highdate, lowdate, _, _ = calculate_max_drawdown(trades)
_, highdate, lowdate, _, _, max_drawdown = calculate_max_drawdown(trades)
drawdown = go.Scatter(
x=[highdate, lowdate],
@@ -169,8 +172,8 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame,
df_comb.loc[timeframe_to_prev_date(timeframe, lowdate), 'cum_profit'],
],
mode='markers',
name=f"Max drawdown {max_drawdown * 100:.2f}%",
text=f"Max drawdown {max_drawdown * 100:.2f}%",
name=f"Max drawdown {max_drawdown:.2%}",
text=f"Max drawdown {max_drawdown:.2%}",
marker=dict(
symbol='square-open',
size=9,
@@ -185,6 +188,48 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame,
return fig
def add_underwater(fig, row, trades: pd.DataFrame) -> make_subplots:
"""
Add underwater plot
"""
try:
underwater = calculate_underwater(trades, value_col="profit_abs")
underwater = go.Scatter(
x=underwater['date'],
y=underwater['drawdown'],
name="Underwater Plot",
fill='tozeroy',
fillcolor='#cc362b',
line={'color': '#cc362b'},
)
fig.add_trace(underwater, row, 1)
except ValueError:
logger.warning("No trades found - not plotting underwater plot")
return fig
def add_parallelism(fig, row, trades: pd.DataFrame, timeframe: str) -> make_subplots:
"""
Add Chart showing trade parallelism
"""
try:
result = analyze_trade_parallelism(trades, timeframe)
drawdown = go.Scatter(
x=result.index,
y=result['open_trades'],
name="Parallel trades",
fill='tozeroy',
fillcolor='#242222',
line={'color': '#242222'},
)
fig.add_trace(drawdown, row, 1)
except ValueError:
logger.warning("No trades found - not plotting Parallelism.")
return fig
def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
"""
Add trades to "fig"
@@ -192,10 +237,12 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots:
# Trades can be empty
if trades is not None and len(trades) > 0:
# Create description for sell summarizing the trade
trades['desc'] = trades.apply(lambda row: f"{round(row['profit_ratio'] * 100, 1)}%, "
f"{row['sell_reason']}, "
f"{row['trade_duration']} min",
axis=1)
trades['desc'] = trades.apply(
lambda row: f"{row['profit_ratio']:.2%}, " +
(f"{row['enter_tag']}, " if row['enter_tag'] is not None else "") +
f"{row['exit_reason']}, " +
f"{row['trade_duration']} min",
axis=1)
trade_buys = go.Scatter(
x=trades["open_date"],
y=trades["open_rate"],
@@ -340,6 +387,35 @@ def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots:
return fig
def create_scatter(
data,
column_name,
color,
direction
) -> Optional[go.Scatter]:
if column_name in data.columns:
df_short = data[data[column_name] == 1]
if len(df_short) > 0:
shorts = go.Scatter(
x=df_short.date,
y=df_short.close,
mode='markers',
name=column_name,
marker=dict(
symbol=f"triangle-{direction}-dot",
size=9,
line=dict(width=1),
color=color,
)
)
return shorts
else:
logger.warning(f"No {column_name}-signals found.")
return None
def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, *,
indicators1: List[str] = [],
indicators2: List[str] = [],
@@ -386,43 +462,15 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
)
fig.add_trace(candles, 1, 1)
if 'buy' in data.columns:
df_buy = data[data['buy'] == 1]
if len(df_buy) > 0:
buys = go.Scatter(
x=df_buy.date,
y=df_buy.close,
mode='markers',
name='buy',
marker=dict(
symbol='triangle-up-dot',
size=9,
line=dict(width=1),
color='green',
)
)
fig.add_trace(buys, 1, 1)
else:
logger.warning("No buy-signals found.")
longs = create_scatter(data, 'enter_long', 'green', 'up')
exit_longs = create_scatter(data, 'exit_long', 'red', 'down')
shorts = create_scatter(data, 'enter_short', 'blue', 'down')
exit_shorts = create_scatter(data, 'exit_short', 'violet', 'up')
for scatter in [longs, exit_longs, shorts, exit_shorts]:
if scatter:
fig.add_trace(scatter, 1, 1)
if 'sell' in data.columns:
df_sell = data[data['sell'] == 1]
if len(df_sell) > 0:
sells = go.Scatter(
x=df_sell.date,
y=df_sell.close,
mode='markers',
name='sell',
marker=dict(
symbol='triangle-down-dot',
size=9,
line=dict(width=1),
color='red',
)
)
fig.add_trace(sells, 1, 1)
else:
logger.warning("No sell-signals found.")
# Add Bollinger Bands
fig = plot_area(fig, 1, data, 'bb_lowerband', 'bb_upperband',
label="Bollinger Band")
@@ -460,7 +508,12 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure:
# Combine close-values for all pairs, rename columns to "pair"
df_comb = combine_dataframes_with_mean(data, "close")
try:
df_comb = combine_dataframes_with_mean(data, "close")
except ValueError:
raise OperationalException(
"No data found. Please make sure that data is available for "
"the timerange and pairs selected.")
# Trim trades to available OHLCV data
trades = extract_trades_of_period(df_comb, trades, date_index=True)
@@ -477,20 +530,30 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
name='Avg close price',
)
fig = make_subplots(rows=3, cols=1, shared_xaxes=True,
row_width=[1, 1, 1],
fig = make_subplots(rows=5, cols=1, shared_xaxes=True,
row_heights=[1, 1, 1, 0.5, 1],
vertical_spacing=0.05,
subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"])
subplot_titles=[
"AVG Close Price",
"Combined Profit",
"Profit per pair",
"Parallelism",
"Underwater",
])
fig['layout'].update(title="Freqtrade Profit plot")
fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
fig['layout']['yaxis4'].update(title='Trade count')
fig['layout']['yaxis5'].update(title='Underwater Plot')
fig['layout']['xaxis']['rangeslider'].update(visible=False)
fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"])
fig.add_trace(avgclose, 1, 1)
fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit')
fig = add_max_drawdown(fig, 2, trades, df_comb, timeframe)
fig = add_parallelism(fig, 4, trades, timeframe)
fig = add_underwater(fig, 5, trades)
for pair in pairs:
profit_col = f'cum_profit_{pair}'

10
freqtrade/plugins/pairlist/AgeFilter.py
View File

@@ -9,6 +9,7 @@ import arrow
from pandas import DataFrame
from freqtrade.configuration import PeriodicCache
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -71,8 +72,8 @@ class AgeFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist
"""
needed_pairs = [
(p, '1d') for p in pairlist
needed_pairs: ListPairsWithTimeframes = [
(p, '1d', self._config['candle_type_def']) for p in pairlist
if p not in self._symbolsChecked and p not in self._symbolsCheckFailed]
if not needed_pairs:
# Remove pairs that have been removed before
@@ -88,7 +89,8 @@ class AgeFilter(IPairList):
candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
if self._enabled:
for p in deepcopy(pairlist):
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
daily_candles = candles[(p, '1d', self._config['candle_type_def'])] if (
p, '1d', self._config['candle_type_def']) in candles else None
if not self._validate_pair_loc(p, daily_candles):
pairlist.remove(p)
self.log_once(f"Validated {len(pairlist)} pairs.", logger.info)
@@ -98,7 +100,7 @@ class AgeFilter(IPairList):
"""
Validate age for the ticker
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:param daily_candles: Downloaded daily candles
:return: True if the pair can stay, false if it should be removed
"""
# Check symbol in cache

14
freqtrade/plugins/pairlist/PerformanceFilter.py
View File

@@ -21,6 +21,7 @@ class PerformanceFilter(IPairList):
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._minutes = pairlistconfig.get('minutes', 0)
self._min_profit = pairlistconfig.get('min_profit', None)
@property
def needstickers(self) -> bool:
@@ -59,6 +60,7 @@ class PerformanceFilter(IPairList):
# Get pairlist from performance dataframe values
list_df = pd.DataFrame({'pair': pairlist})
list_df['prior_idx'] = list_df.index
# Set initial value for pairs with no trades to 0
# Sort the list using:
@@ -66,8 +68,16 @@ class PerformanceFilter(IPairList):
# - then count (low to high, so as to favor same performance with fewer trades)
# - then pair name alphametically
sorted_df = list_df.merge(performance, on='pair', how='left')\
.fillna(0).sort_values(by=['count', 'pair'], ascending=True)\
.sort_values(by=['profit'], ascending=False)
.fillna(0).sort_values(by=['count', 'prior_idx'], ascending=True)\
.sort_values(by=['profit_ratio'], ascending=False)
if self._min_profit is not None:
removed = sorted_df[sorted_df['profit_ratio'] < self._min_profit]
for _, row in removed.iterrows():
self.log_once(
f"Removing pair {row['pair']} since {row['profit_ratio']} is "
f"below {self._min_profit}", logger.info)
sorted_df = sorted_df[sorted_df['profit_ratio'] >= self._min_profit]
pairlist = sorted_df['pair'].tolist()
return pairlist

2
freqtrade/plugins/pairlist/PrecisionFilter.py
View File

@@ -51,7 +51,7 @@ class PrecisionFilter(IPairList):
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
stop_price = ticker['ask'] * self._stoploss
stop_price = ticker['last'] * self._stoploss
# Adjust stop-prices to precision
sp = self._exchange.price_to_precision(pair, stop_price)

7
freqtrade/plugins/pairlist/PriceFilter.py
View File

@@ -50,7 +50,7 @@ class PriceFilter(IPairList):
"""
active_price_filters = []
if self._low_price_ratio != 0:
active_price_filters.append(f"below {self._low_price_ratio * 100}%")
active_price_filters.append(f"below {self._low_price_ratio:.1%}")
if self._min_price != 0:
active_price_filters.append(f"below {self._min_price:.8f}")
if self._max_price != 0:
@@ -82,7 +82,7 @@ class PriceFilter(IPairList):
changeperc = compare / ticker['last']
if changeperc > self._low_price_ratio:
self.log_once(f"Removed {pair} from whitelist, "
f"because 1 unit is {changeperc * 100:.3f}%", logger.info)
f"because 1 unit is {changeperc:.3%}", logger.info)
return False
# Perform low_amount check
@@ -90,8 +90,7 @@ class PriceFilter(IPairList):
price = ticker['last']
market = self._exchange.markets[pair]
limits = market['limits']
if ('amount' in limits and 'min' in limits['amount']
and limits['amount']['min'] is not None):
if (limits['amount']['min'] is not None):
min_amount = limits['amount']['min']
min_precision = market['precision']['amount']

11
freqtrade/plugins/pairlist/ShuffleFilter.py
View File

@@ -5,6 +5,7 @@ import logging
import random
from typing import Any, Dict, List
from freqtrade.enums import RunMode
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -18,7 +19,15 @@ class ShuffleFilter(IPairList):
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._seed = pairlistconfig.get('seed')
# Apply seed in backtesting mode to get comparable results,
# but not in live modes to get a non-repeating order of pairs during live modes.
if config.get('runmode') in (RunMode.LIVE, RunMode.DRY_RUN):
self._seed = None
logger.info("Live mode detected, not applying seed.")
else:
self._seed = pairlistconfig.get('seed')
logger.info(f"Backtesting mode detected, applying seed value: {self._seed}")
self._random = random.Random(self._seed)
@property

11
freqtrade/plugins/pairlist/SpreadFilter.py
View File

@@ -4,6 +4,7 @@ Spread pair list filter
import logging
from typing import Any, Dict
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -20,6 +21,12 @@ class SpreadFilter(IPairList):
self._max_spread_ratio = pairlistconfig.get('max_spread_ratio', 0.005)
self._enabled = self._max_spread_ratio != 0
if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException(
'Exchange does not support fetchTickers, therefore SpreadFilter cannot be used.'
'Please edit your config and restart the bot.'
)
@property
def needstickers(self) -> bool:
"""
@@ -34,7 +41,7 @@ class SpreadFilter(IPairList):
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Filtering pairs with ask/bid diff above "
f"{self._max_spread_ratio * 100}%.")
f"{self._max_spread_ratio:.2%}.")
def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool:
"""
@@ -47,7 +54,7 @@ class SpreadFilter(IPairList):
spread = 1 - ticker['bid'] / ticker['ask']
if spread > self._max_spread_ratio:
self.log_once(f"Removed {pair} from whitelist, because spread "
f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%",
f"{spread:.3%} > {self._max_spread_ratio:.3%}",
logger.info)
return False
else:

12
freqtrade/plugins/pairlist/StaticPairList.py
View File

@@ -4,9 +4,9 @@ Static Pair List provider
Provides pair white list as it configured in config
"""
import logging
from copy import deepcopy
from typing import Any, Dict, List
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -20,10 +20,6 @@ class StaticPairList(IPairList):
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
if self._pairlist_pos != 0:
raise OperationalException(f"{self.name} can only be used in the first position "
"in the list of Pairlist Handlers.")
self._allow_inactive = self._pairlistconfig.get('allow_inactive', False)
@property
@@ -64,4 +60,8 @@ class StaticPairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
return pairlist
pairlist_ = deepcopy(pairlist)
for pair in self._config['exchange']['pair_whitelist']:
if pair not in pairlist_:
pairlist_.append(pair)
return pairlist_

14
freqtrade/plugins/pairlist/VolatilityFilter.py
View File

@@ -8,9 +8,10 @@ from typing import Any, Dict, List, Optional
import arrow
import numpy as np
from cachetools.ttl import TTLCache
from cachetools import TTLCache
from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -33,6 +34,7 @@ class VolatilityFilter(IPairList):
self._min_volatility = pairlistconfig.get('min_volatility', 0)
self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
self._def_candletype = self._config['candle_type_def']
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
@@ -67,7 +69,8 @@ class VolatilityFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
needed_pairs: ListPairsWithTimeframes = [
(p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
since_ms = (arrow.utcnow()
.floor('day')
@@ -81,7 +84,8 @@ class VolatilityFilter(IPairList):
if self._enabled:
for p in deepcopy(pairlist):
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
daily_candles = candles[(p, '1d', self._def_candletype)] if (
p, '1d', self._def_candletype) in candles else None
if not self._validate_pair_loc(p, daily_candles):
pairlist.remove(p)
return pairlist
@@ -90,7 +94,7 @@ class VolatilityFilter(IPairList):
"""
Validate trading range
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:param daily_candles: Downloaded daily candles
:return: True if the pair can stay, false if it should be removed
"""
# Check symbol in cache
@@ -103,7 +107,7 @@ class VolatilityFilter(IPairList):
returns = (np.log(daily_candles.close / daily_candles.close.shift(-1)))
returns.fillna(0, inplace=True)
volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days)
volatility_series = returns.rolling(window=self._days).std() * np.sqrt(self._days)
volatility_avg = volatility_series.mean()
if self._min_volatility <= volatility_avg <= self._max_volatility:

77
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -4,12 +4,12 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from functools import partial
from typing import Any, Dict, List
import arrow
from cachetools.ttl import TTLCache
from cachetools import TTLCache
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import format_ms_time
@@ -43,6 +43,7 @@ class VolumePairList(IPairList):
self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
self._def_candletype = self._config['candle_type_def']
if (self._lookback_days > 0) & (self._lookback_period > 0):
raise OperationalException(
@@ -70,10 +71,13 @@ class VolumePairList(IPairList):
f'to at least {self._tf_in_sec} and restart the bot.'
)
if not self._exchange.exchange_has('fetchTickers'):
if (not self._use_range and not (
self._exchange.exchange_has('fetchTickers')
and self._exchange._ft_has["tickers_have_quoteVolume"])):
raise OperationalException(
'Exchange does not support dynamic whitelist. '
'Please edit your config and restart the bot.'
"Exchange does not support dynamic whitelist in this configuration. "
"Please edit your config and either remove Volumepairlist, "
"or switch to using candles. and restart the bot."
)
if not self._validate_keys(self._sort_key):
@@ -94,7 +98,7 @@ class VolumePairList(IPairList):
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return True
return not self._use_range
def _validate_keys(self, key):
return key in SORT_VALUES
@@ -120,11 +124,20 @@ class VolumePairList(IPairList):
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
filtered_tickers = [
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and (self._use_range or v[self._sort_key] is not None))]
pairlist = [s['symbol'] for s in filtered_tickers]
_pairlist = [k for k in self._exchange.get_markets(
quote_currencies=[self._stake_currency],
tradable_only=True, active_only=True).keys()]
# No point in testing for blacklisted pairs...
_pairlist = self.verify_blacklist(_pairlist, logger.info)
if not self._use_range:
filtered_tickers = [
v for k, v in tickers.items()
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and (self._use_range or v[self._sort_key] is not None)
and v['symbol'] in _pairlist)]
pairlist = [s['symbol'] for s in filtered_tickers]
else:
pairlist = _pairlist
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache['pairlist'] = pairlist.copy()
@@ -139,11 +152,11 @@ class VolumePairList(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
# Use the incoming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
if self._use_range:
# Create bare minimum from tickers structure.
filtered_tickers: List[Dict[str, Any]] = [{'symbol': k} for k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
since_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-(self._lookback_period * self._tf_in_min)
@@ -159,11 +172,10 @@ class VolumePairList(IPairList):
self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}", logger.info)
needed_pairs = [
(p, self._lookback_timeframe) for p in
[
s['symbol'] for s in filtered_tickers
] if p not in self._pair_cache
needed_pairs: ListPairsWithTimeframes = [
(p, self._lookback_timeframe, self._def_candletype) for p in
[s['symbol'] for s in filtered_tickers]
if p not in self._pair_cache
]
# Get all candles
@@ -174,16 +186,22 @@ class VolumePairList(IPairList):
)
for i, p in enumerate(filtered_tickers):
pair_candles = candles[
(p['symbol'], self._lookback_timeframe)
] if (p['symbol'], self._lookback_timeframe) in candles else None
(p['symbol'], self._lookback_timeframe, self._def_candletype)
] if (
p['symbol'], self._lookback_timeframe, self._def_candletype
) in candles else None
# in case of candle data calculate typical price and quoteVolume for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
if self._exchange._ft_has["ohlcv_volume_currency"] == "base":
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
else:
# Exchange ohlcv data is in quote volume already.
pair_candles['quoteVolume'] = pair_candles['volume']
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
quoteVolume = (pair_candles['quoteVolume']
@@ -195,6 +213,9 @@ class VolumePairList(IPairList):
filtered_tickers[i]['quoteVolume'] = quoteVolume
else:
filtered_tickers[i]['quoteVolume'] = 0
else:
# Tickers mode - filter based on incomming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
if self._min_value > 0:
filtered_tickers = [
@@ -204,7 +225,7 @@ class VolumePairList(IPairList):
# Validate whitelist to only have active market pairs
pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
pairs = self.verify_blacklist(pairs, partial(self.log_once, logmethod=logger.info))
pairs = self.verify_blacklist(pairs, logmethod=logger.info)
# Limit pairlist to the requested number of pairs
pairs = pairs[:self._number_pairs]

12
freqtrade/plugins/pairlist/rangestabilityfilter.py
View File

@@ -6,9 +6,10 @@ from copy import deepcopy
from typing import Any, Dict, List, Optional
import arrow
from cachetools.ttl import TTLCache
from cachetools import TTLCache
from pandas import DataFrame
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -28,6 +29,7 @@ class RangeStabilityFilter(IPairList):
self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01)
self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None)
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
self._def_candletype = self._config['candle_type_def']
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
@@ -65,7 +67,8 @@ class RangeStabilityFilter(IPairList):
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
needed_pairs: ListPairsWithTimeframes = [
(p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache]
since_ms = (arrow.utcnow()
.floor('day')
@@ -79,7 +82,8 @@ class RangeStabilityFilter(IPairList):
if self._enabled:
for p in deepcopy(pairlist):
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
daily_candles = candles[(p, '1d', self._def_candletype)] if (
p, '1d', self._def_candletype) in candles else None
if not self._validate_pair_loc(p, daily_candles):
pairlist.remove(p)
return pairlist
@@ -88,7 +92,7 @@ class RangeStabilityFilter(IPairList):
"""
Validate trading range
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:param daily_candles: Downloaded daily candles
:return: True if the pair can stay, false if it should be removed
"""
# Check symbol in cache

23
freqtrade/plugins/pairlistmanager.py
View File

@@ -2,13 +2,15 @@
PairList manager class
"""
import logging
from copy import deepcopy
from functools import partial
from typing import Dict, List
from cachetools import TTLCache, cached
from freqtrade.constants import ListPairsWithTimeframes
from freqtrade.enums import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.mixins import LoggingMixin
from freqtrade.plugins.pairlist.IPairList import IPairList
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import PairListResolver
@@ -17,7 +19,7 @@ from freqtrade.resolvers import PairListResolver
logger = logging.getLogger(__name__)
class PairListManager():
class PairListManager(LoggingMixin):
def __init__(self, exchange, config: dict) -> None:
self._exchange = exchange
@@ -41,6 +43,9 @@ class PairListManager():
if not self._pairlist_handlers:
raise OperationalException("No Pairlist Handlers defined")
refresh_period = config.get('pairlist_refresh_period', 3600)
LoggingMixin.__init__(self, logger, refresh_period)
@property
def whitelist(self) -> List[str]:
"""The current whitelist"""
@@ -108,9 +113,10 @@ class PairListManager():
except ValueError as err:
logger.error(f"Pair blacklist contains an invalid Wildcard: {err}")
return []
for pair in deepcopy(pairlist):
log_once = partial(self.log_once, logmethod=logmethod)
for pair in pairlist.copy():
if pair in blacklist:
logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...")
log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...")
pairlist.remove(pair)
return pairlist
@@ -127,7 +133,6 @@ class PairListManager():
:return: pairlist - whitelisted pairs
"""
try:
whitelist = expand_pairlist(pairlist, self._exchange.get_markets().keys(), keep_invalid)
except ValueError as err:
logger.error(f"Pair whitelist contains an invalid Wildcard: {err}")
@@ -138,4 +143,10 @@ class PairListManager():
"""
Create list of pair tuples with (pair, timeframe)
"""
return [(pair, timeframe or self._config['timeframe']) for pair in pairs]
return [
(
pair,
timeframe or self._config['timeframe'],
self._config.get('candle_type_def', CandleType.SPOT)
) for pair in pairs
]

5
freqtrade/plugins/protections/max_drawdown_protection.py
View File

@@ -36,7 +36,7 @@ class MaxDrawdown(IProtection):
"""
LockReason to use
"""
return (f'{drawdown} > {self._max_allowed_drawdown} in {self.lookback_period_str}, '
return (f'{drawdown} passed {self._max_allowed_drawdown} in {self.lookback_period_str}, '
f'locking for {self.stop_duration_str}.')
def _max_drawdown(self, date_now: datetime) -> ProtectionReturn:
@@ -55,7 +55,8 @@ class MaxDrawdown(IProtection):
# Drawdown is always positive
try:
drawdown, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit')
# TODO: This should use absolute profit calculation, considering account balance.
drawdown, _, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit')
except ValueError:
return False, None, None

18
freqtrade/plugins/protections/stoploss_guard.py
View File

@@ -3,7 +3,7 @@ import logging
from datetime import datetime, timedelta
from typing import Any, Dict
from freqtrade.enums import SellType
from freqtrade.enums import ExitType
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn
@@ -41,21 +41,11 @@ class StoplossGuard(IProtection):
Evaluate recent trades
"""
look_back_until = date_now - timedelta(minutes=self._lookback_period)
# filters = [
# Trade.is_open.is_(False),
# Trade.close_date > look_back_until,
# or_(Trade.sell_reason == SellType.STOP_LOSS.value,
# and_(Trade.sell_reason == SellType.TRAILING_STOP_LOSS.value,
# Trade.close_profit < 0))
# ]
# if pair:
# filters.append(Trade.pair == pair)
# trades = Trade.get_trades(filters).all()
trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until)
trades = [trade for trade in trades1 if (str(trade.sell_reason) in (
SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value,
SellType.STOPLOSS_ON_EXCHANGE.value)
trades = [trade for trade in trades1 if (str(trade.exit_reason) in (
ExitType.TRAILING_STOP_LOSS.value, ExitType.STOP_LOSS.value,
ExitType.STOPLOSS_ON_EXCHANGE.value)
and trade.close_profit and trade.close_profit < 0)]
if len(trades) < self._trade_limit:

1
freqtrade/resolvers/hyperopt_resolver.py
View File

@@ -44,7 +44,6 @@ class HyperOptLossResolver(IResolver):
extra_dir=config.get('hyperopt_path'))
# Assign timeframe to be used in hyperopt
hyperoptloss.__class__.ticker_interval = str(config['timeframe'])
hyperoptloss.__class__.timeframe = str(config['timeframe'])
return hyperoptloss

64
freqtrade/resolvers/iresolver.py
View File

@@ -6,6 +6,7 @@ This module load custom objects
import importlib.util
import inspect
import logging
import sys
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union
@@ -15,6 +16,22 @@ from freqtrade.exceptions import OperationalException
logger = logging.getLogger(__name__)
class PathModifier:
def __init__(self, path: Path):
self.path = path
def __enter__(self):
"""Inject path to allow importing with relative imports."""
sys.path.insert(0, str(self.path))
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Undo insertion of local path."""
str_path = str(self.path)
if str_path in sys.path:
sys.path.remove(str_path)
class IResolver:
"""
This class contains all the logic to load custom classes
@@ -57,27 +74,32 @@ class IResolver:
# Generate spec based on absolute path
# Pass object_name as first argument to have logging print a reasonable name.
spec = importlib.util.spec_from_file_location(object_name or "", str(module_path))
if not spec:
return iter([None])
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module) # type: ignore # importlib does not use typehints
except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
# Catch errors in case a specific module is not installed
logger.warning(f"Could not import {module_path} due to '{err}'")
if enum_failed:
with PathModifier(module_path.parent):
module_name = module_path.stem or ""
spec = importlib.util.spec_from_file_location(module_name, str(module_path))
if not spec:
return iter([None])
valid_objects_gen = (
(obj, inspect.getsource(module)) for
name, obj in inspect.getmembers(
module, inspect.isclass) if ((object_name is None or object_name == name)
and issubclass(obj, cls.object_type)
and obj is not cls.object_type)
)
return valid_objects_gen
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module) # type: ignore # importlib does not use typehints
except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
# Catch errors in case a specific module is not installed
logger.warning(f"Could not import {module_path} due to '{err}'")
if enum_failed:
return iter([None])
valid_objects_gen = (
(obj, inspect.getsource(module)) for
name, obj in inspect.getmembers(
module, inspect.isclass) if ((object_name is None or object_name == name)
and issubclass(obj, cls.object_type)
and obj is not cls.object_type
and obj.__module__ == module_name
)
)
# The __module__ check ensures we only use strategies that are defined in this folder.
return valid_objects_gen
@classmethod
def _search_object(cls, directory: Path, *, object_name: str, add_source: bool = False
@@ -91,7 +113,7 @@ class IResolver:
logger.debug(f"Searching for {cls.object_type.__name__} {object_name} in '{directory}'")
for entry in directory.iterdir():
# Only consider python files
if not str(entry).endswith('.py'):
if entry.suffix != '.py':
logger.debug('Ignoring %s', entry)
continue
if entry.is_symlink() and not entry.is_file():
@@ -169,7 +191,7 @@ class IResolver:
objects = []
for entry in directory.iterdir():
# Only consider python files
if not str(entry).endswith('.py'):
if entry.suffix != '.py':
logger.debug('Ignoring %s', entry)
continue
module_path = entry.resolve()

157
freqtrade/resolvers/strategy_resolver.py
View File

@@ -10,7 +10,9 @@ from inspect import getfullargspec
from pathlib import Path
from typing import Any, Dict, Optional
from freqtrade.configuration.config_validation import validate_migrated_strategy_settings
from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES
from freqtrade.enums import TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.resolvers import IResolver
from freqtrade.strategy.interface import IStrategy
@@ -45,28 +47,24 @@ class StrategyResolver(IResolver):
strategy_name, config=config,
extra_dir=config.get('strategy_path'))
if hasattr(strategy, 'ticker_interval') and not hasattr(strategy, 'timeframe'):
# Assign ticker_interval to timeframe to keep compatibility
if 'timeframe' not in config:
logger.warning(
"DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'."
)
strategy.timeframe = strategy.ticker_interval
if strategy._ft_params_from_file:
# Set parameters from Hyperopt results file
params = strategy._ft_params_from_file
strategy.minimal_roi = params.get('roi', strategy.minimal_roi)
strategy.minimal_roi = params.get('roi', getattr(strategy, 'minimal_roi', {}))
strategy.stoploss = params.get('stoploss', {}).get('stoploss', strategy.stoploss)
strategy.stoploss = params.get('stoploss', {}).get(
'stoploss', getattr(strategy, 'stoploss', -0.1))
trailing = params.get('trailing', {})
strategy.trailing_stop = trailing.get('trailing_stop', strategy.trailing_stop)
strategy.trailing_stop_positive = trailing.get('trailing_stop_positive',
strategy.trailing_stop_positive)
strategy.trailing_stop = trailing.get(
'trailing_stop', getattr(strategy, 'trailing_stop', False))
strategy.trailing_stop_positive = trailing.get(
'trailing_stop_positive', getattr(strategy, 'trailing_stop_positive', None))
strategy.trailing_stop_positive_offset = trailing.get(
'trailing_stop_positive_offset', strategy.trailing_stop_positive_offset)
'trailing_stop_positive_offset',
getattr(strategy, 'trailing_stop_positive_offset', 0))
strategy.trailing_only_offset_is_reached = trailing.get(
'trailing_only_offset_is_reached', strategy.trailing_only_offset_is_reached)
'trailing_only_offset_is_reached',
getattr(strategy, 'trailing_only_offset_is_reached', 0.0))
# Set attributes
# Check if we need to override configuration
@@ -87,12 +85,14 @@ class StrategyResolver(IResolver):
("protections", None),
("startup_candle_count", None),
("unfilledtimeout", None),
("use_sell_signal", True),
("sell_profit_only", False),
("ignore_roi_if_buy_signal", False),
("sell_profit_offset", 0.0),
("use_exit_signal", True),
("exit_profit_only", False),
("ignore_roi_if_entry_signal", False),
("exit_profit_offset", 0.0),
("disable_dataframe_checks", False),
("ignore_buying_expired_candle_after", 0)
("ignore_buying_expired_candle_after", 0),
("position_adjustment_enable", False),
("max_entry_position_adjustment", -1),
]
for attribute, default in attributes:
StrategyResolver._override_attribute_helper(strategy, config,
@@ -139,10 +139,6 @@ class StrategyResolver(IResolver):
"""
Normalize attributes to have the correct type.
"""
# Assign deprecated variable - to not break users code relying on this.
if hasattr(strategy, 'timeframe'):
strategy.ticker_interval = strategy.timeframe
# Sort and apply type conversions
if hasattr(strategy, 'minimal_roi'):
strategy.minimal_roi = dict(sorted(
@@ -153,14 +149,83 @@ class StrategyResolver(IResolver):
return strategy
@staticmethod
def _strategy_sanity_validations(strategy):
def _strategy_sanity_validations(strategy: IStrategy):
# Ensure necessary migrations are performed first.
validate_migrated_strategy_settings(strategy.config)
if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES):
raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. "
f"Order-types mapping is incomplete.")
if not all(k in strategy.order_time_in_force for k in REQUIRED_ORDERTIF):
raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. "
f"Order-time-in-force mapping is incomplete.")
trading_mode = strategy.config.get('trading_mode', TradingMode.SPOT)
if (strategy.can_short and trading_mode == TradingMode.SPOT):
raise ImportError(
"Short strategies cannot run in spot markets. Please make sure that this "
"is the correct strategy and that your trading mode configuration is correct. "
"You can run this strategy in spot markets by setting `can_short=False`"
" in your strategy. Please note that short signals will be ignored in that case."
)
@staticmethod
def validate_strategy(strategy: IStrategy) -> IStrategy:
if strategy.config.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
# Require new method
warn_deprecated_setting(strategy, 'sell_profit_only', 'exit_profit_only', True)
warn_deprecated_setting(strategy, 'sell_profit_offset', 'exit_profit_offset', True)
warn_deprecated_setting(strategy, 'use_sell_signal', 'use_exit_signal', True)
warn_deprecated_setting(strategy, 'ignore_roi_if_buy_signal',
'ignore_roi_if_entry_signal', True)
if not check_override(strategy, IStrategy, 'populate_entry_trend'):
raise OperationalException("`populate_entry_trend` must be implemented.")
if not check_override(strategy, IStrategy, 'populate_exit_trend'):
raise OperationalException("`populate_exit_trend` must be implemented.")
if check_override(strategy, IStrategy, 'check_buy_timeout'):
raise OperationalException("Please migrate your implementation "
"of `check_buy_timeout` to `check_entry_timeout`.")
if check_override(strategy, IStrategy, 'check_sell_timeout'):
raise OperationalException("Please migrate your implementation "
"of `check_sell_timeout` to `check_exit_timeout`.")
if check_override(strategy, IStrategy, 'custom_sell'):
raise OperationalException(
"Please migrate your implementation of `custom_sell` to `custom_exit`.")
else:
# TODO: Implementing one of the following methods should show a deprecation warning
# buy_trend and sell_trend, custom_sell
warn_deprecated_setting(strategy, 'sell_profit_only', 'exit_profit_only')
warn_deprecated_setting(strategy, 'sell_profit_offset', 'exit_profit_offset')
warn_deprecated_setting(strategy, 'use_sell_signal', 'use_exit_signal')
warn_deprecated_setting(strategy, 'ignore_roi_if_buy_signal',
'ignore_roi_if_entry_signal')
if (
not check_override(strategy, IStrategy, 'populate_buy_trend')
and not check_override(strategy, IStrategy, 'populate_entry_trend')
):
raise OperationalException(
"`populate_entry_trend` or `populate_buy_trend` must be implemented.")
if (
not check_override(strategy, IStrategy, 'populate_sell_trend')
and not check_override(strategy, IStrategy, 'populate_exit_trend')
):
raise OperationalException(
"`populate_exit_trend` or `populate_sell_trend` must be implemented.")
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
if any(x == 2 for x in [
strategy._populate_fun_len,
strategy._buy_fun_len,
strategy._sell_fun_len
]):
strategy.INTERFACE_VERSION = 1
return strategy
@staticmethod
def _load_strategy(strategy_name: str,
@@ -193,23 +258,35 @@ class StrategyResolver(IResolver):
# register temp path with the bot
abs_paths.insert(0, temp.resolve())
strategy = StrategyResolver._load_object(paths=abs_paths,
object_name=strategy_name,
add_source=True,
kwargs={'config': config},
)
if strategy:
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
if any(x == 2 for x in [strategy._populate_fun_len,
strategy._buy_fun_len,
strategy._sell_fun_len]):
strategy.INTERFACE_VERSION = 1
strategy = StrategyResolver._load_object(
paths=abs_paths,
object_name=strategy_name,
add_source=True,
kwargs={'config': config},
)
return strategy
if strategy:
return StrategyResolver.validate_strategy(strategy)
raise OperationalException(
f"Impossible to load Strategy '{strategy_name}'. This class does not exist "
"or contains Python code errors."
)
def warn_deprecated_setting(strategy: IStrategy, old: str, new: str, error=False):
if hasattr(strategy, old):
errormsg = f"DEPRECATED: Using '{old}' moved to '{new}'."
if error:
raise OperationalException(errormsg)
logger.warning(errormsg)
setattr(strategy, new, getattr(strategy, f'{old}'))
def check_override(object, parentclass, attribute):
"""
Checks if a object overrides the parent class attribute.
:returns: True if the object is overridden.
"""
return getattr(type(object), attribute) != getattr(parentclass, attribute)

101
freqtrade/rpc/api_server/api_backtest.py
View File

@@ -1,13 +1,17 @@
import asyncio
import logging
from copy import deepcopy
from typing import Any, Dict, List
from fastapi import APIRouter, BackgroundTasks, Depends
from freqtrade.configuration.config_validation import validate_config_consistency
from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
from freqtrade.enums import BacktestState
from freqtrade.exceptions import DependencyException
from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse
from freqtrade.rpc.api_server.deps import get_config
from freqtrade.rpc.api_server.api_schemas import (BacktestHistoryEntry, BacktestRequest,
BacktestResponse)
from freqtrade.rpc.api_server.deps import get_config, is_webserver_mode
from freqtrade.rpc.api_server.webserver import ApiServer
from freqtrade.rpc.rpc import RPCException
@@ -19,8 +23,9 @@ router = APIRouter()
@router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
# flake8: noqa: C901
async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
config=Depends(get_config)):
config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
"""Start backtesting if not done so already"""
if ApiServer._bgtask_running:
raise RPCException('Bot Background task already running')
@@ -31,52 +36,73 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
for setting in settings.keys():
if settings[setting] is not None:
btconfig[setting] = settings[setting]
try:
btconfig['stake_amount'] = float(btconfig['stake_amount'])
except ValueError:
pass
# Force dry-run for backtesting
btconfig['dry_run'] = True
# Start backtesting
# Initialize backtesting object
def run_backtest():
from freqtrade.optimize.optimize_reports import generate_backtest_stats
from freqtrade.optimize.optimize_reports import (generate_backtest_stats,
store_backtest_stats)
from freqtrade.resolvers import StrategyResolver
asyncio.set_event_loop(asyncio.new_event_loop())
try:
# Reload strategy
lastconfig = ApiServer._bt_last_config
strat = StrategyResolver.load_strategy(btconfig)
validate_config_consistency(btconfig)
if (
not ApiServer._bt
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('timeframe_detail') != btconfig.get('timeframe_detail')
or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
or lastconfig.get('timerange') != btconfig['timerange']
):
from freqtrade.optimize.backtesting import Backtesting
ApiServer._bt = Backtesting(btconfig)
if ApiServer._bt.timeframe_detail:
ApiServer._bt.load_bt_data_detail()
ApiServer._bt.load_bt_data_detail()
else:
ApiServer._bt.config = btconfig
ApiServer._bt.init_backtest()
# Only reload data if timeframe changed.
if (
not ApiServer._bt_data
or not ApiServer._bt_timerange
or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
or lastconfig.get('protections') != btconfig.get('protections', [])
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('timerange') != btconfig['timerange']
):
lastconfig['timerange'] = btconfig['timerange']
lastconfig['protections'] = btconfig.get('protections', [])
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
lastconfig['timeframe'] = strat.timeframe
ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
lastconfig['timerange'] = btconfig['timerange']
lastconfig['timeframe'] = strat.timeframe
lastconfig['protections'] = btconfig.get('protections', [])
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
ApiServer._bt.results = {}
ApiServer._bt.load_prior_backtest()
ApiServer._bt.abort = False
min_date, max_date = ApiServer._bt.backtest_one_strategy(
strat, ApiServer._bt_data, ApiServer._bt_timerange)
ApiServer._bt.results = generate_backtest_stats(
ApiServer._bt_data, ApiServer._bt.all_results,
min_date=min_date, max_date=max_date)
if (ApiServer._bt.results and
strat.get_strategy_name() in ApiServer._bt.results['strategy']):
# When previous result hash matches - reuse that result and skip backtesting.
logger.info(f'Reusing result of previous backtest for {strat.get_strategy_name()}')
else:
min_date, max_date = ApiServer._bt.backtest_one_strategy(
strat, ApiServer._bt_data, ApiServer._bt_timerange)
ApiServer._bt.results = generate_backtest_stats(
ApiServer._bt_data, ApiServer._bt.all_results,
min_date=min_date, max_date=max_date)
if btconfig.get('export', 'none') == 'trades':
store_backtest_stats(btconfig['exportfilename'], ApiServer._bt.results)
logger.info("Backtest finished.")
except DependencyException as e:
@@ -98,7 +124,7 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
@router.get('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_get_backtest():
def api_get_backtest(ws_mode=Depends(is_webserver_mode)):
"""
Get backtesting result.
Returns Result after backtesting has been ran.
@@ -134,7 +160,7 @@ def api_get_backtest():
@router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_delete_backtest():
def api_delete_backtest(ws_mode=Depends(is_webserver_mode)):
"""Reset backtesting"""
if ApiServer._bgtask_running:
return {
@@ -160,7 +186,7 @@ def api_delete_backtest():
@router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_backtest_abort():
def api_backtest_abort(ws_mode=Depends(is_webserver_mode)):
if not ApiServer._bgtask_running:
return {
"status": "not_running",
@@ -177,3 +203,30 @@ def api_backtest_abort():
"progress": 0,
"status_msg": "Backtest ended",
}
@router.get('/backtest/history', response_model=List[BacktestHistoryEntry], tags=['webserver', 'backtest'])
def api_backtest_history(config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
# Get backtest result history, read from metadata files
return get_backtest_resultlist(config['user_data_dir'] / 'backtest_results')
@router.get('/backtest/history/result', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_backtest_history_result(filename: str, strategy: str, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)):
# Get backtest result history, read from metadata files
fn = config['user_data_dir'] / 'backtest_results' / filename
results: Dict[str, Any] = {
'metadata': {},
'strategy': {},
'strategy_comparison': [],
}
load_and_merge_backtest_result(strategy, fn, results)
return {
"status": "ended",
"running": False,
"step": "",
"progress": 1,
"status_msg": "Historic result",
"backtest_result": results,
}

115
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Union
from pydantic import BaseModel
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.enums import OrderTypeValues, SignalDirection, TradingMode
class Ping(BaseModel):
@@ -37,6 +38,11 @@ class Balance(BaseModel):
used: float
est_stake: float
stake: str
# Starting with 2.x
side: str
leverage: float
is_position: bool
position: float
class Balances(BaseModel):
@@ -63,6 +69,8 @@ class Count(BaseModel):
class PerformanceEntry(BaseModel):
pair: str
profit: float
profit_ratio: float
profit_pct: float
profit_abs: float
count: int
@@ -93,6 +101,7 @@ class Profit(BaseModel):
avg_duration: str
best_pair: str
best_rate: float
best_pair_profit_ratio: float
winning_trades: int
losing_trades: int
@@ -104,8 +113,8 @@ class SellReason(BaseModel):
class Stats(BaseModel):
sell_reasons: Dict[str, SellReason]
durations: Dict[str, Union[str, float]]
exit_reasons: Dict[str, SellReason]
durations: Dict[str, Optional[float]]
class DailyRecord(BaseModel):
@@ -121,10 +130,33 @@ class Daily(BaseModel):
stake_currency: str
class UnfilledTimeout(BaseModel):
entry: Optional[int]
exit: Optional[int]
unit: Optional[str]
exit_timeout_count: Optional[int]
class OrderTypes(BaseModel):
entry: OrderTypeValues
exit: OrderTypeValues
emergency_exit: Optional[OrderTypeValues]
force_exit: Optional[OrderTypeValues]
force_entry: Optional[OrderTypeValues]
stoploss: OrderTypeValues
stoploss_on_exchange: bool
stoploss_on_exchange_interval: Optional[int]
class ShowConfig(BaseModel):
version: str
strategy_version: Optional[str]
api_version: float
dry_run: bool
trading_mode: str
short_allowed: bool
stake_currency: str
stake_amount: Union[float, str]
stake_amount: str
available_capital: Optional[float]
stake_currency_decimals: int
max_open_trades: int
@@ -134,30 +166,54 @@ class ShowConfig(BaseModel):
trailing_stop_positive: Optional[float]
trailing_stop_positive_offset: Optional[float]
trailing_only_offset_is_reached: Optional[bool]
unfilledtimeout: UnfilledTimeout
order_types: Optional[OrderTypes]
use_custom_stoploss: Optional[bool]
timeframe: Optional[str]
timeframe_ms: int
timeframe_min: int
exchange: str
strategy: Optional[str]
forcebuy_enabled: bool
ask_strategy: Dict[str, Any]
bid_strategy: Dict[str, Any]
force_entry_enable: bool
exit_pricing: Dict[str, Any]
entry_pricing: Dict[str, Any]
bot_name: str
state: str
runmode: str
position_adjustment_enable: bool
max_entry_position_adjustment: int
class OrderSchema(BaseModel):
pair: str
order_id: str
status: str
remaining: float
amount: float
safe_price: float
cost: float
filled: float
ft_order_side: str
order_type: str
is_open: bool
order_timestamp: Optional[int]
order_filled_timestamp: Optional[int]
class TradeSchema(BaseModel):
trade_id: int
pair: str
base_currency: str
quote_currency: str
is_open: bool
is_short: bool
exchange: str
amount: float
amount_requested: float
stake_amount: float
strategy: str
buy_tag: Optional[str]
buy_tag: Optional[str] # Deprecated
enter_tag: Optional[str]
timeframe: int
fee_open: Optional[float]
fee_open_cost: Optional[float]
@@ -181,8 +237,9 @@ class TradeSchema(BaseModel):
profit_pct: Optional[float]
profit_abs: Optional[float]
profit_fiat: Optional[float]
sell_reason: Optional[str]
sell_order_status: Optional[str]
sell_reason: Optional[str] # Deprecated
exit_reason: Optional[str]
exit_order_status: Optional[str]
stop_loss_abs: Optional[float]
stop_loss_ratio: Optional[float]
stop_loss_pct: Optional[float]
@@ -195,6 +252,12 @@ class TradeSchema(BaseModel):
min_rate: Optional[float]
max_rate: Optional[float]
open_order_id: Optional[str]
orders: List[OrderSchema]
leverage: Optional[float]
interest_rate: Optional[float]
funding_fees: Optional[float]
trading_mode: Optional[TradingMode]
class OpenTradeSchema(TradeSchema):
@@ -216,7 +279,7 @@ class TradeResponse(BaseModel):
total_trades: int
class ForceBuyResponse(BaseModel):
class ForceEnterResponse(BaseModel):
__root__: Union[TradeSchema, StatusMsg]
@@ -246,13 +309,18 @@ class Logs(BaseModel):
logs: List[List]
class ForceBuyPayload(BaseModel):
class ForceEnterPayload(BaseModel):
pair: str
side: SignalDirection = SignalDirection.LONG
price: Optional[float]
ordertype: Optional[OrderTypeValues]
stakeamount: Optional[float]
entry_tag: Optional[str]
class ForceSellPayload(BaseModel):
class ForceExitPayload(BaseModel):
tradeid: str
ordertype: Optional[OrderTypeValues]
class BlacklistPayload(BaseModel):
@@ -314,6 +382,10 @@ class PairHistory(BaseModel):
length: int
buy_signals: int
sell_signals: int
enter_long_signals: int
exit_long_signals: int
enter_short_signals: int
exit_short_signals: int
last_analyzed: datetime
last_analyzed_ts: int
data_start_ts: int
@@ -333,7 +405,7 @@ class BacktestRequest(BaseModel):
timeframe_detail: Optional[str]
timerange: Optional[str]
max_open_trades: Optional[int]
stake_amount: Optional[Union[float, str]]
stake_amount: Optional[str]
enable_protections: bool
dry_run_wallet: Optional[float]
@@ -347,3 +419,20 @@ class BacktestResponse(BaseModel):
trade_count: Optional[float]
# TODO: Properly type backtestresult...
backtest_result: Optional[Dict[str, Any]]
class BacktestHistoryEntry(BaseModel):
filename: str
strategy: str
run_id: str
backtest_start_time: int
class SysInfo(BaseModel):
cpu_pct: List[float]
ram_pct: float
class Health(BaseModel):
last_process: datetime
last_process_ts: int

99
freqtrade/rpc/api_server/api_v1.py
View File

@@ -3,28 +3,41 @@ from copy import deepcopy
from pathlib import Path
from typing import List, Optional
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, Query
from fastapi.exceptions import HTTPException
from freqtrade import __version__
from freqtrade.constants import USERPATH_STRATEGIES
from freqtrade.data.history import get_datahandler
from freqtrade.enums import CandleType, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.rpc import RPC
from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload,
BlacklistResponse, Count, Daily,
DeleteLockRequest, DeleteTrade, ForceBuyPayload,
ForceBuyResponse, ForceSellPayload, Locks, Logs,
OpenTradeSchema, PairHistory, PerformanceEntry,
Ping, PlotConfig, Profit, ResultMsg, ShowConfig,
Stats, StatusMsg, StrategyListResponse,
StrategyResponse, Version, WhitelistResponse)
from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
DeleteLockRequest, DeleteTrade, ForceEnterPayload,
ForceEnterResponse, ForceExitPayload, Health,
Locks, Logs, OpenTradeSchema, PairHistory,
PerformanceEntry, Ping, PlotConfig, Profit,
ResultMsg, ShowConfig, Stats, StatusMsg,
StrategyListResponse, StrategyResponse, SysInfo,
Version, WhitelistResponse)
from freqtrade.rpc.api_server.deps import get_config, get_exchange, get_rpc, get_rpc_optional
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# API version
# Pre-1.1, no version was provided
# Version increments should happen in "small" steps (1.1, 1.12, ...) unless big changes happen.
# 1.11: forcebuy and forcesell accept ordertype
# 1.12: add blacklist delete endpoint
# 1.13: forcebuy supports stake_amount
# versions 2.xx -> futures/short branch
# 2.14: Add entry/exit orders to trade response
# 2.15: Add backtest history endpoints
API_VERSION = 2.15
# Public API, requires no auth.
router_public = APIRouter()
# Private API, protected by authentication
@@ -114,24 +127,40 @@ def edge(rpc: RPC = Depends(get_rpc)):
@router.get('/show_config', response_model=ShowConfig, tags=['info'])
def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(get_config)):
state = ''
strategy_version = None
if rpc:
state = rpc._freqtrade.state
return RPC._rpc_show_config(config, state)
strategy_version = rpc._freqtrade.strategy.version()
resp = RPC._rpc_show_config(config, state, strategy_version)
resp['api_version'] = API_VERSION
return resp
@router.post('/forcebuy', response_model=ForceBuyResponse, tags=['trading'])
def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)):
trade = rpc._rpc_forcebuy(payload.pair, payload.price)
# /forcebuy is deprecated with short addition. use /forceentry instead
@router.post('/forceenter', response_model=ForceEnterResponse, tags=['trading'])
@router.post('/forcebuy', response_model=ForceEnterResponse, tags=['trading'])
def force_entry(payload: ForceEnterPayload, rpc: RPC = Depends(get_rpc)):
ordertype = payload.ordertype.value if payload.ordertype else None
stake_amount = payload.stakeamount if payload.stakeamount else None
entry_tag = payload.entry_tag if payload.entry_tag else 'force_entry'
trade = rpc._rpc_force_entry(payload.pair, payload.price, order_side=payload.side,
order_type=ordertype, stake_amount=stake_amount,
enter_tag=entry_tag)
if trade:
return ForceBuyResponse.parse_obj(trade.to_json())
return ForceEnterResponse.parse_obj(trade.to_json())
else:
return ForceBuyResponse.parse_obj({"status": f"Error buying pair {payload.pair}."})
return ForceEnterResponse.parse_obj(
{"status": f"Error entering {payload.side} trade for pair {payload.pair}."})
# /forcesell is deprecated with short addition. use /forceexit instead
@router.post('/forceexit', response_model=ResultMsg, tags=['trading'])
@router.post('/forcesell', response_model=ResultMsg, tags=['trading'])
def forcesell(payload: ForceSellPayload, rpc: RPC = Depends(get_rpc)):
return rpc._rpc_forcesell(payload.tradeid)
def forceexit(payload: ForceExitPayload, rpc: RPC = Depends(get_rpc)):
ordertype = payload.ordertype.value if payload.ordertype else None
return rpc._rpc_force_exit(payload.tradeid, ordertype)
@router.get('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist'])
@@ -144,6 +173,13 @@ def blacklist_post(payload: BlacklistPayload, rpc: RPC = Depends(get_rpc)):
return rpc._rpc_blacklist(payload.blacklist)
@router.delete('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist'])
def blacklist_delete(pairs_to_delete: List[str] = Query([]), rpc: RPC = Depends(get_rpc)):
"""Provide a list of pairs to delete from the blacklist"""
return rpc._rpc_blacklist_delete(pairs_to_delete)
@router.get('/whitelist', response_model=WhitelistResponse, tags=['info', 'pairlist'])
def whitelist(rpc: RPC = Depends(get_rpc)):
return rpc._rpc_whitelist()
@@ -190,18 +226,21 @@ def reload_config(rpc: RPC = Depends(get_rpc)):
@router.get('/pair_candles', response_model=PairHistory, tags=['candle data'])
def pair_candles(pair: str, timeframe: str, limit: Optional[int], rpc: RPC = Depends(get_rpc)):
def pair_candles(
pair: str, timeframe: str, limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)):
return rpc._rpc_analysed_dataframe(pair, timeframe, limit)
@router.get('/pair_history', response_model=PairHistory, tags=['candle data'])
def pair_history(pair: str, timeframe: str, timerange: str, strategy: str,
config=Depends(get_config)):
config=Depends(get_config), exchange=Depends(get_exchange)):
# The initial call to this endpoint can be slow, as it may need to initialize
# the exchange class.
config = deepcopy(config)
config.update({
'strategy': strategy,
})
return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange)
return RPC._rpc_analysed_history_full(config, pair, timeframe, timerange, exchange)
@router.get('/plot_config', response_model=PlotConfig, tags=['candle data'])
@@ -239,16 +278,22 @@ def get_strategy(strategy: str, config=Depends(get_config)):
@router.get('/available_pairs', response_model=AvailablePairs, tags=['candle data'])
def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None,
config=Depends(get_config)):
candletype: Optional[CandleType] = None, config=Depends(get_config)):
dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None))
pair_interval = dh.ohlcv_get_available_data(config['datadir'])
trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
pair_interval = dh.ohlcv_get_available_data(config['datadir'], trading_mode)
if timeframe:
pair_interval = [pair for pair in pair_interval if pair[1] == timeframe]
if stake_currency:
pair_interval = [pair for pair in pair_interval if pair[0].endswith(stake_currency)]
if candletype:
pair_interval = [pair for pair in pair_interval if pair[2] == candletype]
else:
candle_type = CandleType.get_default(trading_mode)
pair_interval = [pair for pair in pair_interval if pair[2] == candle_type]
pair_interval = sorted(pair_interval, key=lambda x: x[0])
pairs = list({x[0] for x in pair_interval})
@@ -259,3 +304,13 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option
'pair_interval': pair_interval,
}
return result
@router.get('/sysinfo', response_model=SysInfo, tags=['info'])
def sysinfo():
return RPC._rpc_sysinfo()
@router.get('/health', response_model=Health, tags=['info'])
def health(rpc: RPC = Depends(get_rpc)):
return rpc._health()

26
freqtrade/rpc/api_server/deps.py
View File

@@ -1,5 +1,9 @@
from typing import Any, Dict, Optional
from typing import Any, Dict, Iterator, Optional
from fastapi import Depends
from freqtrade.enums import RunMode
from freqtrade.persistence import Trade
from freqtrade.rpc.rpc import RPC, RPCException
from .webserver import ApiServer
@@ -11,10 +15,12 @@ def get_rpc_optional() -> Optional[RPC]:
return None
def get_rpc() -> Optional[RPC]:
def get_rpc() -> Optional[Iterator[RPC]]:
_rpc = get_rpc_optional()
if _rpc:
return _rpc
Trade.query.session.rollback()
yield _rpc
Trade.query.session.rollback()
else:
raise RPCException('Bot is not in the correct state')
@@ -25,3 +31,17 @@ def get_config() -> Dict[str, Any]:
def get_api_config() -> Dict[str, Any]:
return ApiServer._config['api_server']
def get_exchange(config=Depends(get_config)):
if not ApiServer._exchange:
from freqtrade.resolvers import ExchangeResolver
ApiServer._exchange = ExchangeResolver.load_exchange(
config['exchange']['name'], config)
return ApiServer._exchange
def is_webserver_mode(config=Depends(get_config)):
if config['runmode'] != RunMode.WEBSERVER:
raise RPCException('Bot is not in the correct state')
return None

2
freqtrade/rpc/api_server/uvicorn_threaded.py
View File

@@ -47,7 +47,7 @@ class UvicornServer(uvicorn.Server):
else:
asyncio.set_event_loop(uvloop.new_event_loop())
try:
loop = asyncio.get_event_loop()
loop = asyncio.get_running_loop()
except RuntimeError:
# When running in a thread, we'll not have an eventloop yet.
loop = asyncio.new_event_loop()

2
freqtrade/rpc/api_server/webserver.py
View File

@@ -41,6 +41,8 @@ class ApiServer(RPCHandler):
_has_rpc: bool = False
_bgtask_running: bool = False
_config: Dict[str, Any] = {}
# Exchange - only available in webserver mode.
_exchange = None
def __new__(cls, *args, **kwargs):
"""

24
freqtrade/rpc/fiat_convert.py
View File

@@ -7,7 +7,7 @@ import datetime
import logging
from typing import Dict, List
from cachetools.ttl import TTLCache
from cachetools import TTLCache
from pycoingecko import CoinGeckoAPI
from requests.exceptions import RequestException
@@ -17,6 +17,16 @@ from freqtrade.constants import SUPPORTED_FIAT
logger = logging.getLogger(__name__)
# Manually map symbol to ID for some common coins
# with duplicate coingecko entries
coingecko_mapping = {
'eth': 'ethereum',
'bnb': 'binancecoin',
'sol': 'solana',
'usdt': 'tether',
}
class CryptoToFiatConverter:
"""
Main class to initiate Crypto to FIAT.
@@ -53,7 +63,7 @@ class CryptoToFiatConverter:
except RequestException as request_exception:
if "429" in str(request_exception):
logger.warning(
"Too many requests for Coingecko API, backing off and trying again later.")
"Too many requests for CoinGecko API, backing off and trying again later.")
# Set backoff timestamp to 60 seconds in the future
self._backoff = datetime.datetime.now().timestamp() + 60
return
@@ -76,13 +86,17 @@ class CryptoToFiatConverter:
return None
else:
return None
found = [x for x in self._coinlistings if x['symbol'] == crypto_symbol]
found = [x for x in self._coinlistings if x['symbol'].lower() == crypto_symbol]
if crypto_symbol in coingecko_mapping.keys():
found = [x for x in self._coinlistings if x['id'] == coingecko_mapping[crypto_symbol]]
if len(found) == 1:
return found[0]['id']
if len(found) > 0:
# Wrong!
logger.warning(f"Found multiple mappings in goingekko for {crypto_symbol}.")
logger.warning(f"Found multiple mappings in CoinGecko for {crypto_symbol}.")
return None
def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float:
@@ -146,7 +160,7 @@ class CryptoToFiatConverter:
def _find_price(self, crypto_symbol: str, fiat_symbol: str) -> float:
"""
Call CoinGekko API to retrieve the price in the FIAT
Call CoinGecko API to retrieve the price in the FIAT
:param crypto_symbol: Crypto-currency you want to convert (e.g btc)
:param fiat_symbol: FIAT currency you want to convert to (e.g usd)
:return: float, price of the crypto-currency in Fiat

398
freqtrade/rpc/rpc.py
View File

@@ -8,13 +8,18 @@ from math import isnan
from typing import Any, Dict, List, Optional, Tuple, Union
import arrow
import psutil
from dateutil.relativedelta import relativedelta
from dateutil.tz import tzlocal
from numpy import NAN, inf, int64, mean
from pandas import DataFrame
from pandas import DataFrame, NaT
from freqtrade import __version__
from freqtrade.configuration.timerange import TimeRange
from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT
from freqtrade.data.history import load_data
from freqtrade.enums import SellType, State
from freqtrade.enums import (CandleType, ExitCheckTuple, ExitType, SignalDirection, State,
TradingMode)
from freqtrade.exceptions import ExchangeError, PricingError
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
from freqtrade.loggers import bufferHandler
@@ -23,7 +28,7 @@ from freqtrade.persistence import PairLocks, Trade
from freqtrade.persistence.models import PairLock
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
from freqtrade.strategy.interface import SellCheckTuple
from freqtrade.wallets import PositionWallet, Wallet
logger = logging.getLogger(__name__)
@@ -95,17 +100,22 @@ class RPC:
self._fiat_converter = CryptoToFiatConverter()
@staticmethod
def _rpc_show_config(config, botstate: Union[State, str]) -> Dict[str, Any]:
def _rpc_show_config(config, botstate: Union[State, str],
strategy_version: Optional[str] = None) -> Dict[str, Any]:
"""
Return a dict of config options.
Explicitly does NOT return the full config to avoid leakage of sensitive
information via rpc.
"""
val = {
'version': __version__,
'strategy_version': strategy_version,
'dry_run': config['dry_run'],
'trading_mode': config.get('trading_mode', 'spot'),
'short_allowed': config.get('trading_mode', 'spot') != 'spot',
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'stake_amount': config['stake_amount'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'stake_amount': str(config['stake_amount']),
'available_capital': config.get('available_capital'),
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
@@ -115,7 +125,9 @@ class RPC:
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset'),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'),
'unfilledtimeout': config.get('unfilledtimeout'),
'use_custom_stoploss': config.get('use_custom_stoploss'),
'order_types': config.get('order_types'),
'bot_name': config.get('bot_name', 'freqtrade'),
'timeframe': config.get('timeframe'),
'timeframe_ms': timeframe_to_msecs(config['timeframe']
@@ -124,11 +136,16 @@ class RPC:
) if 'timeframe' in config else 0,
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),
'ask_strategy': config.get('ask_strategy', {}),
'bid_strategy': config.get('bid_strategy', {}),
'force_entry_enable': config.get('force_entry_enable', False),
'exit_pricing': config.get('exit_pricing', {}),
'entry_pricing': config.get('entry_pricing', {}),
'state': str(botstate),
'runmode': config['runmode'].value
'runmode': config['runmode'].value,
'position_adjustment_enable': config.get('position_adjustment_enable', False),
'max_entry_position_adjustment': (
config.get('max_entry_position_adjustment', -1)
if config.get('max_entry_position_adjustment') != float('inf')
else -1)
}
return val
@@ -139,7 +156,7 @@ class RPC:
"""
# Fetch open trades
if trade_ids:
trades = Trade.get_trades(trade_filter=Trade.id.in_(trade_ids)).all()
trades: List[Trade] = Trade.get_trades(trade_filter=Trade.id.in_(trade_ids)).all()
else:
trades = Trade.get_open_trades()
@@ -155,7 +172,7 @@ class RPC:
if trade.is_open:
try:
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
trade.pair, side='exit', is_short=trade.is_short, refresh=False)
except (ExchangeError, PricingError):
current_rate = NAN
else:
@@ -180,7 +197,6 @@ class RPC:
trade_dict = trade.to_json()
trade_dict.update(dict(
base_currency=self._freqtrade.config['stake_currency'],
close_profit=trade.close_profit if trade.close_profit is not None else None,
current_rate=current_rate,
current_profit=current_profit, # Deprecated
@@ -205,7 +221,8 @@ class RPC:
def _rpc_status_table(self, stake_currency: str,
fiat_display_currency: str) -> Tuple[List, List, float]:
trades = Trade.get_open_trades()
trades: List[Trade] = Trade.get_open_trades()
nonspot = self._config.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT
if not trades:
raise RPCException('no active trade')
else:
@@ -215,12 +232,12 @@ class RPC:
# calculate profit and send message to user
try:
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
trade.pair, side='exit', is_short=trade.is_short, refresh=False)
except (PricingError, ExchangeError):
current_rate = NAN
trade_percent = (100 * trade.calc_profit_ratio(current_rate))
trade_profit = trade.calc_profit(current_rate)
profit_str = f'{trade_percent:.2f}%'
profit_str = f'{trade.calc_profit_ratio(current_rate):.2%}'
direction_str = ('S' if trade.is_short else 'L') if nonspot else ''
if self._fiat_converter:
fiat_profit = self._fiat_converter.convert_amount(
trade_profit,
@@ -231,25 +248,38 @@ class RPC:
profit_str += f" ({fiat_profit:.2f})"
fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \
else fiat_profit_sum + fiat_profit
trades_list.append([
trade.id,
detail_trade = [
f'{trade.id} {direction_str}',
trade.pair + ('*' if (trade.open_order_id is not None
and trade.close_rate_requested is None) else '')
+ ('**' if (trade.close_rate_requested is not None) else ''),
+ ('**' if (trade.close_rate_requested is not None) else ''),
shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)),
profit_str
])
]
if self._config.get('position_adjustment_enable', False):
max_entry_str = ''
if self._config.get('max_entry_position_adjustment', -1) > 0:
max_entry_str = f"/{self._config['max_entry_position_adjustment'] + 1}"
filled_entries = trade.nr_of_successful_entries
detail_trade.append(f"{filled_entries}{max_entry_str}")
trades_list.append(detail_trade)
profitcol = "Profit"
if self._fiat_converter:
profitcol += " (" + fiat_display_currency + ")"
columns = ['ID', 'Pair', 'Since', profitcol]
columns = [
'ID L/S' if nonspot else 'ID',
'Pair',
'Since',
profitcol]
if self._config.get('position_adjustment_enable', False):
columns.append('# Entries')
return trades_list, columns, fiat_profit_sum
def _rpc_daily_profit(
self, timescale: int,
stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
today = datetime.utcnow().date()
today = datetime.now(timezone.utc).date()
profit_days: Dict[date, Dict] = {}
if not (isinstance(timescale, int) and timescale > 0):
@@ -288,6 +318,91 @@ class RPC:
'data': data
}
def _rpc_weekly_profit(
self, timescale: int,
stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
today = datetime.now(timezone.utc).date()
first_iso_day_of_week = today - timedelta(days=today.weekday()) # Monday
profit_weeks: Dict[date, Dict] = {}
if not (isinstance(timescale, int) and timescale > 0):
raise RPCException('timescale must be an integer greater than 0')
for week in range(0, timescale):
profitweek = first_iso_day_of_week - timedelta(weeks=week)
trades = Trade.get_trades(trade_filter=[
Trade.is_open.is_(False),
Trade.close_date >= profitweek,
Trade.close_date < (profitweek + timedelta(weeks=1))
]).order_by(Trade.close_date).all()
curweekprofit = sum(
trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None)
profit_weeks[profitweek] = {
'amount': curweekprofit,
'trades': len(trades)
}
data = [
{
'date': key,
'abs_profit': value["amount"],
'fiat_value': self._fiat_converter.convert_amount(
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
'trade_count': value["trades"],
}
for key, value in profit_weeks.items()
]
return {
'stake_currency': stake_currency,
'fiat_display_currency': fiat_display_currency,
'data': data
}
def _rpc_monthly_profit(
self, timescale: int,
stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]:
first_day_of_month = datetime.now(timezone.utc).date().replace(day=1)
profit_months: Dict[date, Dict] = {}
if not (isinstance(timescale, int) and timescale > 0):
raise RPCException('timescale must be an integer greater than 0')
for month in range(0, timescale):
profitmonth = first_day_of_month - relativedelta(months=month)
trades = Trade.get_trades(trade_filter=[
Trade.is_open.is_(False),
Trade.close_date >= profitmonth,
Trade.close_date < (profitmonth + relativedelta(months=1))
]).order_by(Trade.close_date).all()
curmonthprofit = sum(
trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None)
profit_months[profitmonth] = {
'amount': curmonthprofit,
'trades': len(trades)
}
data = [
{
'date': f"{key.year}-{key.month:02d}",
'abs_profit': value["amount"],
'fiat_value': self._fiat_converter.convert_amount(
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
'trade_count': value["trades"],
}
for key, value in profit_months.items()
]
return {
'stake_currency': stake_currency,
'fiat_display_currency': fiat_display_currency,
'data': data
}
def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict:
""" Returns the X last trades """
order_by = Trade.id if order_by_id else Trade.close_date.desc()
@@ -317,13 +432,13 @@ class RPC:
return 'losses'
else:
return 'draws'
trades = trades = Trade.get_trades([Trade.is_open.is_(False)])
trades: List[Trade] = Trade.get_trades([Trade.is_open.is_(False)])
# Sell reason
sell_reasons = {}
exit_reasons = {}
for trade in trades:
if trade.sell_reason not in sell_reasons:
sell_reasons[trade.sell_reason] = {'wins': 0, 'losses': 0, 'draws': 0}
sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1
if trade.exit_reason not in exit_reasons:
exit_reasons[trade.exit_reason] = {'wins': 0, 'losses': 0, 'draws': 0}
exit_reasons[trade.exit_reason][trade_win_loss(trade)] += 1
# Duration
dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []}
@@ -332,12 +447,12 @@ class RPC:
trade_dur = (trade.close_date - trade.open_date).total_seconds()
dur[trade_win_loss(trade)].append(trade_dur)
wins_dur = sum(dur['wins']) / len(dur['wins']) if len(dur['wins']) > 0 else 'N/A'
draws_dur = sum(dur['draws']) / len(dur['draws']) if len(dur['draws']) > 0 else 'N/A'
losses_dur = sum(dur['losses']) / len(dur['losses']) if len(dur['losses']) > 0 else 'N/A'
wins_dur = sum(dur['wins']) / len(dur['wins']) if len(dur['wins']) > 0 else None
draws_dur = sum(dur['draws']) / len(dur['draws']) if len(dur['draws']) > 0 else None
losses_dur = sum(dur['losses']) / len(dur['losses']) if len(dur['losses']) > 0 else None
durations = {'wins': wins_dur, 'draws': draws_dur, 'losses': losses_dur}
return {'sell_reasons': sell_reasons, 'durations': durations}
return {'exit_reasons': exit_reasons, 'durations': durations}
def _rpc_trade_statistics(
self, stake_currency: str, fiat_display_currency: str,
@@ -345,7 +460,7 @@ class RPC:
""" Returns cumulative profit statistics """
trade_filter = ((Trade.is_open.is_(False) & (Trade.close_date >= start_date)) |
Trade.is_open.is_(True))
trades = Trade.get_trades(trade_filter).order_by(Trade.id).all()
trades: List[Trade] = Trade.get_trades(trade_filter).order_by(Trade.id).all()
profit_all_coin = []
profit_all_ratio = []
@@ -375,7 +490,7 @@ class RPC:
# Get current rate
try:
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
trade.pair, side='exit', is_short=trade.is_short, refresh=False)
except (PricingError, ExchangeError):
current_rate = NAN
profit_ratio = trade.calc_profit_ratio(rate=current_rate)
@@ -443,14 +558,15 @@ class RPC:
'latest_trade_timestamp': int(last_date.timestamp() * 1000) if last_date else 0,
'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0],
'best_pair': best_pair[0] if best_pair else '',
'best_rate': round(best_pair[1] * 100, 2) if best_pair else 0,
'best_rate': round(best_pair[1] * 100, 2) if best_pair else 0, # Deprecated
'best_pair_profit_ratio': best_pair[1] if best_pair else 0,
'winning_trades': winning_trades,
'losing_trades': losing_trades,
}
def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict:
""" Returns current account balance per crypto """
output = []
currencies = []
total = 0.0
try:
tickers = self._freqtrade.exchange.get_tickers(cached=True)
@@ -461,7 +577,8 @@ class RPC:
starting_capital = self._freqtrade.wallets.get_starting_balance()
starting_cap_fiat = self._fiat_converter.convert_amount(
starting_capital, stake_currency, fiat_display_currency) if self._fiat_converter else 0
coin: str
balance: Wallet
for coin, balance in self._freqtrade.wallets.get_all_balances().items():
if not balance.total:
continue
@@ -470,10 +587,13 @@ class RPC:
if coin == stake_currency:
rate = 1.0
est_stake = balance.total
if self._config.get('trading_mode', TradingMode.SPOT) != TradingMode.SPOT:
# in Futures, "total" includes the locked stake, and therefore all positions
est_stake = balance.free
else:
try:
pair = self._freqtrade.exchange.get_valid_pair_combination(coin, stake_currency)
rate = tickers.get(pair, {}).get('bid', None)
rate = tickers.get(pair, {}).get('last', None)
if rate:
if pair.startswith(stake_currency) and not pair.endswith(stake_currency):
rate = 1.0 / rate
@@ -482,29 +602,47 @@ class RPC:
logger.warning(f" Could not get rate for pair {coin}.")
continue
total = total + (est_stake or 0)
output.append({
currencies.append({
'currency': coin,
# TODO: The below can be simplified if we don't assign None to values.
'free': balance.free if balance.free is not None else 0,
'balance': balance.total if balance.total is not None else 0,
'used': balance.used if balance.used is not None else 0,
'est_stake': est_stake or 0,
'stake': stake_currency,
'side': 'long',
'leverage': 1,
'position': 0,
'is_position': False,
})
symbol: str
position: PositionWallet
for symbol, position in self._freqtrade.wallets.get_all_positions().items():
total += position.collateral
currencies.append({
'currency': symbol,
'free': 0,
'balance': 0,
'used': 0,
'position': position.position,
'est_stake': position.collateral,
'stake': stake_currency,
'leverage': position.leverage,
'side': position.side,
'is_position': True
})
if total == 0.0:
if self._freqtrade.config['dry_run']:
raise RPCException('Running in Dry Run, balances are not available.')
else:
raise RPCException('All balances are zero.')
value = self._fiat_converter.convert_amount(
total, stake_currency, fiat_display_currency) if self._fiat_converter else 0
trade_count = len(Trade.get_trades_proxy())
starting_capital_ratio = 0.0
starting_capital_ratio = (total / starting_capital) - 1 if starting_capital else 0.0
starting_cap_fiat_ratio = (value / starting_cap_fiat) - 1 if starting_cap_fiat else 0.0
return {
'currencies': output,
'currencies': currencies,
'total': total,
'symbol': fiat_display_currency,
'value': value,
@@ -515,6 +653,7 @@ class RPC:
'starting_capital_fiat': starting_cap_fiat,
'starting_capital_fiat_ratio': starting_cap_fiat_ratio,
'starting_capital_fiat_pct': round(starting_cap_fiat_ratio * 100, 2),
'trade_count': trade_count,
'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else ''
}
@@ -549,31 +688,35 @@ class RPC:
return {'status': 'No more buy will occur from now. Run /reload_config to reset.'}
def _rpc_forcesell(self, trade_id: str) -> Dict[str, str]:
def _rpc_force_exit(self, trade_id: str, ordertype: Optional[str] = None) -> Dict[str, str]:
"""
Handler for forcesell <id>.
Handler for forceexit <id>.
Sells the given trade at current price
"""
def _exec_forcesell(trade: Trade) -> None:
def _exec_force_exit(trade: Trade) -> None:
# Check if there is there is an open order
fully_canceled = False
if trade.open_order_id:
order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
if order['side'] == 'buy':
if order['side'] == trade.entry_side:
fully_canceled = self._freqtrade.handle_cancel_enter(
trade, order, CANCEL_REASON['FORCE_SELL'])
trade, order, CANCEL_REASON['FORCE_EXIT'])
if order['side'] == 'sell':
if order['side'] == trade.exit_side:
# Cancel order - so it is placed anew with a fresh price.
self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_SELL'])
self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_EXIT'])
if not fully_canceled:
# Get current rate and execute sell
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
self._freqtrade.execute_trade_exit(trade, current_rate, sell_reason)
trade.pair, side='exit', is_short=trade.is_short, refresh=True)
exit_check = ExitCheckTuple(exit_type=ExitType.FORCE_EXIT)
order_type = ordertype or self._freqtrade.strategy.order_types.get(
"force_exit", self._freqtrade.strategy.order_types["exit"])
self._freqtrade.execute_trade_exit(
trade, current_rate, exit_check, ordertype=order_type)
# ---- EOF def _exec_forcesell ----
if self._freqtrade.state != State.RUNNING:
@@ -583,7 +726,7 @@ class RPC:
if trade_id == 'all':
# Execute sell for all open orders
for trade in Trade.get_open_trades():
_exec_forcesell(trade)
_exec_force_exit(trade)
Trade.commit()
self._freqtrade.wallets.update()
return {'result': 'Created sell orders for all open trades.'}
@@ -593,26 +736,33 @@ class RPC:
trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ]
).first()
if not trade:
logger.warning('forcesell: Invalid argument received')
logger.warning('force_exit: Invalid argument received')
raise RPCException('invalid argument')
_exec_forcesell(trade)
_exec_force_exit(trade)
Trade.commit()
self._freqtrade.wallets.update()
return {'result': f'Created sell order for trade {trade_id}.'}
def _rpc_forcebuy(self, pair: str, price: Optional[float]) -> Optional[Trade]:
def _rpc_force_entry(self, pair: str, price: Optional[float], *,
order_type: Optional[str] = None,
order_side: SignalDirection = SignalDirection.LONG,
stake_amount: Optional[float] = None,
enter_tag: Optional[str] = 'force_entry') -> Optional[Trade]:
"""
Handler for forcebuy <asset> <price>
Buys a pair trade at the given or current price
"""
if not self._freqtrade.config.get('forcebuy_enable', False):
raise RPCException('Forcebuy not enabled.')
if not self._freqtrade.config.get('force_entry_enable', False):
raise RPCException('Force_entry not enabled.')
if self._freqtrade.state != State.RUNNING:
raise RPCException('trader is not running')
if order_side == SignalDirection.SHORT and self._freqtrade.trading_mode == TradingMode.SPOT:
raise RPCException("Can't go short on Spot markets.")
# Check if pair quote currency equals to the stake currency.
stake_currency = self._freqtrade.config.get('stake_currency')
if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency:
@@ -621,20 +771,31 @@ class RPC:
# check if valid pair
# check if pair already has an open pair
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
trade: Trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
is_short = (order_side == SignalDirection.SHORT)
if trade:
raise RPCException(f'position for {pair} already open - id: {trade.id}')
is_short = trade.is_short
if not self._freqtrade.strategy.position_adjustment_enable:
raise RPCException(f'position for {pair} already open - id: {trade.id}')
# gen stake amount
stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
if not stake_amount:
# gen stake amount
stake_amount = self._freqtrade.wallets.get_trade_stake_amount(pair)
# execute buy
if self._freqtrade.execute_entry(pair, stakeamount, price, forcebuy=True):
if not order_type:
order_type = self._freqtrade.strategy.order_types.get(
'force_entry', self._freqtrade.strategy.order_types['entry'])
if self._freqtrade.execute_entry(pair, stake_amount, price,
ordertype=order_type, trade=trade,
is_short=is_short,
enter_tag=enter_tag,
):
Trade.commit()
trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first()
return trade
else:
return None
raise RPCException(f'Failed to enter position for {pair}.')
def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]:
"""
@@ -681,10 +842,32 @@ class RPC:
Shows a performance statistic from finished trades
"""
pair_rates = Trade.get_overall_performance()
# Round and convert to %
[x.update({'profit': round(x['profit'] * 100, 2)}) for x in pair_rates]
return pair_rates
def _rpc_enter_tag_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]:
"""
Handler for buy tag performance.
Shows a performance statistic from finished trades
"""
return Trade.get_enter_tag_performance(pair)
def _rpc_exit_reason_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]:
"""
Handler for exit reason performance.
Shows a performance statistic from finished trades
"""
return Trade.get_exit_reason_performance(pair)
def _rpc_mix_tag_performance(self, pair: Optional[str]) -> List[Dict[str, Any]]:
"""
Handler for mix tag (enter_tag + exit_reason) performance.
Shows a performance statistic from finished trades
"""
mix_tags = Trade.get_mix_tag_performance(pair)
return mix_tags
def _rpc_count(self) -> Dict[str, float]:
""" Returns the number of trades running """
if self._freqtrade.state != State.RUNNING:
@@ -733,6 +916,20 @@ class RPC:
}
return res
def _rpc_blacklist_delete(self, delete: List[str]) -> Dict:
""" Removes pairs from currently active blacklist """
errors = {}
for pair in delete:
if pair in self._freqtrade.pairlists.blacklist:
self._freqtrade.pairlists.blacklist.remove(pair)
else:
errors[pair] = {
'error_msg': f"Pair {pair} is not in the current blacklist."
}
resp = self._rpc_blacklist()
resp['errors'] = errors
return resp
def _rpc_blacklist(self, add: List[str] = None) -> Dict:
""" Returns the currently active blacklist"""
errors = {}
@@ -787,22 +984,31 @@ class RPC:
def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame,
last_analyzed: datetime) -> Dict[str, Any]:
has_content = len(dataframe) != 0
buy_signals = 0
sell_signals = 0
signals = {
'enter_long': 0,
'exit_long': 0,
'enter_short': 0,
'exit_short': 0,
}
if has_content:
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
# Move open to separate column when signal for easy plotting
if 'buy' in dataframe.columns:
buy_mask = (dataframe['buy'] == 1)
buy_signals = int(buy_mask.sum())
dataframe.loc[buy_mask, '_buy_signal_open'] = dataframe.loc[buy_mask, 'open']
if 'sell' in dataframe.columns:
sell_mask = (dataframe['sell'] == 1)
sell_signals = int(sell_mask.sum())
dataframe.loc[sell_mask, '_sell_signal_open'] = dataframe.loc[sell_mask, 'open']
dataframe = dataframe.replace([inf, -inf], NAN)
dataframe = dataframe.replace({NAN: None})
# Move signal close to separate column when signal for easy plotting
for sig_type in signals.keys():
if sig_type in dataframe.columns:
mask = (dataframe[sig_type] == 1)
signals[sig_type] = int(mask.sum())
dataframe.loc[mask, f'_{sig_type}_signal_close'] = dataframe.loc[mask, 'close']
# band-aid until this is fixed:
# https://github.com/pandas-dev/pandas/issues/45836
datetime_types = ['datetime', 'datetime64', 'datetime64[ns, UTC]']
date_columns = dataframe.select_dtypes(include=datetime_types)
for date_column in date_columns:
# replace NaT with `None`
dataframe[date_column] = dataframe[date_column].astype(object).replace({NaT: None})
dataframe = dataframe.replace({inf: None, -inf: None, NAN: None})
res = {
'pair': pair,
@@ -812,8 +1018,12 @@ class RPC:
'columns': list(dataframe.columns),
'data': dataframe.values.tolist(),
'length': len(dataframe),
'buy_signals': buy_signals,
'sell_signals': sell_signals,
'buy_signals': signals['enter_long'], # Deprecated
'sell_signals': signals['exit_long'], # Deprecated
'enter_long_signals': signals['enter_long'],
'exit_long_signals': signals['exit_long'],
'enter_short_signals': signals['enter_short'],
'exit_short_signals': signals['exit_short'],
'last_analyzed': last_analyzed,
'last_analyzed_ts': int(last_analyzed.timestamp()),
'data_start': '',
@@ -843,7 +1053,7 @@ class RPC:
@staticmethod
def _rpc_analysed_history_full(config, pair: str, timeframe: str,
timerange: str) -> Dict[str, Any]:
timerange: str, exchange) -> Dict[str, Any]:
timerange_parsed = TimeRange.parse_timerange(timerange)
_data = load_data(
@@ -852,13 +1062,14 @@ class RPC:
timeframe=timeframe,
timerange=timerange_parsed,
data_format=config.get('dataformat_ohlcv', 'json'),
candle_type=config.get('candle_type_def', CandleType.SPOT)
)
if pair not in _data:
raise RPCException(f"No data for {pair}, {timeframe} in {timerange} found.")
from freqtrade.data.dataprovider import DataProvider
from freqtrade.resolvers.strategy_resolver import StrategyResolver
strategy = StrategyResolver.load_strategy(config)
strategy.dp = DataProvider(config, exchange=None, pairlists=None)
strategy.dp = DataProvider(config, exchange=exchange, pairlists=None)
df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair})
@@ -870,3 +1081,18 @@ class RPC:
'subplots' not in self._freqtrade.strategy.plot_config):
self._freqtrade.strategy.plot_config['subplots'] = {}
return self._freqtrade.strategy.plot_config
@staticmethod
def _rpc_sysinfo() -> Dict[str, Any]:
return {
"cpu_pct": psutil.cpu_percent(interval=1, percpu=True),
"ram_pct": psutil.virtual_memory().percent
}
def _health(self) -> Dict[str, Union[str, int]]:
last_p = self._freqtrade.last_process
return {
'last_process': str(last_p),
'last_process_loc': last_p.astimezone(tzlocal()).strftime(DATETIME_PRINT_FORMAT),
'last_process_ts': int(last_p.timestamp()),
}

6
freqtrade/rpc/rpc_manager.py
View File

@@ -60,6 +60,10 @@ class RPCManager:
}
"""
logger.info('Sending rpc message: %s', msg)
if 'pair' in msg:
msg.update({
'base_currency': self._rpc._freqtrade.exchange.get_pair_base_currency(msg['pair'])
})
for mod in self.registered_modules:
logger.debug('Forwarding message to rpc.%s', mod.name)
try:
@@ -81,12 +85,14 @@ class RPCManager:
timeframe = config['timeframe']
exchange_name = config['exchange']['name']
strategy_name = config.get('strategy', '')
pos_adjust_enabled = 'On' if config['position_adjustment_enable'] else 'Off'
self.send_msg({
'type': RPCMessageType.STARTUP,
'status': f'*Exchange:* `{exchange_name}`\n'
f'*Stake per trade:* `{stake_amount} {stake_currency}`\n'
f'*Minimum ROI:* `{minimal_roi}`\n'
f'*{"Trailing " if trailing_stop else ""}Stoploss:* `{stoploss}`\n'
f'*Position adjustment:* `{pos_adjust_enabled}`\n'
f'*Timeframe:* `{timeframe}`\n'
f'*Strategy:* `{strategy_name}`'
})

822
freqtrade/rpc/telegram.py
View File

File diff suppressed because it is too large Load Diff

71
freqtrade/rpc/webhook.py
View File

@@ -2,6 +2,7 @@
This module manages webhook communication
"""
import logging
import time
from typing import Any, Dict
from requests import RequestException, post
@@ -28,12 +29,9 @@ class Webhook(RPCHandler):
super().__init__(rpc, config)
self._url = self._config['webhook']['url']
self._format = self._config['webhook'].get('format', 'form')
if self._format != 'form' and self._format != 'json':
raise NotImplementedError('Unknown webhook format `{}`, possible values are '
'`form` (default) and `json`'.format(self._format))
self._retries = self._config['webhook'].get('retries', 0)
self._retry_delay = self._config['webhook'].get('retry_delay', 0.1)
def cleanup(self) -> None:
"""
@@ -45,23 +43,23 @@ class Webhook(RPCHandler):
def send_msg(self, msg: Dict[str, Any]) -> None:
""" Send a message to telegram channel """
try:
if msg['type'] == RPCMessageType.BUY:
valuedict = self._config['webhook'].get('webhookbuy', None)
elif msg['type'] == RPCMessageType.BUY_CANCEL:
valuedict = self._config['webhook'].get('webhookbuycancel', None)
elif msg['type'] == RPCMessageType.BUY_FILL:
valuedict = self._config['webhook'].get('webhookbuyfill', None)
elif msg['type'] == RPCMessageType.SELL:
valuedict = self._config['webhook'].get('webhooksell', None)
elif msg['type'] == RPCMessageType.SELL_FILL:
valuedict = self._config['webhook'].get('webhooksellfill', None)
elif msg['type'] == RPCMessageType.SELL_CANCEL:
valuedict = self._config['webhook'].get('webhooksellcancel', None)
whconfig = self._config['webhook']
if msg['type'] in [RPCMessageType.ENTRY]:
valuedict = whconfig.get('webhookentry', None)
elif msg['type'] in [RPCMessageType.ENTRY_CANCEL]:
valuedict = whconfig.get('webhookentrycancel', None)
elif msg['type'] in [RPCMessageType.ENTRY_FILL]:
valuedict = whconfig.get('webhookentryfill', None)
elif msg['type'] == RPCMessageType.EXIT:
valuedict = whconfig.get('webhookexit', None)
elif msg['type'] == RPCMessageType.EXIT_FILL:
valuedict = whconfig.get('webhookexitfill', None)
elif msg['type'] == RPCMessageType.EXIT_CANCEL:
valuedict = whconfig.get('webhookexitcancel', None)
elif msg['type'] in (RPCMessageType.STATUS,
RPCMessageType.STARTUP,
RPCMessageType.WARNING):
valuedict = self._config['webhook'].get('webhookstatus', None)
valuedict = whconfig.get('webhookstatus', None)
else:
raise NotImplementedError('Unknown message type: {}'.format(msg['type']))
if not valuedict:
@@ -77,13 +75,30 @@ class Webhook(RPCHandler):
def _send_msg(self, payload: dict) -> None:
"""do the actual call to the webhook"""
try:
if self._format == 'form':
post(self._url, data=payload)
elif self._format == 'json':
post(self._url, json=payload)
else:
raise NotImplementedError('Unknown format: {}'.format(self._format))
success = False
attempts = 0
while not success and attempts <= self._retries:
if attempts:
if self._retry_delay:
time.sleep(self._retry_delay)
logger.info("Retrying webhook...")
except RequestException as exc:
logger.warning("Could not call webhook url. Exception: %s", exc)
attempts += 1
try:
if self._format == 'form':
response = post(self._url, data=payload)
elif self._format == 'json':
response = post(self._url, json=payload)
elif self._format == 'raw':
response = post(self._url, data=payload['data'],
headers={'Content-Type': 'text/plain'})
else:
raise NotImplementedError('Unknown format: {}'.format(self._format))
# Throw a RequestException if the post was not successful
response.raise_for_status()
success = True
except RequestException as exc:
logger.warning("Could not call webhook url. Exception: %s", exc)

5
freqtrade/strategy/hyper.py
View File

@@ -292,7 +292,7 @@ class BooleanParameter(CategoricalParameter):
load=load, **kwargs)
class HyperStrategyMixin(object):
class HyperStrategyMixin:
"""
A helper base class which allows HyperOptAuto class to reuse implementations of buy/sell
strategy logic.
@@ -381,7 +381,8 @@ class HyperStrategyMixin(object):
if filename.is_file():
logger.info(f"Loading parameters from file {filename}")
try:
params = json_load(filename.open('r'))
with filename.open('r') as f:
params = json_load(f)
if params.get('strategy_name') != self.__class__.__name__:
raise OperationalException('Invalid parameter file provided.')
return params

Some files were not shown because too many files have changed in this diff Show More