adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feat/kevinjulian/add-buy-signal-name

Browse Source
This commit is contained in:
Kevin Julian
2021-07-20 19:19:46 +07:00
committed by GitHub
parent ed30c023cd f870c0099b
commit edf9c08f06
100 changed files with 2487 additions and 635 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
freqtrade/commands/__init__.py
View File

@@ -20,3 +20,4 @@ from freqtrade.commands.optimize_commands import start_backtesting, start_edge,
from freqtrade.commands.pairlist_commands import start_test_pairlist
from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit
from freqtrade.commands.trade_commands import start_trading
from freqtrade.commands.webserver_commands import start_webserver

16
freqtrade/commands/arguments.py
View File

@@ -16,6 +16,8 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
ARGS_WEBSERVER: List[str] = []
ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
"max_open_trades", "stake_amount", "fee", "pairs"]
@@ -29,7 +31,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"epochs", "spaces", "print_all",
"print_colorized", "print_json", "hyperopt_jobs",
"hyperopt_random_state", "hyperopt_min_trades",
"hyperopt_loss"]
"hyperopt_loss", "disableparamexport"]
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
@@ -85,7 +87,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable",
"hyperoptexportfilename", "export_csv"]
ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index",
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header"]
"print_json", "hyperoptexportfilename", "hyperopt_show_no_header",
"disableparamexport"]
NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes",
"list-markets", "list-pairs", "list-strategies", "list-data",
@@ -175,7 +178,8 @@ class Arguments:
start_list_markets, start_list_strategies,
start_list_timeframes, start_new_config, start_new_hyperopt,
start_new_strategy, start_plot_dataframe, start_plot_profit,
start_show_trades, start_test_pairlist, start_trading)
start_show_trades, start_test_pairlist, start_trading,
start_webserver)
subparsers = self.parser.add_subparsers(dest='command',
# Use custom message when no subhandler is added
@@ -383,3 +387,9 @@ class Arguments:
)
plot_profit_cmd.set_defaults(func=start_plot_profit)
self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
# Add webserver subcommand
webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
parents=[_common_parser])
webserver_cmd.set_defaults(func=start_webserver)
self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)

5
freqtrade/commands/cli_options.py
View File

@@ -178,6 +178,11 @@ AVAILABLE_CLI_OPTIONS = {
'Example: `--export-filename=user_data/backtest_results/backtest_today.json`',
metavar='PATH',
),
"disableparamexport": Arg(
'--disable-param-export',
help="Disable automatic hyperopt parameter export.",
action='store_true',
),
"fee": Arg(
'--fee',
help='Specify fee ratio. Will be applied twice (on trade entry and exit).',

3
freqtrade/commands/data_commands.py
View File

@@ -48,7 +48,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
# Init exchange
exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
# Manual validations of relevant settings
exchange.validate_pairs(config['pairs'])
if not config['exchange'].get('skip_pair_validation', False):
exchange.validate_pairs(config['pairs'])
expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))
logger.info(f"About to download pairs: {expanded_pairs}, "

5
freqtrade/commands/hyperopt_commands.py
View File

@@ -129,9 +129,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
metrics = val['results_metrics']
if 'strategy_name' in metrics:
show_backtest_result(metrics['strategy_name'], metrics,
strategy_name = metrics['strategy_name']
show_backtest_result(strategy_name, metrics,
metrics['stake_currency'])
HyperoptTools.try_export_params(config, strategy_name, val)
HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header,
header_str="Epoch details")

4
freqtrade/commands/list_commands.py
View File

@@ -14,7 +14,7 @@ from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.enums import RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import market_is_active, validate_exchanges
from freqtrade.misc import plural
from freqtrade.misc import parse_db_uri_for_logging, plural
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -225,7 +225,7 @@ def start_show_trades(args: Dict[str, Any]) -> None:
if 'db_url' not in config:
raise OperationalException("--db-url is required for this command.")
logger.info(f'Using DB: "{config["db_url"]}"')
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
init_db(config['db_url'], clean_open_orders=False)
tfilter = []

15
freqtrade/commands/webserver_commands.py Normal file
View File

@@ -0,0 +1,15 @@
from typing import Any, Dict
from freqtrade.enums import RunMode
def start_webserver(args: Dict[str, Any]) -> None:
"""
Main entry point for webserver mode
"""
from freqtrade.configuration import Configuration
from freqtrade.rpc.api_server import ApiServer
# Initialize configuration
config = Configuration(args, RunMode.WEBSERVER).get_config()
ApiServer(config, standalone=True)

8
freqtrade/configuration/configuration.py
View File

@@ -15,7 +15,7 @@ from freqtrade.configuration.load_config import load_config_file, load_file
from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts
from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging
logger = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ class Configuration:
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:
config['internals'] = {}
@@ -144,7 +144,7 @@ class Configuration:
config['db_url'] = constants.DEFAULT_DB_PROD_URL
logger.info('Dry run is disabled')
logger.info(f'Using DB: "{config["db_url"]}"')
logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"')
def _process_common_options(self, config: Dict[str, Any]) -> None:
@@ -260,6 +260,8 @@ class Configuration:
self._args_to_config(config, argname='export',
logstring='Parameter --export detected: {} ...')
self._args_to_config(config, argname='disableparamexport',
logstring='Parameter --disableparamexport detected: {} ...')
# Edge section:
if 'stoploss_range' in self.args and self.args["stoploss_range"]:
txt_range = eval(self.args["stoploss_range"])

12
freqtrade/constants.py
View File

@@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily',
'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily']
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
'SpreadFilter', 'VolatilityFilter']
'AgeFilter', 'OffsetFilter', 'PerformanceFilter',
'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter',
'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter']
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
DRY_RUN_WALLET = 1000
@@ -40,6 +40,7 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume']
DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost']
LAST_BT_RESULT_FN = '.last_result.json'
FTHYPT_FILEVERSION = 'fthypt_fileversion'
USERPATH_HYPEROPTS = 'hyperopts'
USERPATH_STRATEGIES = 'strategies'
@@ -112,6 +113,10 @@ CONF_SCHEMA = {
'maximum': 1,
'default': 0.99
},
'available_capital': {
'type': 'number',
'minimum': 0,
},
'amend_last_stake_amount': {'type': 'boolean', 'default': False},
'last_stake_amount_min_ratio': {
'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5
@@ -312,6 +317,7 @@ CONF_SCHEMA = {
},
'db_url': {'type': 'string'},
'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'},
'disableparamexport': {'type': 'boolean'},
'initial_state': {'type': 'string', 'enum': ['running', 'stopped']},
'forcebuy_enable': {'type': 'boolean'},
'disable_dataframe_checks': {'type': 'boolean'},

6
freqtrade/data/history/history_utils.py
View File

@@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path,
new_data = exchange.get_historic_ohlcv(pair=pair,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(
days=-new_pairs_days).float_timestamp) * 1000
arrow.utcnow().shift(
days=-new_pairs_days).int_timestamp * 1000
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -272,7 +272,7 @@ def _download_trades_history(exchange: Exchange,
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:
since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000
trades = data_handler.trades_load(pair)

1
freqtrade/enums/__init__.py
View File

@@ -1,4 +1,5 @@
# flake8: noqa: F401
from freqtrade.enums.backteststate import BacktestState
from freqtrade.enums.rpcmessagetype import RPCMessageType
from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode
from freqtrade.enums.selltype import SellType

15
freqtrade/enums/backteststate.py Normal file
View File

@@ -0,0 +1,15 @@
from enum import Enum
class BacktestState(Enum):
"""
Bot application states
"""
STARTUP = 1
DATALOAD = 2
ANALYZE = 3
CONVERT = 4
BACKTEST = 5
def __str__(self):
return f"{self.name.lower()}"

1
freqtrade/enums/runmode.py
View File

@@ -14,6 +14,7 @@ class RunMode(Enum):
UTIL_EXCHANGE = "util_exchange"
UTIL_NO_EXCHANGE = "util_no_exchange"
PLOT = "plot"
WEBSERVER = "webserver"
OTHER = "other"

102
freqtrade/exchange/exchange.py
View File

@@ -551,7 +551,7 @@ class Exchange:
amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent',
DEFAULT_AMOUNT_RESERVE_PERCENT)
amount_reserve_percent = (
amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
)
# it should not be more than 50%
amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1)
@@ -578,7 +578,7 @@ class Exchange:
'side': side,
'remaining': _amount,
'datetime': arrow.utcnow().isoformat(),
'timestamp': int(arrow.utcnow().int_timestamp * 1000),
'timestamp': arrow.utcnow().int_timestamp * 1000,
'status': "closed" if ordertype == "market" else "open",
'fee': None,
'info': {}
@@ -999,94 +999,64 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(e) from e
def get_buy_rate(self, pair: str, refresh: bool) -> float:
def get_rate(self, pair: str, refresh: bool, side: str) -> float:
"""
Calculates bid target between current ask price and last price
Calculates bid/ask target
bid rate - between current ask price and last price
ask rate - either using ticker bid or first bid based on orderbook
or remain static in any other case since it's not updating.
:param pair: Pair to get rate for
:param refresh: allow cached data
:param side: "buy" or "sell"
:return: float: Price
:raises PricingError if orderbook price could not be determined.
"""
cache_rate: TTLCache = self._buy_rate_cache if side == "buy" else self._sell_rate_cache
[strat_name, name] = ['bid_strategy', 'Buy'] if side == "buy" else ['ask_strategy', 'Sell']
if not refresh:
rate = self._buy_rate_cache.get(pair)
rate = cache_rate.get(pair)
# Check if cache has been invalidated
if rate:
logger.debug(f"Using cached buy rate for {pair}.")
logger.debug(f"Using cached {side} rate for {pair}.")
return rate
bid_strategy = self._config.get('bid_strategy', {})
if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
conf_strategy = self._config.get(strat_name, {})
order_book_top = bid_strategy.get('order_book_top', 1)
if conf_strategy.get('use_order_book', False) and ('use_order_book' in conf_strategy):
order_book_top = conf_strategy.get('order_book_top', 1)
order_book = self.fetch_l2_order_book(pair, order_book_top)
logger.debug('order_book %s', order_book)
# top 1 = index 0
try:
rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0]
rate = order_book[f"{conf_strategy['price_side']}s"][order_book_top - 1][0]
except (IndexError, KeyError) as e:
logger.warning(
"Buy Price from orderbook could not be determined."
f"Orderbook: {order_book}"
)
raise PricingError from e
logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
used_rate = rate_from_l2
else:
logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
ticker = self.fetch_ticker(pair)
ticker_rate = ticker[bid_strategy['price_side']]
if ticker['last'] and ticker_rate > ticker['last']:
balance = bid_strategy['ask_last_balance']
ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
used_rate = ticker_rate
self._buy_rate_cache[pair] = used_rate
return used_rate
def get_sell_rate(self, pair: str, refresh: bool) -> float:
"""
Get sell rate - either using ticker bid or first bid based on orderbook
or remain static in any other case since it's not updating.
:param pair: Pair to get rate for
:param refresh: allow cached data
:return: Bid rate
:raises PricingError if price could not be determined.
"""
if not refresh:
rate = self._sell_rate_cache.get(pair)
# Check if cache has been invalidated
if rate:
logger.debug(f"Using cached sell rate for {pair}.")
return rate
ask_strategy = self._config.get('ask_strategy', {})
if ask_strategy.get('use_order_book', False):
logger.debug(
f"Getting price from order book {ask_strategy['price_side'].capitalize()} side."
)
order_book_top = ask_strategy.get('order_book_top', 1)
order_book = self.fetch_l2_order_book(pair, order_book_top)
try:
rate = order_book[f"{ask_strategy['price_side']}s"][order_book_top - 1][0]
except (IndexError, KeyError) as e:
logger.warning(
f"Sell Price at location {order_book_top} from orderbook could not be "
f"{name} Price at location {order_book_top} from orderbook could not be "
f"determined. Orderbook: {order_book}"
)
raise PricingError from e
logger.info(f"{name} price from orderbook {conf_strategy['price_side'].capitalize()}"
f"side - top {order_book_top} order book {side} rate {rate:.8f}")
else:
logger.info(f"Using Last {conf_strategy['price_side'].capitalize()} / Last Price")
ticker = self.fetch_ticker(pair)
ticker_rate = ticker[ask_strategy['price_side']]
if ticker['last'] and ticker_rate < ticker['last']:
balance = ask_strategy.get('bid_last_balance', 0.0)
ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
ticker_rate = ticker[conf_strategy['price_side']]
if ticker['last']:
if side == 'buy' and ticker_rate > ticker['last']:
balance = conf_strategy['ask_last_balance']
ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate)
elif side == 'sell' and ticker_rate < ticker['last']:
balance = conf_strategy.get('bid_last_balance', 0.0)
ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last'])
rate = ticker_rate
if rate is None:
raise PricingError(f"Sell-Rate for {pair} was empty.")
self._sell_rate_cache[pair] = rate
raise PricingError(f"{name}-Rate for {pair} was empty.")
cache_rate[pair] = rate
return rate
# Fee handling
@@ -1318,8 +1288,8 @@ class Exchange:
self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000
# keeping parsed dataframe in cache
ohlcv_df = ohlcv_to_dataframe(
ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
ticks, timeframe, pair=pair, fill_missing=True,
drop_incomplete=self._ohlcv_partial_candle)
results_df[(pair, timeframe)] = ohlcv_df
if cache:
self._klines[(pair, timeframe)] = ohlcv_df

38
freqtrade/freqtradebot.py
View File

@@ -424,16 +424,10 @@ class FreqtradeBot(LoggingMixin):
if buy and not sell:
stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
if not stake_amount:
logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
return False
logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
f"{stake_amount} ...")
bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
if ((bid_check_dom.get('enabled', False)) and
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
if self._check_depth_of_market_buy(pair, bid_check_dom):
return self.execute_buy(pair, stake_amount, buy_signal_name=buy_signal_name)
else:
@@ -481,20 +475,29 @@ class FreqtradeBot(LoggingMixin):
buy_limit_requested = price
else:
# Calculate price
buy_limit_requested = self.exchange.get_buy_rate(pair, True)
buy_limit_requested = self.exchange.get_rate(pair, refresh=True, side="buy")
if not buy_limit_requested:
raise PricingError('Could not determine buy price.')
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested,
self.strategy.stoploss)
if min_stake_amount is not None and min_stake_amount > stake_amount:
logger.warning(
f"Can't open a new trade for {pair}: stake amount "
f"is too small ({stake_amount} < {min_stake_amount})"
)
if not self.edge:
max_stake_amount = self.wallets.get_available_stake_amount()
stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
default_retval=stake_amount)(
pair=pair, current_time=datetime.now(timezone.utc),
current_rate=buy_limit_requested, proposed_stake=stake_amount,
min_stake=min_stake_amount, max_stake=max_stake_amount)
stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
if not stake_amount:
return False
logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
f"{stake_amount} ...")
amount = stake_amount / buy_limit_requested
order_type = self.strategy.order_types['buy']
if forcebuy:
@@ -607,7 +610,7 @@ class FreqtradeBot(LoggingMixin):
"""
Sends rpc notification when a buy cancel occurred.
"""
current_rate = self.exchange.get_buy_rate(trade.pair, False)
current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="buy")
msg = {
'trade_id': trade.id,
@@ -693,7 +696,7 @@ class FreqtradeBot(LoggingMixin):
(buy, sell, _) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df)
logger.debug('checking sell')
sell_rate = self.exchange.get_sell_rate(trade.pair, True)
sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell")
if self._check_and_execute_sell(trade, sell_rate, buy, sell):
return True
@@ -1130,7 +1133,8 @@ class FreqtradeBot(LoggingMixin):
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
# Use cached rates here - it was updated seconds ago.
current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None
current_rate = self.exchange.get_rate(
trade.pair, refresh=False, side="sell") if not fill else None
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"
@@ -1175,7 +1179,7 @@ class FreqtradeBot(LoggingMixin):
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
current_rate = self.exchange.get_sell_rate(trade.pair, False)
current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="sell")
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"

14
freqtrade/misc.py
View File

@@ -8,6 +8,7 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Iterator, List
from typing.io import IO
from urllib.parse import urlparse
import rapidjson
@@ -214,3 +215,16 @@ def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
"""
for chunk in range(0, len(lst), n):
yield (lst[chunk:chunk + n])
def parse_db_uri_for_logging(uri: str):
"""
Helper method to parse the DB URI and return the same DB URI with the password censored
if it contains it. Otherwise, return the DB URI unchanged
:param uri: DB URI to parse for logging
"""
parsed_db_uri = urlparse(uri)
if not parsed_db_uri.netloc: # No need for censoring as no password was provided
return uri
pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0]
return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')

57
freqtrade/optimize/backtesting.py
View File

@@ -17,10 +17,11 @@ from freqtrade.data import history
from freqtrade.data.btanalysis import trade_list_to_dataframe
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import SellType
from freqtrade.enums import BacktestState, SellType
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
from freqtrade.mixins import LoggingMixin
from freqtrade.optimize.bt_progress import BTProgress
from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results,
store_backtest_stats)
from freqtrade.persistence import LocalTrade, PairLocks, Trade
@@ -58,6 +59,7 @@ class Backtesting:
LoggingMixin.show_output = False
self.config = config
self.results: Optional[Dict[str, Any]] = None
# Reset keys for backtesting
remove_credentials(self.config)
@@ -117,6 +119,10 @@ class Backtesting:
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe)
self.progress = BTProgress()
self.abort = False
def __del__(self):
LoggingMixin.show_output = True
@@ -129,6 +135,8 @@ class Backtesting:
"""
self.strategy: IStrategy = strategy
strategy.dp = self.dataprovider
# Attach Wallets to Strategy baseclass
IStrategy.wallets = self.wallets
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
@@ -145,6 +153,8 @@ class Backtesting:
Loads backtest data and returns the data combined with the timerange
as tuple.
"""
self.progress.init_step(BacktestState.DATALOAD, 1)
timerange = TimeRange.parse_timerange(None if self.config.get(
'timerange') is None else str(self.config.get('timerange')))
@@ -168,6 +178,7 @@ class Backtesting:
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
self.required_startup, min_date)
self.progress.set_new_value(1)
return data, timerange
def prepare_backtest(self, enable_protections):
@@ -182,6 +193,15 @@ class Backtesting:
self.rejected_trades = 0
self.dataprovider.clear_cache()
def check_abort(self):
"""
Check if abort was requested, raise DependencyException if that's the case
Only applies to Interactive backtest mode (webserver mode)
"""
if self.abort:
self.abort = False
raise DependencyException("Stop requested")
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
"""
Helper function to convert a processed dataframes into lists for performance reasons.
@@ -192,8 +212,12 @@ class Backtesting:
# and eventually change the constants for indexes at the top
headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_signal_name']
data: Dict = {}
self.progress.init_step(BacktestState.CONVERT, len(processed))
# Create dict with data
for pair, pair_data in processed.items():
self.check_abort()
self.progress.increment()
if not pair_data.empty:
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
@@ -314,7 +338,18 @@ class Backtesting:
stake_amount = self.wallets.get_trade_stake_amount(pair, None)
except DependencyException:
return None
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) or 0
max_stake_amount = self.wallets.get_available_stake_amount()
stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount,
default_retval=stake_amount)(
pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX],
proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount)
stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount)
if not stake_amount:
return None
order_type = self.strategy.order_types['buy']
time_in_force = self.strategy.order_time_in_force['sell']
@@ -407,10 +442,13 @@ class Backtesting:
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
open_trade_count = 0
self.progress.init_step(BacktestState.BACKTEST, int(
(end_date - start_date) / timedelta(minutes=self.timeframe_min)))
# Loop timerange and get candle for each pair at that point in time
while tmp <= end_date:
open_trade_count_start = open_trade_count
self.check_abort()
for i, pair in enumerate(data):
row_index = indexes[pair]
try:
@@ -466,6 +504,7 @@ class Backtesting:
self.protections.global_stop(tmp)
# Move time one configured time_interval ahead.
self.progress.increment()
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
@@ -481,6 +520,8 @@ class Backtesting:
}
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
self.progress.init_step(BacktestState.ANALYZE, 0)
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
backtest_start_time = datetime.now(timezone.utc)
self._set_strategy(strat)
@@ -507,6 +548,7 @@ class Backtesting:
"No data left after adjusting for startup candles.")
min_date, max_date = history.get_timerange(preprocessed)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days).')
@@ -541,11 +583,12 @@ class Backtesting:
for strat in self.strategylist:
min_date, max_date = self.backtest_one_strategy(strat, data, timerange)
if len(self.strategylist) > 0:
stats = generate_backtest_stats(data, self.all_results,
min_date=min_date, max_date=max_date)
self.results = generate_backtest_stats(data, self.all_results,
min_date=min_date, max_date=max_date)
if self.config.get('export', 'none') == 'trades':
store_backtest_stats(self.config['exportfilename'], stats)
store_backtest_stats(self.config['exportfilename'], self.results)
# Show backtest results
show_backtest_results(self.config, stats)
show_backtest_results(self.config, self.results)

33
freqtrade/optimize/bt_progress.py Normal file
View File

@@ -0,0 +1,33 @@
from freqtrade.enums import BacktestState
class BTProgress:
_action: BacktestState = BacktestState.STARTUP
_progress: float = 0
_max_steps: float = 0
def __init__(self):
pass
def init_step(self, action: BacktestState, max_steps: float):
self._action = action
self._max_steps = max_steps
self._proress = 0
def set_new_value(self, new_value: float):
self._progress = new_value
def increment(self):
self._progress += 1
@property
def progress(self):
"""
Get progress as ratio, capped to be between 0 and 1 (to avoid small calculation errors).
"""
return max(min(round(self._progress / self._max_steps, 5)
if self._max_steps > 0 else 0, 1), 0)
@property
def action(self):
return str(self._action)

49
freqtrade/optimize/hyperopt.py
View File

@@ -12,7 +12,6 @@ from math import ceil
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import progressbar
import rapidjson
from colorama import Fore, Style
@@ -20,16 +19,16 @@ from colorama import init as colorama_init
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.misc import file_dump_json, plural
from freqtrade.misc import deep_merge_dicts, file_dump_json, plural
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_auto import HyperOptAuto
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.optimize.hyperopt_tools import HyperoptTools
from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer
from freqtrade.optimize.optimize_reports import generate_strategy_stats
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
@@ -78,8 +77,11 @@ class Hyperopt:
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
self.auto_hyperopt = True
else:
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
self.auto_hyperopt = False
self.backtesting._set_strategy(self.backtesting.strategylist[0])
self.custom_hyperopt.strategy = self.backtesting.strategy
@@ -163,13 +165,9 @@ class Hyperopt:
While not a valid json object - this allows appending easily.
:param epoch: result dictionary for this epoch.
"""
def default_parser(x):
if isinstance(x, np.integer):
return int(x)
return str(x)
epoch[FTHYPT_FILEVERSION] = 2
with self.results_file.open('a') as f:
rapidjson.dump(epoch, f, default=default_parser,
rapidjson.dump(epoch, f, default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN)
f.write("\n")
@@ -201,6 +199,25 @@ class Hyperopt:
return result
def _get_no_optimize_details(self) -> Dict[str, Any]:
"""
Get non-optimized parameters
"""
result: Dict[str, Any] = {}
strategy = self.backtesting.strategy
if not HyperoptTools.has_space(self.config, 'roi'):
result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()}
if not HyperoptTools.has_space(self.config, 'stoploss'):
result['stoploss'] = {'stoploss': strategy.stoploss}
if not HyperoptTools.has_space(self.config, 'trailing'):
result['trailing'] = {
'trailing_stop': strategy.trailing_stop,
'trailing_stop_positive': strategy.trailing_stop_positive,
'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset,
'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached,
}
return result
def print_results(self, results) -> None:
"""
Log results if it is better than any previous evaluation
@@ -310,7 +327,8 @@ class Hyperopt:
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
not_optimized = self.backtesting.strategy.get_params_dict()
not_optimized = self.backtesting.strategy.get_no_optimize_params()
not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details())
trade_count = strat_stats['total_trades']
total_profit = strat_stats['profit_total']
@@ -324,7 +342,8 @@ class Hyperopt:
loss = self.calculate_loss(results=backtesting_results['results'],
trade_count=trade_count,
min_date=min_date, max_date=max_date,
config=self.config, processed=processed)
config=self.config, processed=processed,
backtest_stats=strat_stats)
return {
'loss': loss,
'params_dict': params_dict,
@@ -469,6 +488,12 @@ class Hyperopt:
f"saved to '{self.results_file}'.")
if self.current_best_epoch:
if self.auto_hyperopt:
HyperoptTools.try_export_params(
self.config,
self.backtesting.strategy.get_strategy_name(),
self.current_best_epoch)
HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs,
self.print_json)
else:

3
freqtrade/optimize/hyperopt_loss_interface.py
View File

@@ -5,7 +5,7 @@ This module defines the interface for the loss-function for hyperopt
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Dict
from typing import Any, Dict
from pandas import DataFrame
@@ -22,6 +22,7 @@ class IHyperOptLoss(ABC):
def hyperopt_loss_function(results: DataFrame, trade_count: int,
min_date: datetime, max_date: datetime,
config: Dict, processed: Dict[str, DataFrame],
backtest_stats: Dict[str, Any],
*args, **kwargs) -> float:
"""
Objective function, returns smaller number for better results

104
freqtrade/optimize/hyperopt_tools.py
View File

@@ -1,23 +1,82 @@
import io
import logging
from copy import deepcopy
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
import numpy as np
import rapidjson
import tabulate
from colorama import Fore, Style
from pandas import isna, json_normalize
from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.misc import round_coin_value, round_dict
from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2
logger = logging.getLogger(__name__)
NON_OPT_PARAM_APPENDIX = " # value loaded from strategy"
def hyperopt_serializer(x):
if isinstance(x, np.integer):
return int(x)
if isinstance(x, np.bool_):
return bool(x)
return str(x)
class HyperoptTools():
@staticmethod
def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]:
"""
Get Strategy-location (filename) from strategy_name
"""
from freqtrade.resolvers.strategy_resolver import StrategyResolver
directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES))
strategy_objs = StrategyResolver.search_all_objects(directory, False)
strategies = [s for s in strategy_objs if s['name'] == strategy_name]
if strategies:
strategy = strategies[0]
return Path(strategy['location'])
return None
@staticmethod
def export_params(params, strategy_name: str, filename: Path):
"""
Generate files
"""
final_params = deepcopy(params['params_not_optimized'])
final_params = deep_merge_dicts(params['params_details'], final_params)
final_params = {
'strategy_name': strategy_name,
'params': final_params,
'ft_stratparam_v': 1,
'export_time': datetime.now(timezone.utc),
}
logger.info(f"Dumping parameters to {filename}")
rapidjson.dump(final_params, filename.open('w'), indent=2,
default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
)
@staticmethod
def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict):
if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False):
# Export parameters ...
fn = HyperoptTools.get_strategy_filename(config, strategy_name)
if fn:
HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json'))
else:
logger.warning("Strategy not found, not exporting parameter file.")
@staticmethod
def has_space(config: Dict[str, Any], space: str) -> bool:
"""
@@ -99,9 +158,9 @@ class HyperoptTools():
non_optimized)
HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
non_optimized)
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized)
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized)
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized)
@staticmethod
def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None:
@@ -127,23 +186,34 @@ class HyperoptTools():
def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
if space in params or space in non_optimized:
space_params = HyperoptTools._space_params(params, space, 5)
no_params = HyperoptTools._space_params(non_optimized, space, 5)
appendix = ''
if not space_params and not no_params:
# No parameters - don't print
return
if not space_params:
# Not optimized parameters - append string
appendix = NON_OPT_PARAM_APPENDIX
result = f"\n# {header}\n"
if space == 'stoploss':
result += f"stoploss = {space_params.get('stoploss')}"
elif space == 'roi':
if space == "stoploss":
stoploss = safe_value_fallback2(space_params, no_params, space, space)
result += (f"stoploss = {stoploss}{appendix}")
elif space == "roi":
result = result[:-1] + f'{appendix}\n'
minimal_roi_result = rapidjson.dumps({
str(k): v for k, v in space_params.items()
str(k): v for k, v in (space_params or no_params).items()
}, default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
result += f"minimal_roi = {minimal_roi_result}"
elif space == 'trailing':
for k, v in space_params.items():
result += f'{k} = {v}\n'
elif space == "trailing":
for k, v in (space_params or no_params).items():
result += f"{k} = {v}{appendix}\n"
else:
no_params = HyperoptTools._space_params(non_optimized, space, 5)
# Buy / sell parameters
result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}"
result = result.replace("\n", "\n ")
print(result)
@@ -157,7 +227,7 @@ class HyperoptTools():
return {}
@staticmethod
def _pprint(params, non_optimized, indent: int = 4):
def _pprint_dict(params, non_optimized, indent: int = 4):
"""
Pretty-print hyperopt results (based on 2 dicts - with add. comment)
"""
@@ -169,7 +239,7 @@ class HyperoptTools():
result += " " * indent + f'"{k}": '
result += f'"{param}",' if isinstance(param, str) else f'{param},'
if k in non_optimized:
result += " # value loaded from strategy"
result += NON_OPT_PARAM_APPENDIX
result += "\n"
result += '}'
return result

20
freqtrade/optimize/optimize_reports.py
View File

@@ -229,8 +229,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
winning_trades = results.loc[results['profit_ratio'] > 0]
draw_trades = results.loc[results['profit_ratio'] == 0]
losing_trades = results.loc[results['profit_ratio'] < 0]
zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) &
(results['sell_reason'] == 'trailing_stop_loss')])
holding_avg = (timedelta(minutes=round(results['trade_duration'].mean()))
if not results.empty else timedelta())
@@ -249,7 +247,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
'winner_holding_avg_s': winner_holding_avg.total_seconds(),
'loser_holding_avg': loser_holding_avg,
'loser_holding_avg_s': loser_holding_avg.total_seconds(),
'zero_duration_trades': zero_duration_trades,
}
@@ -264,6 +261,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
'winning_days': 0,
'draw_days': 0,
'losing_days': 0,
'daily_profit_list': [],
}
daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
@@ -274,6 +272,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
winning_days = sum(daily_profit > 0)
draw_days = sum(daily_profit == 0)
losing_days = sum(daily_profit < 0)
daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()]
return {
'backtest_best_day': best_rel,
@@ -283,6 +282,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
'winning_days': winning_days,
'draw_days': draw_days,
'losing_days': losing_days,
'daily_profit': daily_profit_list,
}
@@ -325,8 +325,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
if not results.empty:
results['open_timestamp'] = results['open_date'].view(int64) // 1e6
results['close_timestamp'] = results['close_date'].view(int64) // 1e6
backtest_days = (max_date - min_date).days
strat_stats = {
@@ -542,14 +543,6 @@ def text_table_add_metrics(strat_results: Dict) -> str:
# Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
# command stores these results and newer version of freqtrade must be able to handle old
# results with missing new fields.
zero_duration_trades = '--'
if 'zero_duration_trades' in strat_results:
zero_duration_trades_per = \
100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades']
zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \
f'({strat_results["zero_duration_trades"]})'
metrics = [
('Backtesting from', strat_results['backtest_start']),
('Backtesting to', strat_results['backtest_end']),
@@ -585,7 +578,6 @@ def text_table_add_metrics(strat_results: Dict) -> str:
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
('Zero Duration Trades', zero_duration_trades),
('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
('', ''), # Empty line to improve readability

13
freqtrade/persistence/models.py
View File

@@ -804,6 +804,19 @@ class Trade(_DECL_BASE, LocalTrade):
Trade.is_open.is_(False),
]).all()
@staticmethod
def get_total_closed_profit() -> float:
"""
Retrieves total realized profit
"""
if Trade.use_db:
total_profit = Trade.query.with_entities(
func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar()
else:
total_profit = sum(
t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False))
return total_profit or 0
@staticmethod
def total_open_trades_stakes() -> float:
"""

39
freqtrade/plugins/pairlist/AgeFilter.py
View File

@@ -27,6 +27,7 @@ class AgeFilter(IPairList):
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._min_days_listed = pairlistconfig.get('min_days_listed', 10)
self._max_days_listed = pairlistconfig.get('max_days_listed', None)
if self._min_days_listed < 1:
raise OperationalException("AgeFilter requires min_days_listed to be >= 1")
@@ -34,6 +35,12 @@ class AgeFilter(IPairList):
raise OperationalException("AgeFilter requires min_days_listed to not exceed "
"exchange max request size "
f"({exchange.ohlcv_candle_limit('1d')})")
if self._max_days_listed and self._max_days_listed <= self._min_days_listed:
raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted")
if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'):
raise OperationalException("AgeFilter requires max_days_listed to not exceed "
"exchange max request size "
f"({exchange.ohlcv_candle_limit('1d')})")
@property
def needstickers(self) -> bool:
@@ -48,8 +55,13 @@ class AgeFilter(IPairList):
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Filtering pairs with age less than "
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.")
return (
f"{self.name} - Filtering pairs with age less than "
f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}"
) + ((
" or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else '')
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
@@ -61,9 +73,12 @@ class AgeFilter(IPairList):
if not needed_pairs:
return pairlist
since_days = -(
self._max_days_listed if self._max_days_listed else self._min_days_listed
) - 1
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._min_days_listed - 1)
.shift(days=since_days)
.float_timestamp) * 1000
candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False)
if self._enabled:
@@ -86,14 +101,22 @@ class AgeFilter(IPairList):
return True
if daily_candles is not None:
if len(daily_candles) >= self._min_days_listed:
if (
len(daily_candles) >= self._min_days_listed
and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed)
):
# We have fetched at least the minimum required number of daily candles
# Add to cache, store the time we last checked this symbol
self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000
self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000
return True
else:
self.log_once(f"Removed {pair} from whitelist, because age "
f"{len(daily_candles)} is less than {self._min_days_listed} "
f"{plural(self._min_days_listed, 'day')}", logger.info)
self.log_once((
f"Removed {pair} from whitelist, because age "
f"{len(daily_candles)} is less than {self._min_days_listed} "
f"{plural(self._min_days_listed, 'day')}"
) + ((
" or more than "
f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}"
) if self._max_days_listed else ''), logger.info)
return False
return False

54
freqtrade/plugins/pairlist/OffsetFilter.py Normal file
View File

@@ -0,0 +1,54 @@
"""
Offset pair list filter
"""
import logging
from typing import Any, Dict, List
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class OffsetFilter(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._offset = pairlistconfig.get('offset', 0)
if self._offset < 0:
raise OperationalException("OffsetFilter requires offset to be >= 0")
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty Dict is passed
as tickers argument to filter_pairlist
"""
return False
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return f"{self.name} - Offseting pairs by {self._offset}."
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Filters and sorts pairlist and returns the whitelist again.
Called on each bot iteration - please use internal caching if necessary
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new whitelist
"""
if self._offset > len(pairlist):
self.log_once(f"Offset of {self._offset} is larger than " +
f"pair count of {len(pairlist)}", logger.warning)
pairs = pairlist[self._offset:]
self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info)
return pairs

8
freqtrade/plugins/pairlist/VolatilityFilter.py
View File

@@ -69,10 +69,10 @@ class VolatilityFilter(IPairList):
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.float_timestamp) * 1000
since_ms = (arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.int_timestamp) * 1000
# Get all candles
candles = {}
if needed_pairs:

94
freqtrade/plugins/pairlist/VolumePairList.py
View File

@@ -6,9 +6,12 @@ Provides dynamic pair list based on trade volumes
import logging
from typing import Any, Dict, List
import arrow
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import format_ms_time
from freqtrade.plugins.pairlist.IPairList import IPairList
@@ -36,6 +39,35 @@ class VolumePairList(IPairList):
self._min_value = self._pairlistconfig.get('min_value', 0)
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
if (self._lookback_days > 0) & (self._lookback_period > 0):
raise OperationalException(
'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
'config. Please set lookback_days only or lookback_period and lookback_timeframe '
'and restart the bot.'
)
# overwrite lookback timeframe and days when lookback_days is set
if self._lookback_days > 0:
self._lookback_timeframe = '1d'
self._lookback_period = self._lookback_days
# get timeframe in minutes and seconds
self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
self._tf_in_sec = self._tf_in_min * 60
# wether to use range lookback or not
self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)
if self._use_range & (self._refresh_period < self._tf_in_sec):
raise OperationalException(
f'Refresh period of {self._refresh_period} seconds is smaller than one '
f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
f'to at least {self._tf_in_sec} and restart the bot.'
)
if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException(
@@ -47,6 +79,13 @@ class VolumePairList(IPairList):
raise OperationalException(
f'key {self._sort_key} not in {SORT_VALUES}')
if self._lookback_period < 0:
raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe):
raise OperationalException("VolumeFilter requires lookback_period to not "
"exceed exchange max request size "
f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})")
@property
def needstickers(self) -> bool:
"""
@@ -78,7 +117,6 @@ class VolumePairList(IPairList):
# Item found - no refresh necessary
return pairlist
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
filtered_tickers = [
@@ -103,6 +141,60 @@ class VolumePairList(IPairList):
# Use the incoming pairlist.
filtered_tickers = [v for k, v in tickers.items() if k in pairlist]
# get lookback period in ms, for exchange ohlcv fetch
if self._use_range:
since_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-(self._lookback_period * self._tf_in_min)
- self._tf_in_min)
.int_timestamp) * 1000
to_ms = int(arrow.utcnow()
.floor('minute')
.shift(minutes=-self._tf_in_min)
.int_timestamp) * 1000
# todo: utc date output for starting date
self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
f"till {format_ms_time(to_ms)}", logger.info)
needed_pairs = [
(p, self._lookback_timeframe) for p in
[
s['symbol'] for s in filtered_tickers
] if p not in self._pair_cache
]
# Get all candles
candles = {}
if needed_pairs:
candles = self._exchange.refresh_latest_ohlcv(
needed_pairs, since_ms=since_ms, cache=False
)
for i, p in enumerate(filtered_tickers):
pair_candles = candles[
(p['symbol'], self._lookback_timeframe)
] if (p['symbol'], self._lookback_timeframe) in candles else None
# in case of candle data calculate typical price and quoteVolume for candle
if pair_candles is not None and not pair_candles.empty:
pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
+ pair_candles['close']) / 3
pair_candles['quoteVolume'] = (
pair_candles['volume'] * pair_candles['typical_price']
)
# ensure that a rolling sum over the lookback_period is built
# if pair_candles contains more candles than lookback_period
quoteVolume = (pair_candles['quoteVolume']
.rolling(self._lookback_period)
.sum()
.iloc[-1])
# replace quoteVolume with range quoteVolume sum calculated above
filtered_tickers[i]['quoteVolume'] = quoteVolume
else:
filtered_tickers[i]['quoteVolume'] = 0
if self._min_value > 0:
filtered_tickers = [
v for v in filtered_tickers if v[self._sort_key] > self._min_value]

8
freqtrade/plugins/pairlist/rangestabilityfilter.py
View File

@@ -62,10 +62,10 @@ class RangeStabilityFilter(IPairList):
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.float_timestamp) * 1000
since_ms = (arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.int_timestamp) * 1000
# Get all candles
candles = {}
if needed_pairs:

15
freqtrade/resolvers/strategy_resolver.py
View File

@@ -53,6 +53,21 @@ class StrategyResolver(IResolver):
)
strategy.timeframe = strategy.ticker_interval
if strategy._ft_params_from_file:
# Set parameters from Hyperopt results file
params = strategy._ft_params_from_file
strategy.minimal_roi = params.get('roi', strategy.minimal_roi)
strategy.stoploss = params.get('stoploss', {}).get('stoploss', strategy.stoploss)
trailing = params.get('trailing', {})
strategy.trailing_stop = trailing.get('trailing_stop', strategy.trailing_stop)
strategy.trailing_stop_positive = trailing.get('trailing_stop_positive',
strategy.trailing_stop_positive)
strategy.trailing_stop_positive_offset = trailing.get(
'trailing_stop_positive_offset', strategy.trailing_stop_positive_offset)
strategy.trailing_only_offset_is_reached = trailing.get(
'trailing_only_offset_is_reached', strategy.trailing_only_offset_is_reached)
# Set attributes
# Check if we need to override configuration
# (Attribute name, default, subkey)

176
freqtrade/rpc/api_server/api_backtest.py Normal file
View File

@@ -0,0 +1,176 @@
import asyncio
import logging
from copy import deepcopy
from fastapi import APIRouter, BackgroundTasks, Depends
from freqtrade.enums import BacktestState
from freqtrade.exceptions import DependencyException
from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse
from freqtrade.rpc.api_server.deps import get_config
from freqtrade.rpc.api_server.webserver import ApiServer
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# Private API, protected by authentication
router = APIRouter()
@router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: BackgroundTasks,
config=Depends(get_config)):
"""Start backtesting if not done so already"""
if ApiServer._bgtask_running:
raise RPCException('Bot Background task already running')
btconfig = deepcopy(config)
settings = dict(bt_settings)
# Pydantic models will contain all keys, but non-provided ones are None
for setting in settings.keys():
if settings[setting] is not None:
btconfig[setting] = settings[setting]
# Start backtesting
# Initialize backtesting object
def run_backtest():
from freqtrade.optimize.optimize_reports import generate_backtest_stats
from freqtrade.resolvers import StrategyResolver
asyncio.set_event_loop(asyncio.new_event_loop())
try:
# Reload strategy
lastconfig = ApiServer._bt_last_config
strat = StrategyResolver.load_strategy(btconfig)
if (
not ApiServer._bt
or lastconfig.get('timeframe') != strat.timeframe
or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0)
):
from freqtrade.optimize.backtesting import Backtesting
ApiServer._bt = Backtesting(btconfig)
# Only reload data if timeframe or timerange changed.
if (
not ApiServer._bt_data
or not ApiServer._bt_timerange
or lastconfig.get('timerange') != btconfig['timerange']
or lastconfig.get('stake_amount') != btconfig.get('stake_amount')
or lastconfig.get('enable_protections') != btconfig.get('enable_protections')
or lastconfig.get('protections') != btconfig.get('protections', [])
or lastconfig.get('timeframe') != strat.timeframe
):
lastconfig['timerange'] = btconfig['timerange']
lastconfig['protections'] = btconfig.get('protections', [])
lastconfig['enable_protections'] = btconfig.get('enable_protections')
lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet')
lastconfig['timeframe'] = strat.timeframe
ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data()
ApiServer._bt.abort = False
min_date, max_date = ApiServer._bt.backtest_one_strategy(
strat, ApiServer._bt_data, ApiServer._bt_timerange)
ApiServer._bt.results = generate_backtest_stats(
ApiServer._bt_data, ApiServer._bt.all_results,
min_date=min_date, max_date=max_date)
logger.info("Backtest finished.")
except DependencyException as e:
logger.info(f"Backtesting caused an error: {e}")
pass
finally:
ApiServer._bgtask_running = False
background_tasks.add_task(run_backtest)
ApiServer._bgtask_running = True
return {
"status": "running",
"running": True,
"progress": 0,
"step": str(BacktestState.STARTUP),
"status_msg": "Backtest started",
}
@router.get('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_get_backtest():
"""
Get backtesting result.
Returns Result after backtesting has been ran.
"""
from freqtrade.persistence import LocalTrade
if ApiServer._bgtask_running:
return {
"status": "running",
"running": True,
"step": ApiServer._bt.progress.action if ApiServer._bt else str(BacktestState.STARTUP),
"progress": ApiServer._bt.progress.progress if ApiServer._bt else 0,
"trade_count": len(LocalTrade.trades),
"status_msg": "Backtest running",
}
if not ApiServer._bt:
return {
"status": "not_started",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest not yet executed"
}
return {
"status": "ended",
"running": False,
"status_msg": "Backtest ended",
"step": "finished",
"progress": 1,
"backtest_result": ApiServer._bt.results,
}
@router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_delete_backtest():
"""Reset backtesting"""
if ApiServer._bgtask_running:
return {
"status": "running",
"running": True,
"step": "",
"progress": 0,
"status_msg": "Backtest running",
}
if ApiServer._bt:
del ApiServer._bt
ApiServer._bt = None
del ApiServer._bt_data
ApiServer._bt_data = None
logger.info("Backtesting reset")
return {
"status": "reset",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest reset",
}
@router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest'])
def api_backtest_abort():
if not ApiServer._bgtask_running:
return {
"status": "not_running",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest ended",
}
ApiServer._bt.abort = True
return {
"status": "stopping",
"running": False,
"step": "",
"progress": 0,
"status_msg": "Backtest ended",
}

34
freqtrade/rpc/api_server/api_schemas.py
View File

@@ -67,12 +67,16 @@ class Profit(BaseModel):
profit_closed_ratio_mean: float
profit_closed_percent_sum: float
profit_closed_ratio_sum: float
profit_closed_percent: float
profit_closed_ratio: float
profit_closed_fiat: float
profit_all_coin: float
profit_all_percent_mean: float
profit_all_ratio_mean: float
profit_all_percent_sum: float
profit_all_ratio_sum: float
profit_all_percent: float
profit_all_ratio: float
profit_all_fiat: float
trade_count: int
closed_trade_count: int
@@ -115,20 +119,21 @@ class ShowConfig(BaseModel):
dry_run: bool
stake_currency: str
stake_amount: Union[float, str]
available_capital: Optional[float]
stake_currency_decimals: int
max_open_trades: int
minimal_roi: Dict[str, Any]
stoploss: float
trailing_stop: bool
stoploss: Optional[float]
trailing_stop: Optional[bool]
trailing_stop_positive: Optional[float]
trailing_stop_positive_offset: Optional[float]
trailing_only_offset_is_reached: Optional[bool]
use_custom_stoploss: Optional[bool]
timeframe: str
timeframe: Optional[str]
timeframe_ms: int
timeframe_min: int
exchange: str
strategy: str
strategy: Optional[str]
forcebuy_enabled: bool
ask_strategy: Dict[str, Any]
bid_strategy: Dict[str, Any]
@@ -313,3 +318,24 @@ class PairHistory(BaseModel):
json_encoders = {
datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT),
}
class BacktestRequest(BaseModel):
strategy: str
timeframe: Optional[str]
timerange: Optional[str]
max_open_trades: Optional[int]
stake_amount: Optional[Union[float, str]]
enable_protections: bool
dry_run_wallet: Optional[float]
class BacktestResponse(BaseModel):
status: str
running: bool
status_msg: str
step: str
progress: float
trade_count: Optional[float]
# TODO: Properly type backtestresult...
backtest_result: Optional[Dict[str, Any]]

5
freqtrade/rpc/api_server/api_v1.py
View File

@@ -1,3 +1,4 @@
import logging
from copy import deepcopy
from pathlib import Path
from typing import List, Optional
@@ -22,6 +23,8 @@ from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
from freqtrade.rpc.rpc import RPCException
logger = logging.getLogger(__name__)
# Public API, requires no auth.
router_public = APIRouter()
# Private API, protected by authentication
@@ -249,7 +252,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option
pair_interval = sorted(pair_interval, key=lambda x: x[0])
pairs = list({x[0] for x in pair_interval})
pairs.sort()
result = {
'length': len(pairs),
'pairs': pairs,

11
freqtrade/rpc/api_server/web_ui.py
View File

@@ -18,6 +18,17 @@ async def fallback():
return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html'))
@router_ui.get('/ui_version', include_in_schema=False)
async def ui_version():
from freqtrade.commands.deploy_commands import read_ui_version
uibase = Path(__file__).parent / 'ui/installed/'
version = read_ui_version(uibase)
return {
"version": version if version else "not_installed",
}
@router_ui.get('/{rest_of_path:path}', include_in_schema=False)
async def index_html(rest_of_path: str):
"""

63
freqtrade/rpc/api_server/webserver.py
View File

@@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
from starlette.responses import JSONResponse
from freqtrade.exceptions import OperationalException
from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer
from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler
@@ -28,17 +29,37 @@ class FTJSONResponse(JSONResponse):
class ApiServer(RPCHandler):
__instance = None
__initialized = False
_rpc: RPC
# Backtesting type: Backtesting
_bt = None
_bt_data = None
_bt_timerange = None
_bt_last_config: Dict[str, Any] = {}
_has_rpc: bool = False
_bgtask_running: bool = False
_config: Dict[str, Any] = {}
def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None:
super().__init__(rpc, config)
self._server = None
def __new__(cls, *args, **kwargs):
"""
This class is a singleton.
We'll only have one instance of it around.
"""
if ApiServer.__instance is None:
ApiServer.__instance = object.__new__(cls)
ApiServer.__initialized = False
return ApiServer.__instance
ApiServer._rpc = rpc
ApiServer._has_rpc = True
def __init__(self, config: Dict[str, Any], standalone: bool = False) -> None:
ApiServer._config = config
if self.__initialized and (standalone or self._standalone):
return
self._standalone: bool = standalone
self._server = None
ApiServer.__initialized = True
api_config = self._config['api_server']
self.app = FastAPI(title="Freqtrade API",
@@ -50,12 +71,33 @@ class ApiServer(RPCHandler):
self.start_api()
def add_rpc_handler(self, rpc: RPC):
"""
Attach rpc handler
"""
if not self._has_rpc:
ApiServer._rpc = rpc
ApiServer._has_rpc = True
else:
# This should not happen assuming we didn't mess up.
raise OperationalException('RPC Handler already attached.')
def cleanup(self) -> None:
""" Cleanup pending module resources """
if self._server:
ApiServer._has_rpc = False
del ApiServer._rpc
if self._server and not self._standalone:
logger.info("Stopping API Server")
self._server.cleanup()
@classmethod
def shutdown(cls):
cls.__initialized = False
del cls.__instance
cls.__instance = None
cls._has_rpc = False
cls._rpc = None
def send_msg(self, msg: Dict[str, str]) -> None:
pass
@@ -68,6 +110,7 @@ class ApiServer(RPCHandler):
def configure_app(self, app: FastAPI, config):
from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login
from freqtrade.rpc.api_server.api_backtest import router as api_backtest
from freqtrade.rpc.api_server.api_v1 import router as api_v1
from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public
from freqtrade.rpc.api_server.web_ui import router_ui
@@ -77,6 +120,9 @@ class ApiServer(RPCHandler):
app.include_router(api_v1, prefix="/api/v1",
dependencies=[Depends(http_basic_or_jwt_token)],
)
app.include_router(api_backtest, prefix="/api/v1",
dependencies=[Depends(http_basic_or_jwt_token)],
)
app.include_router(router_login, prefix="/api/v1", tags=["auth"])
# UI Router MUST be last!
app.include_router(router_ui, prefix='')
@@ -125,6 +171,9 @@ class ApiServer(RPCHandler):
)
try:
self._server = UvicornServer(uvconfig)
self._server.run_in_thread()
if self._standalone:
self._server.run()
else:
self._server.run_in_thread()
except Exception:
logger.exception("Api server failed to start.")

36
freqtrade/rpc/rpc.py
View File

@@ -106,6 +106,7 @@ class RPC:
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'stake_amount': config['stake_amount'],
'available_capital': config.get('available_capital'),
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {},
@@ -118,9 +119,9 @@ class RPC:
'bot_name': config.get('bot_name', 'freqtrade'),
'timeframe': config.get('timeframe'),
'timeframe_ms': timeframe_to_msecs(config['timeframe']
) if 'timeframe' in config else '',
) if 'timeframe' in config else 0,
'timeframe_min': timeframe_to_minutes(config['timeframe']
) if 'timeframe' in config else '',
) if 'timeframe' in config else 0,
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),
@@ -153,7 +154,8 @@ class RPC:
# calculate profit and send message to user
if trade.is_open:
try:
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
except (ExchangeError, PricingError):
current_rate = NAN
else:
@@ -212,7 +214,8 @@ class RPC:
for trade in trades:
# calculate profit and send message to user
try:
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
except (PricingError, ExchangeError):
current_rate = NAN
trade_percent = (100 * trade.calc_profit_ratio(current_rate))
@@ -271,10 +274,10 @@ class RPC:
'date': key,
'abs_profit': value["amount"],
'fiat_value': self._fiat_converter.convert_amount(
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
value['amount'],
stake_currency,
fiat_display_currency
) if self._fiat_converter else 0,
'trade_count': value["trades"],
}
for key, value in profit_days.items()
@@ -371,7 +374,8 @@ class RPC:
else:
# Get current rate
try:
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
except (PricingError, ExchangeError):
current_rate = NAN
profit_ratio = trade.calc_profit_ratio(rate=current_rate)
@@ -396,7 +400,12 @@ class RPC:
profit_all_coin_sum = round(sum(profit_all_coin), 8)
profit_all_ratio_mean = float(mean(profit_all_ratio) if profit_all_ratio else 0.0)
# Doing the sum is not right - overall profit needs to be based on initial capital
profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0
starting_balance = self._freqtrade.wallets.get_starting_balance()
profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance
profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance
profit_all_fiat = self._fiat_converter.convert_amount(
profit_all_coin_sum,
stake_currency,
@@ -412,12 +421,16 @@ class RPC:
'profit_closed_ratio_mean': profit_closed_ratio_mean,
'profit_closed_percent_sum': round(profit_closed_ratio_sum * 100, 2),
'profit_closed_ratio_sum': profit_closed_ratio_sum,
'profit_closed_ratio': profit_closed_ratio_fromstart,
'profit_closed_percent': round(profit_closed_ratio_fromstart * 100, 2),
'profit_closed_fiat': profit_closed_fiat,
'profit_all_coin': profit_all_coin_sum,
'profit_all_percent_mean': round(profit_all_ratio_mean * 100, 2),
'profit_all_ratio_mean': profit_all_ratio_mean,
'profit_all_percent_sum': round(profit_all_ratio_sum * 100, 2),
'profit_all_ratio_sum': profit_all_ratio_sum,
'profit_all_ratio': profit_all_ratio_fromstart,
'profit_all_percent': round(profit_all_ratio_fromstart * 100, 2),
'profit_all_fiat': profit_all_fiat,
'trade_count': len(trades),
'closed_trade_count': len([t for t in trades if not t.is_open]),
@@ -541,7 +554,8 @@ class RPC:
if not fully_canceled:
# Get current rate and execute sell
current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False)
current_rate = self._freqtrade.exchange.get_rate(
trade.pair, refresh=False, side="sell")
sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL)
self._freqtrade.execute_sell(trade, current_rate, sell_reason)
# ---- EOF def _exec_forcesell ----
@@ -761,7 +775,7 @@ class RPC:
sell_signals = 0
if has_content:
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000
dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000
# Move open to seperate column when signal for easy plotting
if 'buy' in dataframe.columns:
buy_mask = (dataframe['buy'] == 1)

11
freqtrade/rpc/rpc_manager.py
View File

@@ -1,5 +1,5 @@
"""
This module contains class to manage RPC communications (Telegram, Slack, ...)
This module contains class to manage RPC communications (Telegram, API, ...)
"""
import logging
from typing import Any, Dict, List
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
class RPCManager:
"""
Class to manage RPC objects (Telegram, Slack, ...)
Class to manage RPC objects (Telegram, API, ...)
"""
def __init__(self, freqtrade) -> None:
""" Initializes all enabled rpc modules """
@@ -36,15 +36,16 @@ class RPCManager:
if config.get('api_server', {}).get('enabled', False):
logger.info('Enabling rpc.api_server')
from freqtrade.rpc.api_server import ApiServer
self.registered_modules.append(ApiServer(self._rpc, config))
apiserver = ApiServer(config)
apiserver.add_rpc_handler(self._rpc)
self.registered_modules.append(apiserver)
def cleanup(self) -> None:
""" Stops all enabled rpc modules """
logger.info('Cleaning up rpc modules ...')
while self.registered_modules:
mod = self.registered_modules.pop()
logger.debug('Cleaning up rpc.%s ...', mod.name)
logger.info('Cleaning up rpc.%s ...', mod.name)
mod.cleanup()
del mod

27
freqtrade/rpc/telegram.py
View File

@@ -24,7 +24,7 @@ from freqtrade.__init__ import __version__
from freqtrade.constants import DUST_PER_COIN
from freqtrade.enums import RPCMessageType
from freqtrade.exceptions import OperationalException
from freqtrade.misc import chunks, round_coin_value
from freqtrade.misc import chunks, plural, round_coin_value
from freqtrade.rpc import RPC, RPCException, RPCHandler
@@ -494,11 +494,11 @@ class Telegram(RPCHandler):
start_date)
profit_closed_coin = stats['profit_closed_coin']
profit_closed_percent_mean = stats['profit_closed_percent_mean']
profit_closed_percent_sum = stats['profit_closed_percent_sum']
profit_closed_percent = stats['profit_closed_percent']
profit_closed_fiat = stats['profit_closed_fiat']
profit_all_coin = stats['profit_all_coin']
profit_all_percent_mean = stats['profit_all_percent_mean']
profit_all_percent_sum = stats['profit_all_percent_sum']
profit_all_percent = stats['profit_all_percent']
profit_all_fiat = stats['profit_all_fiat']
trade_count = stats['trade_count']
first_trade_date = stats['first_trade_date']
@@ -514,7 +514,7 @@ class Telegram(RPCHandler):
markdown_msg = ("*ROI:* Closed trades\n"
f"∙ `{round_coin_value(profit_closed_coin, stake_cur)} "
f"({profit_closed_percent_mean:.2f}%) "
f"({profit_closed_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"({profit_closed_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"∙ `{round_coin_value(profit_closed_fiat, fiat_disp_cur)}`\n")
else:
markdown_msg = "`No closed trade` \n"
@@ -523,7 +523,7 @@ class Telegram(RPCHandler):
f"*ROI:* All trades\n"
f"∙ `{round_coin_value(profit_all_coin, stake_cur)} "
f"({profit_all_percent_mean:.2f}%) "
f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"({profit_all_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n"
f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n"
f"*Total Trade Count:* `{trade_count}`\n"
f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* "
@@ -598,7 +598,10 @@ class Telegram(RPCHandler):
"Starting capital: "
f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n"
)
total_dust_balance = 0
total_dust_currencies = 0
for curr in result['currencies']:
curr_output = ''
if curr['est_stake'] > balance_dust_level:
curr_output = (
f"*{curr['currency']}:*\n"
@@ -607,9 +610,9 @@ class Telegram(RPCHandler):
f"\t`Pending: {curr['used']:.8f}`\n"
f"\t`Est. {curr['stake']}: "
f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n")
else:
curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} "
f"{curr['stake']} amount \n")
elif curr['est_stake'] <= balance_dust_level:
total_dust_balance += curr['est_stake']
total_dust_currencies += 1
# Handle overflowing message length
if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH:
@@ -618,6 +621,14 @@ class Telegram(RPCHandler):
else:
output += curr_output
if total_dust_balance > 0:
output += (
f"*{total_dust_currencies} Other "
f"{plural(total_dust_currencies, 'Currency', 'Currencies')} "
f"(< {balance_dust_level} {result['stake']}):*\n"
f"\t`Est. {result['stake']}: "
f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n")
output += ("\n*Estimated Value*:\n"
f"\t`{result['stake']}: {result['total']: .8f}`\n"
f"\t`{result['symbol']}: "

36
freqtrade/strategy/hyper.py
View File

@@ -5,8 +5,10 @@ This module defines a base class for auto-hyperoptable strategies.
import logging
from abc import ABC, abstractmethod
from contextlib import suppress
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union
from freqtrade.misc import deep_merge_dicts, json_load
from freqtrade.optimize.hyperopt_tools import HyperoptTools
@@ -333,10 +335,36 @@ class HyperStrategyMixin(object):
"""
Load Hyperoptable parameters
"""
self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt)
self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt)
params = self.load_params_from_file()
params = params.get('params', {})
self._ft_params_from_file = params
buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', None))
sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', None))
def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None:
self._load_params(buy_params, 'buy', hyperopt)
self._load_params(sell_params, 'sell', hyperopt)
def load_params_from_file(self) -> Dict:
filename_str = getattr(self, '__file__', '')
if not filename_str:
return {}
filename = Path(filename_str).with_suffix('.json')
if filename.is_file():
logger.info(f"Loading parameters from file {filename}")
try:
params = json_load(filename.open('r'))
if params.get('strategy_name') != self.__class__.__name__:
raise OperationalException('Invalid parameter file provided.')
return params
except ValueError:
logger.warning("Invalid parameter file format.")
return {}
logger.info("Found no parameter file.")
return {}
def _load_params(self, params: Dict, space: str, hyperopt: bool = False) -> None:
"""
Set optimizable parameter values.
:param params: Dictionary with new parameter values.
@@ -363,7 +391,7 @@ class HyperStrategyMixin(object):
else:
logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}')
def get_params_dict(self):
def get_no_optimize_params(self):
"""
Returns list of Parameters that are not part of the current optimize job
"""

18
freqtrade/strategy/interface.py
View File

@@ -62,6 +62,7 @@ class IStrategy(ABC, HyperStrategyMixin):
_populate_fun_len: int = 0
_buy_fun_len: int = 0
_sell_fun_len: int = 0
_ft_params_from_file: Dict = {}
# associated minimal roi
minimal_roi: Dict
@@ -303,6 +304,23 @@ class IStrategy(ABC, HyperStrategyMixin):
"""
return None
def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float,
proposed_stake: float, min_stake: float, max_stake: float,
**kwargs) -> float:
"""
Customize stake size for each new trade. This method is not called when edge module is
enabled.
:param pair: Pair that's currently analyzed
:param current_time: datetime object, containing the current datetime
:param current_rate: Rate, calculated based on pricing settings in ask_strategy.
:param proposed_stake: A stake amount proposed by the bot.
:param min_stake: Minimal stake size allowed by exchange.
:param max_stake: Balance available for trading.
:return: A stake size, which is between min_stake and max_stake.
"""
return proposed_stake
def informative_pairs(self) -> ListPairsWithTimeframes:
"""
Define additional, informative pair/interval combinations to be cached from the exchange.

48
freqtrade/templates/strategy_analysis_example.ipynb
View File

@@ -188,6 +188,52 @@
"trades.groupby(\"pair\")[\"sell_reason\"].value_counts()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Plotting daily profit / equity line"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)\n",
"\n",
"from freqtrade.configuration import Configuration\n",
"from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n",
"import plotly.express as px\n",
"import pandas as pd\n",
"\n",
"# strategy = 'SampleStrategy'\n",
"# config = Configuration.from_files([\"user_data/config.json\"])\n",
"# backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n",
"\n",
"stats = load_backtest_stats(backtest_dir)\n",
"strategy_stats = stats['strategy'][strategy]\n",
"\n",
"dates = []\n",
"profits = []\n",
"for date_profit in strategy_stats['daily_profit']:\n",
" dates.append(date_profit[0])\n",
" profits.append(date_profit[1])\n",
"\n",
"equity = 0\n",
"equity_daily = []\n",
"for daily_profit in profits:\n",
" equity_daily.append(equity)\n",
" equity += float(daily_profit)\n",
"\n",
"\n",
"df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n",
"\n",
"fig = px.line(df, x=\"dates\", y=\"equity_daily\")\n",
"fig.show()\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -329,7 +375,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.8.5"
},
"mimetype": "text/x-python",
"name": "python",

77
freqtrade/wallets.py
View File

@@ -70,9 +70,7 @@ class Wallets:
# If not backtesting...
# TODO: potentially remove the ._log workaround to determine backtest mode.
if self._log:
closed_trades = Trade.get_trades_proxy(is_open=False)
tot_profit = sum(
[trade.close_profit_abs for trade in closed_trades if trade.close_profit_abs])
tot_profit = Trade.get_total_closed_profit()
else:
tot_profit = LocalTrade.total_profit
tot_in_trades = sum([trade.stake_amount for trade in open_trades])
@@ -131,7 +129,41 @@ class Wallets:
def get_all_balances(self) -> Dict[str, Any]:
return self._wallets
def _get_available_stake_amount(self, val_tied_up: float) -> float:
def get_starting_balance(self) -> float:
"""
Retrieves starting balance - based on either available capital,
or by using current balance subtracting
"""
if "available_capital" in self._config:
return self._config['available_capital']
else:
tot_profit = Trade.get_total_closed_profit()
open_stakes = Trade.total_open_trades_stakes()
available_balance = self.get_free(self._config['stake_currency'])
return available_balance - tot_profit + open_stakes
def get_total_stake_amount(self):
"""
Return the total currently available balance in stake currency, including tied up stake and
respecting tradable_balance_ratio.
Calculated as
(<open_trade stakes> + free amount) * tradable_balance_ratio
"""
val_tied_up = Trade.total_open_trades_stakes()
if "available_capital" in self._config:
starting_balance = self._config['available_capital']
tot_profit = Trade.get_total_closed_profit()
available_amount = starting_balance + tot_profit
else:
# Ensure <tradable_balance_ratio>% is used from the overall balance
# Otherwise we'd risk lowering stakes with each open trade.
# (tied up + current free) * ratio) - tied up
available_amount = ((val_tied_up + self.get_free(self._config['stake_currency'])) *
self._config['tradable_balance_ratio'])
return available_amount
def get_available_stake_amount(self) -> float:
"""
Return the total currently available balance in stake currency,
respecting tradable_balance_ratio.
@@ -139,12 +171,8 @@ class Wallets:
(<open_trade stakes> + free amount) * tradable_balance_ratio - <open_trade stakes>
"""
# Ensure <tradable_balance_ratio>% is used from the overall balance
# Otherwise we'd risk lowering stakes with each open trade.
# (tied up + current free) * ratio) - tied up
available_amount = ((val_tied_up + self.get_free(self._config['stake_currency'])) *
self._config['tradable_balance_ratio']) - val_tied_up
return available_amount
free = self.get_free(self._config['stake_currency'])
return min(self.get_total_stake_amount() - Trade.total_open_trades_stakes(), free)
def _calculate_unlimited_stake_amount(self, available_amount: float,
val_tied_up: float) -> float:
@@ -193,7 +221,7 @@ class Wallets:
# Ensure wallets are uptodate.
self.update()
val_tied_up = Trade.total_open_trades_stakes()
available_amount = self._get_available_stake_amount(val_tied_up)
available_amount = self.get_available_stake_amount()
if edge:
stake_amount = edge.stake_amount(
@@ -209,3 +237,30 @@ class Wallets:
available_amount, val_tied_up)
return self._check_available_stake_amount(stake_amount, available_amount)
def _validate_stake_amount(self, pair, stake_amount, min_stake_amount):
if not stake_amount:
logger.debug(f"Stake amount is {stake_amount}, ignoring possible trade for {pair}.")
return 0
max_stake_amount = self.get_available_stake_amount()
if min_stake_amount > max_stake_amount:
if self._log:
logger.warning("Minimum stake amount > available balance.")
return 0
if min_stake_amount is not None and stake_amount < min_stake_amount:
stake_amount = min_stake_amount
if self._log:
logger.info(
f"Stake amount for pair {pair} is too small "
f"({stake_amount} < {min_stake_amount}), adjusting to {min_stake_amount}."
)
if stake_amount > max_stake_amount:
stake_amount = max_stake_amount
if self._log:
logger.info(
f"Stake amount for pair {pair} is too big "
f"({stake_amount} > {max_stake_amount}), adjusting to {max_stake_amount}."
)
return stake_amount