Merge branch 'develop' into feat/short

This commit is contained in:
Matthias 2021-10-17 10:40:08 +02:00
commit 69fdb8edfa
17 changed files with 94 additions and 28 deletions

View File

@ -11,8 +11,13 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
&& curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \ && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \
&& curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \ && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
&& ./configure --prefix=${INSTALL_LOC}/ \ && ./configure --prefix=${INSTALL_LOC}/ \
&& make -j$(nproc) \ && make
&& which sudo && sudo make install || make install if [ $? -ne 0 ]; then
echo "Failed building ta-lib."
cd .. && rm -rf ./ta-lib/
exit 1
fi
which sudo && sudo make install || make install
if [ -x "$(command -v apt-get)" ]; then if [ -x "$(command -v apt-get)" ]; then
echo "Updating library path using ldconfig" echo "Updating library path using ldconfig"
sudo ldconfig sudo ldconfig

View File

@ -51,6 +51,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
[--print-all] [--no-color] [--print-json] [-j JOBS] [--print-all] [--no-color] [--print-json] [-j JOBS]
[--random-state INT] [--min-trades INT] [--random-state INT] [--min-trades INT]
[--hyperopt-loss NAME] [--disable-param-export] [--hyperopt-loss NAME] [--disable-param-export]
[--ignore-missing-spaces]
optional arguments: optional arguments:
-h, --help show this help message and exit -h, --help show this help message and exit
@ -118,6 +119,9 @@ optional arguments:
MaxDrawDownHyperOptLoss MaxDrawDownHyperOptLoss
--disable-param-export --disable-param-export
Disable automatic hyperopt parameter export. Disable automatic hyperopt parameter export.
--ignore-missing-spaces, --ignore-unparameterized-spaces
Suppress errors for any requested Hyperopt spaces that
do not contain any parameters.
Common arguments: Common arguments:
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages). -v, --verbose Verbose mode (-vv for more, -vvv to get all messages).

View File

@ -194,17 +194,22 @@ Trade count is used as a tie breaker.
You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window). You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window).
Not defining this parameter (or setting it to 0) will use all-time performance. Not defining this parameter (or setting it to 0) will use all-time performance.
The optional `min_profit` parameter defines the minimum profit a pair must have to be considered.
Pairs below this level will be filtered out.
Using this parameter without `minutes` is highly discouraged, as it can lead to an empty pairlist without without a way to recover.
```json ```json
"pairlists": [ "pairlists": [
// ... // ...
{ {
"method": "PerformanceFilter", "method": "PerformanceFilter",
"minutes": 1440 // rolling 24h "minutes": 1440, // rolling 24h
"min_profit": 0.01
} }
], ],
``` ```
!!! Note !!! Warning "Backtesting"
`PerformanceFilter` does not support backtesting mode. `PerformanceFilter` does not support backtesting mode.
#### PrecisionFilter #### PrecisionFilter

View File

@ -31,7 +31,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
"epochs", "spaces", "print_all", "epochs", "spaces", "print_all",
"print_colorized", "print_json", "hyperopt_jobs", "print_colorized", "print_json", "hyperopt_jobs",
"hyperopt_random_state", "hyperopt_min_trades", "hyperopt_random_state", "hyperopt_min_trades",
"hyperopt_loss", "disableparamexport"] "hyperopt_loss", "disableparamexport",
"hyperopt_ignore_missing_space"]
ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]

View File

@ -558,4 +558,10 @@ AVAILABLE_CLI_OPTIONS = {
help='Do not print epoch details header.', help='Do not print epoch details header.',
action='store_true', action='store_true',
), ),
"hyperopt_ignore_missing_space": Arg(
"--ignore-missing-spaces", "--ignore-unparameterized-spaces",
help=("Suppress errors for any requested Hyperopt spaces "
"that do not contain any parameters."),
action="store_true",
),
} }

View File

@ -369,6 +369,9 @@ class Configuration:
self._args_to_config(config, argname='hyperopt_show_no_header', self._args_to_config(config, argname='hyperopt_show_no_header',
logstring='Parameter --no-header detected: {}') logstring='Parameter --no-header detected: {}')
self._args_to_config(config, argname="hyperopt_ignore_missing_space",
logstring="Paramter --ignore-missing-space detected: {}")
def _process_plot_options(self, config: Dict[str, Any]) -> None: def _process_plot_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='pairs', self._args_to_config(config, argname='pairs',

View File

@ -258,6 +258,7 @@ class Hyperopt:
if HyperoptTools.has_space(self.config, 'trailing'): if HyperoptTools.has_space(self.config, 'trailing'):
logger.debug("Hyperopt has 'trailing' space") logger.debug("Hyperopt has 'trailing' space")
self.trailing_space = self.custom_hyperopt.trailing_space() self.trailing_space = self.custom_hyperopt.trailing_space()
self.dimensions = (self.buy_space + self.sell_space + self.protection_space self.dimensions = (self.buy_space + self.sell_space + self.protection_space
+ self.roi_space + self.stoploss_space + self.trailing_space) + self.roi_space + self.stoploss_space + self.trailing_space)

View File

@ -3,6 +3,7 @@ HyperOptAuto class.
This module implements a convenience auto-hyperopt class, which can be used together with strategies This module implements a convenience auto-hyperopt class, which can be used together with strategies
that implement IHyperStrategy interface. that implement IHyperStrategy interface.
""" """
import logging
from contextlib import suppress from contextlib import suppress
from typing import Callable, Dict, List from typing import Callable, Dict, List
@ -15,11 +16,18 @@ with suppress(ImportError):
from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
def _format_exception_message(space: str) -> str: logger = logging.getLogger(__name__)
raise OperationalException(
f"The '{space}' space is included into the hyperoptimization "
def _format_exception_message(space: str, ignore_missing_space: bool) -> None:
msg = (f"The '{space}' space is included into the hyperoptimization "
f"but no parameter for this space was not found in your Strategy. " f"but no parameter for this space was not found in your Strategy. "
f"Please make sure to have parameters for this space enabled for optimization " )
if ignore_missing_space:
logger.warning(msg + "This space will be ignored.")
else:
raise OperationalException(
msg + f"Please make sure to have parameters for this space enabled for optimization "
f"or remove the '{space}' space from hyperoptimization.") f"or remove the '{space}' space from hyperoptimization.")
@ -48,13 +56,16 @@ class HyperOptAuto(IHyperOpt):
if attr.optimize: if attr.optimize:
yield attr.get_space(attr_name) yield attr.get_space(attr_name)
def _get_indicator_space(self, category): def _get_indicator_space(self, category) -> List:
# TODO: is this necessary, or can we call "generate_space" directly? # TODO: is this necessary, or can we call "generate_space" directly?
indicator_space = list(self._generate_indicator_space(category)) indicator_space = list(self._generate_indicator_space(category))
if len(indicator_space) > 0: if len(indicator_space) > 0:
return indicator_space return indicator_space
else: else:
_format_exception_message(category) _format_exception_message(
category,
self.config.get("hyperopt_ignore_missing_space", False))
return []
def buy_indicator_space(self) -> List['Dimension']: def buy_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy') return self._get_indicator_space('buy')

View File

@ -21,6 +21,7 @@ class PerformanceFilter(IPairList):
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._minutes = pairlistconfig.get('minutes', 0) self._minutes = pairlistconfig.get('minutes', 0)
self._min_profit = pairlistconfig.get('min_profit', None)
@property @property
def needstickers(self) -> bool: def needstickers(self) -> bool:
@ -68,6 +69,14 @@ class PerformanceFilter(IPairList):
sorted_df = list_df.merge(performance, on='pair', how='left')\ sorted_df = list_df.merge(performance, on='pair', how='left')\
.fillna(0).sort_values(by=['count', 'pair'], ascending=True)\ .fillna(0).sort_values(by=['count', 'pair'], ascending=True)\
.sort_values(by=['profit'], ascending=False) .sort_values(by=['profit'], ascending=False)
if self._min_profit is not None:
removed = sorted_df[sorted_df['profit'] < self._min_profit]
for _, row in removed.iterrows():
self.log_once(
f"Removing pair {row['pair']} since {row['profit']} is "
f"below {self._min_profit}", logger.info)
sorted_df = sorted_df[sorted_df['profit'] >= self._min_profit]
pairlist = sorted_df['pair'].tolist() pairlist = sorted_df['pair'].tolist()
return pairlist return pairlist

View File

@ -1,5 +1,6 @@
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from freqtrade.persistence import Trade
from freqtrade.rpc.rpc import RPC, RPCException from freqtrade.rpc.rpc import RPC, RPCException
from .webserver import ApiServer from .webserver import ApiServer
@ -14,6 +15,7 @@ def get_rpc_optional() -> Optional[RPC]:
def get_rpc() -> Optional[RPC]: def get_rpc() -> Optional[RPC]:
_rpc = get_rpc_optional() _rpc = get_rpc_optional()
if _rpc: if _rpc:
Trade.query.session.rollback()
return _rpc return _rpc
else: else:
raise RPCException('Bot is not in the correct state') raise RPCException('Bot is not in the correct state')

View File

@ -25,6 +25,7 @@ from freqtrade.constants import DUST_PER_COIN
from freqtrade.enums import RPCMessageType from freqtrade.enums import RPCMessageType
from freqtrade.exceptions import OperationalException from freqtrade.exceptions import OperationalException
from freqtrade.misc import chunks, plural, round_coin_value from freqtrade.misc import chunks, plural, round_coin_value
from freqtrade.persistence import Trade
from freqtrade.rpc import RPC, RPCException, RPCHandler from freqtrade.rpc import RPC, RPCException, RPCHandler
@ -59,7 +60,8 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]:
update.message.chat_id update.message.chat_id
) )
return wrapper return wrapper
# Rollback session to avoid getting data stored in a transaction.
Trade.query.session.rollback()
logger.debug( logger.debug(
'Executing handler: %s for chat_id: %s', 'Executing handler: %s for chat_id: %s',
command_handler.__name__, command_handler.__name__,

View File

@ -55,8 +55,8 @@ theme:
primary: "blue grey" primary: "blue grey"
accent: "tear" accent: "tear"
toggle: toggle:
icon: material/toggle-switch-off-outline icon: material/toggle-switch
name: Switch to dark mode name: Switch to light mode
extra_css: extra_css:
- "stylesheets/ft.extra.css" - "stylesheets/ft.extra.css"
extra_javascript: extra_javascript:

View File

@ -271,6 +271,7 @@ def create_mock_trades(fee, use_db: bool = True):
add_trade(trade) add_trade(trade)
if use_db: if use_db:
Trade.commit()
Trade.query.session.flush() Trade.query.session.flush()
@ -343,6 +344,7 @@ def create_mock_trades_usdt(fee, use_db: bool = True):
add_trade(trade) add_trade(trade)
if use_db: if use_db:
Trade.commit()
Trade.query.session.flush() Trade.query.session.flush()

View File

@ -705,7 +705,7 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non
assert hasattr(hyperopt, "position_stacking") assert hasattr(hyperopt, "position_stacking")
def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: def test_simplified_interface_all_failed(mocker, hyperopt_conf, caplog) -> None:
mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json')
mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
@ -727,7 +727,13 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None:
hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
with pytest.raises(OperationalException, match=r"The 'protection' space is included into *"): with pytest.raises(OperationalException, match=r"The 'protection' space is included into *"):
hyperopt.start() hyperopt.init_spaces()
hyperopt.config['hyperopt_ignore_missing_space'] = True
caplog.clear()
hyperopt.init_spaces()
assert log_has_re(r"The 'protection' space is included into *", caplog)
assert hyperopt.protection_space == []
def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None:

View File

@ -665,11 +665,11 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
@pytest.mark.usefixtures("init_persistence") @pytest.mark.usefixtures("init_persistence")
def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None: def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee, caplog) -> None:
whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC') whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC')
whitelist_conf['pairlists'] = [ whitelist_conf['pairlists'] = [
{"method": "StaticPairList"}, {"method": "StaticPairList"},
{"method": "PerformanceFilter", "minutes": 60} {"method": "PerformanceFilter", "minutes": 60, "min_profit": 0.01}
] ]
mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
exchange = get_patched_exchange(mocker, whitelist_conf) exchange = get_patched_exchange(mocker, whitelist_conf)
@ -681,7 +681,8 @@ def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None:
with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
create_mock_trades(fee) create_mock_trades(fee)
pm.refresh_pairlist() pm.refresh_pairlist()
assert pm.whitelist == ['XRP/BTC', 'ETH/BTC', 'TKN/BTC'] assert pm.whitelist == ['XRP/BTC']
assert log_has_re(r'Removing pair .* since .* is below .*', caplog)
# Move to "outside" of lookback window, so original sorting is restored. # Move to "outside" of lookback window, so original sorting is restored.
t.move_to("2021-09-01 07:00:00 +00:00") t.move_to("2021-09-01 07:00:00 +00:00")

View File

@ -95,7 +95,7 @@ def test_api_not_found(botclient):
assert rc.json() == {"detail": "Not Found"} assert rc.json() == {"detail": "Not Found"}
def test_api_ui_fallback(botclient): def test_api_ui_fallback(botclient, mocker):
ftbot, client = botclient ftbot, client = botclient
rc = client_get(client, "/favicon.ico") rc = client_get(client, "/favicon.ico")
@ -109,9 +109,16 @@ def test_api_ui_fallback(botclient):
rc = client_get(client, "/something") rc = client_get(client, "/something")
assert rc.status_code == 200 assert rc.status_code == 200
# Test directory traversal # Test directory traversal without mock
rc = client_get(client, '%2F%2F%2Fetc/passwd') rc = client_get(client, '%2F%2F%2Fetc/passwd')
assert rc.status_code == 200 assert rc.status_code == 200
# Allow both fallback or real UI
assert '`freqtrade install-ui`' in rc.text or '<!DOCTYPE html>' in rc.text
mocker.patch.object(Path, 'is_file', MagicMock(side_effect=[True, False]))
rc = client_get(client, '%2F%2F%2Fetc/passwd')
assert rc.status_code == 200
assert '`freqtrade install-ui`' in rc.text assert '`freqtrade install-ui`' in rc.text
@ -613,10 +620,11 @@ def test_api_delete_trade(botclient, mocker, fee, markets):
assert_response(rc, 502) assert_response(rc, 502)
create_mock_trades(fee) create_mock_trades(fee)
Trade.query.session.flush()
ftbot.strategy.order_types['stoploss_on_exchange'] = True ftbot.strategy.order_types['stoploss_on_exchange'] = True
trades = Trade.query.all() trades = Trade.query.all()
trades[1].stoploss_order_id = '1234' trades[1].stoploss_order_id = '1234'
Trade.commit()
assert len(trades) > 2 assert len(trades) > 2
rc = client_delete(client, f"{BASE_URI}/trades/1") rc = client_delete(client, f"{BASE_URI}/trades/1")

View File

@ -3389,9 +3389,9 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usd
mocker.patch.multiple( mocker.patch.multiple(
'freqtrade.exchange.Exchange', 'freqtrade.exchange.Exchange',
fetch_ticker=MagicMock(return_value={ fetch_ticker=MagicMock(return_value={
'bid': 0.00000172, 'bid': 2.0,
'ask': 0.00000173, 'ask': 2.0,
'last': 0.00000172 'last': 2.0
}), }),
create_order=MagicMock(side_effect=[ create_order=MagicMock(side_effect=[
limit_buy_order_usdt_open, limit_buy_order_usdt_open,
@ -3419,7 +3419,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usd
# Test if buy-signal is absent # Test if buy-signal is absent
patch_get_signal(freqtrade, enter_long=False, exit_long=True) patch_get_signal(freqtrade, enter_long=False, exit_long=True)
assert freqtrade.handle_trade(trade) is True assert freqtrade.handle_trade(trade) is True
assert trade.sell_reason == SellType.SELL_SIGNAL.value assert trade.sell_reason == SellType.ROI.value
def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog, def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog,