Merge branch 'feat/short' into pr/samgermain/5567

build_helpers/install_ta-lib.sh

@@ -11,8 +11,13 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
   && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \
   && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
   && ./configure --prefix=${INSTALL_LOC}/ \
-  && make -j$(nproc) \
+  && make
-  && which sudo && sudo make install || make install
+  if [ $? -ne 0 ]; then
+    echo "Failed building ta-lib."
+    cd .. && rm -rf ./ta-lib/
+    exit 1
+  fi
+  which sudo && sudo make install || make install
   if [ -x "$(command -v apt-get)" ]; then
     echo "Updating library path using ldconfig"
     sudo ldconfig

docs/hyperopt.md

@@ -51,6 +51,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                           [--print-all] [--no-color] [--print-json] [-j JOBS]
                           [--random-state INT] [--min-trades INT]
                           [--hyperopt-loss NAME] [--disable-param-export]
+                          [--ignore-missing-spaces]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -118,6 +119,9 @@ optional arguments:
                         MaxDrawDownHyperOptLoss
   --disable-param-export
                         Disable automatic hyperopt parameter export.
+  --ignore-missing-spaces, --ignore-unparameterized-spaces
+                        Suppress errors for any requested Hyperopt spaces that
+                        do not contain any parameters.
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/includes/pairlists.md

@@ -194,17 +194,22 @@ Trade count is used as a tie breaker.
 You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window).
 Not defining this parameter (or setting it to 0) will use all-time performance.
 
+The optional `min_profit` parameter defines the minimum profit a pair must have to be considered.
+Pairs below this level will be filtered out.
+Using this parameter without `minutes` is highly discouraged, as it can lead to an empty pairlist without without a way to recover.
+
 ```json
 "pairlists": [
     // ...
     {
         "method": "PerformanceFilter",
-        "minutes": 1440  // rolling 24h
+        "minutes": 1440,  // rolling 24h
+        "min_profit": 0.01
     }
 ],
 ```
 
-!!! Note
+!!! Warning "Backtesting"
     `PerformanceFilter` does not support backtesting mode.
 
 #### PrecisionFilter

freqtrade/commands/arguments.py

@@ -31,7 +31,8 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path",
                                         "epochs", "spaces", "print_all",
                                         "print_colorized", "print_json", "hyperopt_jobs",
                                         "hyperopt_random_state", "hyperopt_min_trades",
-                                        "hyperopt_loss", "disableparamexport"]
+                                        "hyperopt_loss", "disableparamexport",
+                                        "hyperopt_ignore_missing_space"]
 
 ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"]
 

freqtrade/commands/cli_options.py

@@ -558,4 +558,10 @@ AVAILABLE_CLI_OPTIONS = {
         help='Do not print epoch details header.',
         action='store_true',
     ),
+    "hyperopt_ignore_missing_space": Arg(
+        "--ignore-missing-spaces", "--ignore-unparameterized-spaces",
+        help=("Suppress errors for any requested Hyperopt spaces "
+              "that do not contain any parameters."),
+        action="store_true",
+    ),
 }

freqtrade/configuration/configuration.py

@@ -369,6 +369,9 @@ class Configuration:
         self._args_to_config(config, argname='hyperopt_show_no_header',
                              logstring='Parameter --no-header detected: {}')
 
+        self._args_to_config(config, argname="hyperopt_ignore_missing_space",
+                             logstring="Paramter --ignore-missing-space detected: {}")
+
     def _process_plot_options(self, config: Dict[str, Any]) -> None:
 
         self._args_to_config(config, argname='pairs',

freqtrade/optimize/hyperopt.py

@@ -258,6 +258,7 @@ class Hyperopt:
         if HyperoptTools.has_space(self.config, 'trailing'):
             logger.debug("Hyperopt has 'trailing' space")
             self.trailing_space = self.custom_hyperopt.trailing_space()
+
         self.dimensions = (self.buy_space + self.sell_space + self.protection_space
                            + self.roi_space + self.stoploss_space + self.trailing_space)
 

freqtrade/optimize/hyperopt_auto.py

@@ -3,6 +3,7 @@ HyperOptAuto class.
 This module implements a convenience auto-hyperopt class, which can be used together with strategies
  that implement IHyperStrategy interface.
 """
+import logging
 from contextlib import suppress
 from typing import Callable, Dict, List
 
@@ -15,12 +16,19 @@ with suppress(ImportError):
 from freqtrade.optimize.hyperopt_interface import EstimatorType, IHyperOpt
 
 
-def _format_exception_message(space: str) -> str:
+logger = logging.getLogger(__name__)
-    raise OperationalException(
+
-        f"The '{space}' space is included into the hyperoptimization "
+
-        f"but no parameter for this space was not found in your Strategy. "
+def _format_exception_message(space: str, ignore_missing_space: bool) -> None:
-        f"Please make sure to have parameters for this space enabled for optimization "
+    msg = (f"The '{space}' space is included into the hyperoptimization "
-        f"or remove the '{space}' space from hyperoptimization.")
+           f"but no parameter for this space was not found in your Strategy. "
+           )
+    if ignore_missing_space:
+        logger.warning(msg + "This space will be ignored.")
+    else:
+        raise OperationalException(
+            msg + f"Please make sure to have parameters for this space enabled for optimization "
+            f"or remove the '{space}' space from hyperoptimization.")
 
 
 class HyperOptAuto(IHyperOpt):
@@ -48,13 +56,16 @@ class HyperOptAuto(IHyperOpt):
             if attr.optimize:
                 yield attr.get_space(attr_name)
 
-    def _get_indicator_space(self, category):
+    def _get_indicator_space(self, category) -> List:
         # TODO: is this necessary, or can we call "generate_space" directly?
         indicator_space = list(self._generate_indicator_space(category))
         if len(indicator_space) > 0:
             return indicator_space
         else:
-            _format_exception_message(category)
+            _format_exception_message(
+                category,
+                self.config.get("hyperopt_ignore_missing_space", False))
+            return []
 
     def buy_indicator_space(self) -> List['Dimension']:
         return self._get_indicator_space('buy')

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -21,6 +21,7 @@ class PerformanceFilter(IPairList):
         super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
 
         self._minutes = pairlistconfig.get('minutes', 0)
+        self._min_profit = pairlistconfig.get('min_profit', None)
 
     @property
     def needstickers(self) -> bool:
@@ -68,6 +69,14 @@ class PerformanceFilter(IPairList):
         sorted_df = list_df.merge(performance, on='pair', how='left')\
             .fillna(0).sort_values(by=['count', 'pair'], ascending=True)\
             .sort_values(by=['profit'], ascending=False)
+        if self._min_profit is not None:
+            removed = sorted_df[sorted_df['profit'] < self._min_profit]
+            for _, row in removed.iterrows():
+                self.log_once(
+                    f"Removing pair {row['pair']} since {row['profit']} is "
+                    f"below {self._min_profit}", logger.info)
+            sorted_df = sorted_df[sorted_df['profit'] >= self._min_profit]
+
         pairlist = sorted_df['pair'].tolist()
 
         return pairlist

freqtrade/rpc/api_server/deps.py

@@ -1,5 +1,6 @@
 from typing import Any, Dict, Optional
 
+from freqtrade.persistence import Trade
 from freqtrade.rpc.rpc import RPC, RPCException
 
 from .webserver import ApiServer
@@ -14,6 +15,7 @@ def get_rpc_optional() -> Optional[RPC]:
 def get_rpc() -> Optional[RPC]:
     _rpc = get_rpc_optional()
     if _rpc:
+        Trade.query.session.rollback()
         return _rpc
     else:
         raise RPCException('Bot is not in the correct state')

freqtrade/rpc/telegram.py

@@ -25,6 +25,7 @@ from freqtrade.constants import DUST_PER_COIN
 from freqtrade.enums import RPCMessageType
 from freqtrade.exceptions import OperationalException
 from freqtrade.misc import chunks, plural, round_coin_value
+from freqtrade.persistence import Trade
 from freqtrade.rpc import RPC, RPCException, RPCHandler
 
 
@@ -59,7 +60,8 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]:
                 update.message.chat_id
             )
             return wrapper
-
+        # Rollback session to avoid getting data stored in a transaction.
+        Trade.query.session.rollback()
         logger.debug(
             'Executing handler: %s for chat_id: %s',
             command_handler.__name__,

mkdocs.yml

@@ -55,8 +55,8 @@ theme:
       primary: "blue grey"
       accent: "tear"
       toggle:
-        icon: material/toggle-switch-off-outline
+        icon: material/toggle-switch
-        name: Switch to dark mode
+        name: Switch to light mode
 extra_css:
   - "stylesheets/ft.extra.css"
 extra_javascript:

tests/conftest.py

@@ -285,6 +285,7 @@ def create_mock_trades(fee, is_short: bool, use_db: bool = True):
     add_trade(trade)
 
     if use_db:
+        Trade.commit()
         Trade.query.session.flush()
 
 
@@ -357,6 +358,7 @@ def create_mock_trades_usdt(fee, use_db: bool = True):
     add_trade(trade)
 
     if use_db:
+        Trade.commit()
         Trade.query.session.flush()
 
 

tests/optimize/test_hyperopt.py

@@ -705,7 +705,7 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non
     assert hasattr(hyperopt, "position_stacking")
 
 
-def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None:
+def test_simplified_interface_all_failed(mocker, hyperopt_conf, caplog) -> None:
     mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock())
     mocker.patch('freqtrade.optimize.hyperopt.file_dump_json')
     mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data',
@@ -727,7 +727,13 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None:
     hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={})
 
     with pytest.raises(OperationalException, match=r"The 'protection' space is included into *"):
-        hyperopt.start()
+        hyperopt.init_spaces()
+
+    hyperopt.config['hyperopt_ignore_missing_space'] = True
+    caplog.clear()
+    hyperopt.init_spaces()
+    assert log_has_re(r"The 'protection' space is included into *", caplog)
+    assert hyperopt.protection_space == []
 
 
 def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None:

tests/plugins/test_pairlist.py

@@ -666,11 +666,11 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None:
 
 @pytest.mark.usefixtures("init_persistence")
 # TODO-lev: @pytest.mark.parametrize('is_short', [True, False])
-def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None:
+def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee, caplog) -> None:
     whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC')
     whitelist_conf['pairlists'] = [
         {"method": "StaticPairList"},
-        {"method": "PerformanceFilter", "minutes": 60}
+        {"method": "PerformanceFilter", "minutes": 60, "min_profit": 0.01}
     ]
     mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True))
     exchange = get_patched_exchange(mocker, whitelist_conf)
@@ -682,7 +682,8 @@ def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee) -> None:
     with time_machine.travel("2021-09-01 05:00:00 +00:00") as t:
         create_mock_trades(fee, False)
         pm.refresh_pairlist()
-        assert pm.whitelist == ['XRP/BTC', 'ETH/BTC', 'TKN/BTC']
+        assert pm.whitelist == ['XRP/BTC']
+        assert log_has_re(r'Removing pair .* since .* is below .*', caplog)
 
         # Move to "outside" of lookback window, so original sorting is restored.
         t.move_to("2021-09-01 07:00:00 +00:00")

tests/rpc/test_rpc_apiserver.py

@@ -95,7 +95,7 @@ def test_api_not_found(botclient):
     assert rc.json() == {"detail": "Not Found"}
 
 
-def test_api_ui_fallback(botclient):
+def test_api_ui_fallback(botclient, mocker):
     ftbot, client = botclient
 
     rc = client_get(client, "/favicon.ico")
@@ -109,9 +109,16 @@ def test_api_ui_fallback(botclient):
     rc = client_get(client, "/something")
     assert rc.status_code == 200
 
-    # Test directory traversal
+    # Test directory traversal without mock
     rc = client_get(client, '%2F%2F%2Fetc/passwd')
     assert rc.status_code == 200
+    # Allow both fallback or real UI
+    assert '`freqtrade install-ui`' in rc.text or '<!DOCTYPE html>' in rc.text
+
+    mocker.patch.object(Path, 'is_file', MagicMock(side_effect=[True, False]))
+    rc = client_get(client, '%2F%2F%2Fetc/passwd')
+    assert rc.status_code == 200
+
     assert '`freqtrade install-ui`' in rc.text
 
 
@@ -617,10 +624,11 @@ def test_api_delete_trade(botclient, mocker, fee, markets):
     assert_response(rc, 502)
 
     create_mock_trades(fee, False)
-    Trade.query.session.flush()
+
     ftbot.strategy.order_types['stoploss_on_exchange'] = True
     trades = Trade.query.all()
     trades[1].stoploss_order_id = '1234'
+    Trade.commit()
     assert len(trades) > 2
 
     rc = client_delete(client, f"{BASE_URI}/trades/1")

tests/test_freqtradebot.py

@@ -3738,9 +3738,9 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
-            'bid': 0.00000172,
+            'bid': 2.0,
-            'ask': 0.00000173,
+            'ask': 2.0,
-            'last': 0.00000172
+            'last': 2.0
         }),
         create_order=MagicMock(side_effect=[
             limit_order_open[enter_side(is_short)],
@@ -3770,7 +3770,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_order, limit_
     # Test if buy-signal is absent
     patch_get_signal(freqtrade, enter_long=False, exit_long=not is_short, exit_short=is_short)
     assert freqtrade.handle_trade(trade) is True
-    assert trade.sell_reason == SellType.SELL_SIGNAL.value
+    assert trade.sell_reason == SellType.ROI.value
 
 
 def test_get_real_amount_quote(default_conf_usdt, trades_for_order, buy_order_fee, fee, caplog,