Merge branch 'develop' into backtest_live_models

.github/workflows/ci.yml

@@ -407,15 +407,6 @@ jobs:
       run: |
         build_helpers/publish_docker_multi.sh
 
-    - name: Discord notification
-      uses: rjstone/discord-webhook-notify@v1
-      if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && (github.event_name != 'schedule')
-      with:
-          severity: info
-          details: Deploy Succeeded!
-          webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}
-
-
   deploy_arm:
     needs: [ deploy ]
     # Only run on 64bit machines
@@ -443,3 +434,11 @@ jobs:
         BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
       run: |
         build_helpers/publish_docker_arm64.sh
+
+    - name: Discord notification
+      uses: rjstone/discord-webhook-notify@v1
+      if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && (github.event_name != 'schedule')
+      with:
+          severity: info
+          details: Deploy Succeeded!
+          webhookUrl: ${{ secrets.DISCORD_WEBHOOK }}

.pre-commit-config.yaml

@@ -34,7 +34,9 @@ repos:
         exclude: |
             (?x)^(
                 tests/.*|
-                .*\.svg
+                .*\.svg|
+                .*\.yml|
+                .*\.json
             )$
       - id: mixed-line-ending
       - id: debug-statements

config_examples/config_freqai.example.json

@@ -78,7 +78,7 @@
                 10,
                 20
             ],
-            "plot_feature_importance": false
+            "plot_feature_importances": 0
         },
         "data_split_parameters": {
             "test_size": 0.33,

docker/Dockerfile.jupyter

@@ -1,7 +1,8 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-RUN pip install jupyterlab --user --no-cache-dir
+# Pin jupyter-client to avoid tornado version conflict
+RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/docker-compose-jupyter.yml

@@ -10,7 +10,7 @@ services:
     ports:
       - "127.0.0.1:8888:8888"
     volumes:
-      - "./user_data:/freqtrade/user_data"
+      - "../user_data:/freqtrade/user_data"
     # Default command used when running `docker compose up`
     command: >
       jupyter lab --port=8888 --ip 0.0.0.0 --allow-root

freqtrade/constants.py

@@ -552,7 +552,7 @@ CONF_SCHEMA = {
                         "weight_factor": {"type": "number", "default": 0},
                         "principal_component_analysis": {"type": "boolean", "default": False},
                         "use_SVM_to_remove_outliers": {"type": "boolean", "default": False},
-                        "plot_feature_importance": {"type": "boolean", "default": False},
+                        "plot_feature_importances": {"type": "integer", "default": 0},
                         "svm_params": {"type": "object",
                                        "properties": {
                                            "shuffle": {"type": "boolean", "default": False},

freqtrade/rpc/api_server/api_backtest.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from typing import Any, Dict, List
 
 from fastapi import APIRouter, BackgroundTasks, Depends
+from fastapi.exceptions import HTTPException
 
 from freqtrade.configuration.config_validation import validate_config_consistency
 from freqtrade.data.btanalysis import get_backtest_resultlist, load_and_merge_backtest_result
@@ -31,6 +32,9 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac
     if ApiServer._bgtask_running:
         raise RPCException('Bot Background task already running')
 
+    if ':' in bt_settings.strategy:
+        raise HTTPException(status_code=500, detail="base64 encoded strategies are not allowed.")
+
     btconfig = deepcopy(config)
     settings = dict(bt_settings)
     # Pydantic models will contain all keys, but non-provided ones are None

freqtrade/rpc/api_server/api_v1.py

@@ -265,6 +265,8 @@ def list_strategies(config=Depends(get_config)):
 
 @router.get('/strategy/{strategy}', response_model=StrategyResponse, tags=['strategy'])
 def get_strategy(strategy: str, config=Depends(get_config)):
+    if ":" in strategy:
+        raise HTTPException(status_code=500, detail="base64 encoded strategies are not allowed.")
 
     config_ = deepcopy(config)
     from freqtrade.resolvers.strategy_resolver import StrategyResolver

freqtrade/rpc/rpc.py

@@ -25,7 +25,7 @@ from freqtrade.exceptions import ExchangeError, PricingError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs
 from freqtrade.loggers import bufferHandler
 from freqtrade.misc import decimals_per_coin, shorten_date
-from freqtrade.persistence import PairLocks, Trade
+from freqtrade.persistence import Order, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -166,9 +166,9 @@ class RPC:
         else:
             results = []
             for trade in trades:
-                order = None
+                order: Optional[Order] = None
                 if trade.open_order_id:
-                    order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
+                    order = trade.select_order_by_order_id(trade.open_order_id)
                 # calculate profit and send message to user
                 if trade.is_open:
                     try:
@@ -219,7 +219,7 @@ class RPC:
                     stoploss_entry_dist=stoploss_entry_dist,
                     stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8),
                     open_order='({} {} rem={:.8f})'.format(
-                        order['type'], order['side'], order['remaining']
+                        order.order_type, order.side, order.remaining
                     ) if order else None,
                 ))
                 results.append(trade_dict)
@@ -773,6 +773,9 @@ class RPC:
             is_short = trade.is_short
             if not self._freqtrade.strategy.position_adjustment_enable:
                 raise RPCException(f'position for {pair} already open - id: {trade.id}')
+        else:
+            if Trade.get_open_trade_count() >= self._config['max_open_trades']:
+                raise RPCException("Maximum number of trades is reached.")
 
         if not stake_amount:
             # gen stake amount

tests/rpc/test_rpc.py

@@ -45,7 +45,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None:
 
     freqtradebot.enter_positions()
     trades = Trade.get_open_trades()
-    trades[0].open_order_id = None
     freqtradebot.exit_positions(trades)
 
     results = rpc._rpc_trade_status()
@@ -1031,6 +1030,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None:
 
 def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open) -> None:
     default_conf['force_entry_enable'] = True
+    default_conf['max_open_trades'] = 0
     mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
     buy_mm = MagicMock(return_value=limit_buy_order_open)
     mocker.patch.multiple(
@@ -1045,6 +1045,10 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
     patch_get_signal(freqtradebot)
     rpc = RPC(freqtradebot)
     pair = 'ETH/BTC'
+    with pytest.raises(RPCException, match='Maximum number of trades is reached.'):
+        rpc._rpc_force_entry(pair, None)
+    freqtradebot.config['max_open_trades'] = 5
+
     trade = rpc._rpc_force_entry(pair, None)
     assert isinstance(trade, Trade)
     assert trade.pair == pair

tests/rpc/test_rpc_apiserver.py

@@ -1477,6 +1477,10 @@ def test_api_strategy(botclient):
     rc = client_get(client, f"{BASE_URI}/strategy/NoStrat")
     assert_response(rc, 404)
 
+    # Disallow base64 strategies
+    rc = client_get(client, f"{BASE_URI}/strategy/xx:cHJpbnQoImhlbGxvIHdvcmxkIik=")
+    assert_response(rc, 500)
+
 
 def test_list_available_pairs(botclient):
     ftbot, client = botclient
@@ -1650,6 +1654,11 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir):
     assert not result['running']
     assert result['status_msg'] == 'Backtest reset'
 
+    # Disallow base64 strategies
+    data['strategy'] = "xx:cHJpbnQoImhlbGxvIHdvcmxkIik="
+    rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data))
+    assert_response(rc, 500)
+
 
 def test_api_backtest_history(botclient, mocker, testdatadir):
     ftbot, client = botclient