Merge branch 'develop' into feat/refactor-ws

.pre-commit-config.yaml

@@ -15,9 +15,9 @@ repos:
         additional_dependencies:
           - types-cachetools==5.2.1
           - types-filelock==3.2.7
-          - types-requests==2.28.11.4
+          - types-requests==2.28.11.5
           - types-tabulate==0.9.0.0
-          - types-python-dateutil==2.8.19.3
+          - types-python-dateutil==2.8.19.4
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

docs/configuration.md

@@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 ### Using proxy with Freqtrade
 
 To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
+This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests.
 
 ``` bash
 export HTTP_PROXY="http://addr:port"
@@ -672,17 +673,20 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-#### Proxy just exchange requests
+#### Proxy exchange requests
 
-To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration.
+To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
 
 ``` json
-"ccxt_config": {
+{ 
+  "exchange": {
+    "ccxt_config": {
     "aiohttp_proxy": "http://addr:port",
     "proxies": {
-        "http": "http://addr:port",
+      "http": "http://addr:port",
-        "https": "http://addr:port"
+      "https": "http://addr:port"
     },
+  }
 }
 ```
 

docs/producer-consumer.md

@@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
                 "name": "default", // This can be any name you'd like, default is "default"
                 "host": "127.0.0.1", // The host from your producer's api_server config
                 "port": 8080, // The port from your producer's api_server config
+                "secure": false, // Use a secure websockets connection, default false
                 "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config
             }
         ],
@@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
 | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string
 | `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string
 | `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string
+| `producers.secure` | **Optional.**  Use ssl in websockets connection. Default False.<br> **Datatype:** string
 | `producers.ws_token` | **Required.**  `ws_token` as configured on the producer.<br> **Datatype:** string
 | | **Optional settings**
 | `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds.

docs/rest-api.md

@@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th
 }
 ```
 
+#### Reverse Proxy setup
+
+When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is):
+
+Please make sure to replace `<freqtrade_listen_ip>` (and the subsequent port) with the IP and Port matching your configuration/setup.
+
+```
+http {
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    #...
+
+    server {
+        #...
+
+        location / {
+            proxy_http_version 1.1;
+            proxy_pass http://<freqtrade_listen_ip>:8080;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header Host $host;
+        }
+    }
+}
+```
+
+To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets.
+
+- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/)
+- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy)
+
+!!! Tip "SSL certificates"
+    You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies.
+    While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel).
+
 ### OpenAPI interface
 
 To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.

freqtrade/constants.py

@@ -512,6 +512,7 @@ CONF_SCHEMA = {
                                 'minimum': 0,
                                 'maximum': 65535
                             },
+                            'secure': {'type': 'boolean', 'default': False},
                             'ws_token': {'type': 'string'},
                         },
                         'required': ['name', 'host', 'ws_token']

freqtrade/freqtradebot.py

@@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin):
             trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
             self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
                                     stoploss_order=True)
-            # Lock pair for one candle to prevent immediate rebuys
-            self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
-                                    reason='Auto lock')
             self._notify_exit(trade, "stoploss", True)
+            self.handle_protections(trade.pair, trade.trade_direction)
             return True
 
         if trade.open_order_id or not trade.is_open:
@@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin):
         trade.close_rate_requested = limit
         trade.exit_reason = exit_reason
 
-        if not sub_trade_amt:
-            # Lock pair for one candle to prevent immediate re-trading
-            self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
-                                    reason='Auto lock')
-
         self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') in ('closed', 'expired'):
@@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin):
             self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
 
     def handle_protections(self, pair: str, side: LongShort) -> None:
+        # Lock pair for one candle to prevent immediate rebuys
+        self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock')
         prot_trig = self.protections.stop_per_pair(pair, side=side)
         if prot_trig:
             msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }

freqtrade/rpc/external_message_consumer.py

@@ -31,6 +31,7 @@ class Producer(TypedDict):
     name: str
     host: str
     port: int
+    secure: bool
     ws_token: str
 
 
@@ -180,7 +181,8 @@ class ExternalMessageConsumer:
                 host, port = producer['host'], producer['port']
                 token = producer['ws_token']
                 name = producer['name']
-                ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
+                scheme = 'wss' if producer.get('secure', False) else 'ws'
+                ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
 
                 # This will raise InvalidURI if the url is bad
                 async with websockets.connect(

requirements-dev.txt

@@ -8,7 +8,7 @@
 coveralls==3.3.1
 flake8==5.0.4
 flake8-tidy-imports==4.8.0
-mypy==0.990
+mypy==0.991
 pre-commit==2.20.0
 pytest==7.2.0
 pytest-asyncio==0.20.2
@@ -19,14 +19,14 @@ isort==5.10.1
 # For datetime mocking
 time-machine==2.8.2
 # fastapi testing
-httpx==0.23.0
+httpx==0.23.1
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.2.4
+nbconvert==7.2.5
 
 # mypy types
 types-cachetools==5.2.1
 types-filelock==3.2.7
-types-requests==2.28.11.4
+types-requests==2.28.11.5
 types-tabulate==0.9.0.0
-types-python-dateutil==2.8.19.3
+types-python-dateutil==2.8.19.4

requirements.txt

@@ -1,8 +1,8 @@
-numpy==1.23.4
+numpy==1.23.5
 pandas==1.5.1
 pandas-ta==0.3.14b
 
-ccxt==2.1.75
+ccxt==2.1.96
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==38.0.1; platform_machine == 'armv7l'
 cryptography==38.0.3; platform_machine != 'armv7l'
@@ -30,7 +30,7 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.9
 # Properly format api responses
-orjson==3.8.1
+orjson==3.8.2
 
 # Notify systemd
 sdnotify==0.3.2
@@ -38,7 +38,7 @@ sdnotify==0.3.2
 # API Server
 fastapi==0.87.0
 pydantic==1.10.2
-uvicorn==0.19.0
+uvicorn==0.20.0
 pyjwt==2.6.0
 aiofiles==22.1.0
 psutil==5.9.4

scripts/ws_client.py

@@ -199,6 +199,7 @@ async def create_client(
         host,
         port,
         token,
+        scheme='ws',
         name='default',
         protocol=ClientProtocol(),
         sleep_time=10,
@@ -211,13 +212,14 @@ async def create_client(
     :param host: The host
     :param port: The port
     :param token: The websocket auth token
+    :param scheme: `ws` for most connections, `wss` for ssl
     :param name: The name of the producer
     :param **kwargs: Any extra kwargs passed to websockets.connect
     """
 
     while 1:
         try:
-            websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
+            websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
             logger.info(f"Attempting to connect to {name} @ {host}:{port}")
 
             async with websockets.connect(websocket_url, **kwargs) as ws:
@@ -304,6 +306,7 @@ async def _main(args):
         producer['host'],
         producer['port'],
         producer['ws_token'],
+        'wss' if producer.get('secure', False) else 'ws',
         producer['name'],
         sleep_time=sleep_time,
         ping_timeout=ping_timeout,

setup.sh

@@ -82,7 +82,7 @@ function updateenv() {
     dev=$REPLY
     if [[ $REPLY =~ ^[Yy]$ ]]
     then
-        REQUIREMENTS_FREQAI="-r requirements-freqai.txt"
+        REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517"
     fi
 
     ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI}

tests/exchange/test_exchange.py

@@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees(
     assert order1['fee']['rate'] == fee
 
 
-@pytest.mark.parametrize("side,startprice,endprice", [
+@pytest.mark.parametrize("side,price,filled", [
-    ("buy", 25.563, 25.566),
+    # order_book_l2_usd spread:
-    ("sell", 25.566, 25.563)
+    # best ask: 25.566
+    # best bid: 25.563
+    ("buy", 25.563, False),
+    ("buy", 25.566, True),
+    ("sell", 25.566, False),
+    ("sell", 25.563, True),
 ])
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
-def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice,
+def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled,
                                          exchange_name, order_book_l2_usd):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
         ordertype='limit',
         side=side,
         amount=1,
-        rate=startprice,
+        rate=price,
         leverage=1.0
     )
     assert order_book_l2_usd.call_count == 1
@@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
     assert order["side"] == side
     assert order["type"] == "limit"
     assert order["symbol"] == "LTC/USDT"
+    assert order["average"] == price
+    assert order['status'] == 'open' if not filled else 'closed'
     order_book_l2_usd.reset_mock()
 
+    # fetch order again...
     order_closed = exchange.fetch_dry_run_order(order['id'])
-    assert order_book_l2_usd.call_count == 1
+    assert order_book_l2_usd.call_count == (1 if not filled else 0)
-    assert order_closed['status'] == 'open'
+    assert order_closed['status'] == ('open' if not filled else 'closed')
-    assert not order['fee']
+    assert order_closed['filled'] == (0 if not filled else 1)
-    assert order_closed['filled'] == 0
 
     order_book_l2_usd.reset_mock()
-    order_closed['price'] = endprice
-
-    order_closed = exchange.fetch_dry_run_order(order['id'])
-    assert order_closed['status'] == 'closed'
-    assert order['fee']
-    assert order_closed['filled'] == 1
-    assert order_closed['filled'] == order_closed['amount']
 
     # Empty orderbook test
     mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',