merge develop into feat/freqai-rl-dev

.pre-commit-config.yaml

@@ -15,9 +15,9 @@ repos:
         additional_dependencies:
           - types-cachetools==5.2.1
           - types-filelock==3.2.7
-          - types-requests==2.28.11.4
+          - types-requests==2.28.11.5
           - types-tabulate==0.9.0.0
-          - types-python-dateutil==2.8.19.3
+          - types-python-dateutil==2.8.19.4
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort

docs/configuration.md

@@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d
 ### Using proxy with Freqtrade
 
 To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values.
+This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests.
 
 ``` bash
 export HTTP_PROXY="http://addr:port"
@@ -672,11 +673,13 @@ export HTTPS_PROXY="http://addr:port"
 freqtrade
 ```
 
-#### Proxy just exchange requests
+#### Proxy exchange requests
 
-To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration.
+To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration.
 
 ``` json
+{ 
+  "exchange": {
     "ccxt_config": {
     "aiohttp_proxy": "http://addr:port",
     "proxies": {
@@ -684,6 +687,7 @@ To use a proxy just for exchange connections (skips/ignores telegram and coingec
       "https": "http://addr:port"
     },
   }
+}
 ```
 
 ## Next step

docs/producer-consumer.md

@@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
                 "name": "default", // This can be any name you'd like, default is "default"
                 "host": "127.0.0.1", // The host from your producer's api_server config
                 "port": 8080, // The port from your producer's api_server config
+                "secure": false, // Use a secure websockets connection, default false
                 "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config
             }
         ],
@@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect
 | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.<br> **Datatype:** string
 | `producers.host` | **Required.** The hostname or IP address from your producer.<br> **Datatype:** string
 | `producers.port` | **Required.** The port matching the above host.<br> **Datatype:** string
+| `producers.secure` | **Optional.**  Use ssl in websockets connection. Default False.<br> **Datatype:** string
 | `producers.ws_token` | **Required.**  `ws_token` as configured on the producer.<br> **Datatype:** string
 | | **Optional settings**
 | `wait_timeout` | Timeout until we ping again if no message is received. <br>*Defaults to `300`.*<br> **Datatype:** Integer - in seconds.

docs/rest-api.md

@@ -389,6 +389,44 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th
 }
 ```
 
+#### Reverse Proxy setup
+
+When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is):
+
+Please make sure to replace `<freqtrade_listen_ip>` (and the subsequent port) with the IP and Port matching your configuration/setup.
+
+```
+http {
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    #...
+
+    server {
+        #...
+
+        location / {
+            proxy_http_version 1.1;
+            proxy_pass http://<freqtrade_listen_ip>:8080;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header Host $host;
+        }
+    }
+}
+```
+
+To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets.
+
+- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/)
+- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy)
+
+!!! Tip "SSL certificates"
+    You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies.
+    While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel).
+
 ### OpenAPI interface
 
 To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration.

freqtrade/constants.py

@@ -512,6 +512,7 @@ CONF_SCHEMA = {
                                 'minimum': 0,
                                 'maximum': 65535
                             },
+                            'secure': {'type': 'boolean', 'default': False},
                             'ws_token': {'type': 'string'},
                         },
                         'required': ['name', 'host', 'ws_token']

freqtrade/freqtradebot.py

@@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin):
             trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value
             self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order,
                                     stoploss_order=True)
-            # Lock pair for one candle to prevent immediate rebuys
-            self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
-                                    reason='Auto lock')
             self._notify_exit(trade, "stoploss", True)
+            self.handle_protections(trade.pair, trade.trade_direction)
             return True
 
         if trade.open_order_id or not trade.is_open:
@@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin):
         trade.close_rate_requested = limit
         trade.exit_reason = exit_reason
 
-        if not sub_trade_amt:
-            # Lock pair for one candle to prevent immediate re-trading
-            self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
-                                    reason='Auto lock')
-
         self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj)
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') in ('closed', 'expired'):
@@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin):
             self._notify_enter(trade, order, fill=True, sub_trade=sub_trade)
 
     def handle_protections(self, pair: str, side: LongShort) -> None:
+        # Lock pair for one candle to prevent immediate rebuys
+        self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock')
         prot_trig = self.protections.stop_per_pair(pair, side=side)
         if prot_trig:
             msg = {'type': RPCMessageType.PROTECTION_TRIGGER, }

freqtrade/misc.py

@@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union
 from typing.io import IO
 from urllib.parse import urlparse
 
-import pandas
+import orjson
+import pandas as pd
 import rapidjson
 
 from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN
@@ -256,29 +257,37 @@ def parse_db_uri_for_logging(uri: str):
     return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@')
 
 
-def dataframe_to_json(dataframe: pandas.DataFrame) -> str:
+def dataframe_to_json(dataframe: pd.DataFrame) -> str:
     """
     Serialize a DataFrame for transmission over the wire using JSON
     :param dataframe: A pandas DataFrame
     :returns: A JSON string of the pandas DataFrame
     """
-    return dataframe.to_json(orient='split')
+    # https://github.com/pandas-dev/pandas/issues/24889
+    # https://github.com/pandas-dev/pandas/issues/40443
+    # We need to convert to a dict to avoid mem leak
+    def default(z):
+        if isinstance(z, pd.Timestamp):
+            return z.timestamp() * 1e3
+        raise TypeError
+
+    return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8')
 
 
-def json_to_dataframe(data: str) -> pandas.DataFrame:
+def json_to_dataframe(data: str) -> pd.DataFrame:
     """
     Deserialize JSON into a DataFrame
     :param data: A JSON string
     :returns: A pandas DataFrame from the JSON string
     """
-    dataframe = pandas.read_json(data, orient='split')
+    dataframe = pd.read_json(data, orient='split')
     if 'date' in dataframe.columns:
-        dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True)
+        dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True)
 
     return dataframe
 
 
-def remove_entry_exit_signals(dataframe: pandas.DataFrame):
+def remove_entry_exit_signals(dataframe: pd.DataFrame):
     """
     Remove Entry and Exit signals from a DataFrame
 

freqtrade/rpc/api_server/webserver.py

@@ -194,6 +194,9 @@ class ApiServer(RPCHandler):
         try:
             while True:
                 logger.debug("Getting queue messages...")
+                if (qsize := async_queue.qsize()) > 20:
+                    # If the queue becomes too big for too long, this may indicate a problem.
+                    logger.warning(f"Queue size now {qsize}")
                 # Get data from queue
                 message: WSMessageSchemaType = await async_queue.get()
                 logger.debug(f"Found message of type: {message.get('type')}")

freqtrade/rpc/api_server/ws/channel.py

@@ -77,6 +77,7 @@ class WebSocketChannel:
         # until self.drain_timeout for the relay to drain the outgoing queue
         # We can't use asyncio.wait_for here because the queue may have been created with a
         # different eventloop
+        if not self.is_closed():
             start = time.time()
             while self.queue.full():
                 await asyncio.sleep(1)
@@ -91,6 +92,8 @@ class WebSocketChannel:
 
             # If we got here everything is ok
             return True
+        else:
+            return False
 
     async def recv(self):
         """
@@ -109,14 +112,14 @@ class WebSocketChannel:
         Close the WebSocketChannel
         """
 
+        self._closed.set()
+        self._relay_task.cancel()
+
         try:
             await self.raw_websocket.close()
         except Exception:
             pass
 
-        self._closed.set()
-        self._relay_task.cancel()
-
     def is_closed(self) -> bool:
         """
         Closed flag

freqtrade/rpc/external_message_consumer.py

@@ -31,6 +31,7 @@ class Producer(TypedDict):
     name: str
     host: str
     port: int
+    secure: bool
     ws_token: str
 
 
@@ -180,7 +181,8 @@ class ExternalMessageConsumer:
                 host, port = producer['host'], producer['port']
                 token = producer['ws_token']
                 name = producer['name']
-                ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
+                scheme = 'wss' if producer.get('secure', False) else 'ws'
+                ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
 
                 # This will raise InvalidURI if the url is bad
                 async with websockets.connect(

freqtrade/rpc/rpc.py

@@ -774,6 +774,9 @@ class RPC:
             is_short = trade.is_short
             if not self._freqtrade.strategy.position_adjustment_enable:
                 raise RPCException(f'position for {pair} already open - id: {trade.id}')
+            if trade.open_order_id is not None:
+                raise RPCException(f'position for {pair} already open - id: {trade.id} '
+                                   f'and has open order {trade.open_order_id}')
         else:
             if Trade.get_open_trade_count() >= self._config['max_open_trades']:
                 raise RPCException("Maximum number of trades is reached.")

requirements-dev.txt

@@ -9,7 +9,7 @@
 coveralls==3.3.1
 flake8==5.0.4
 flake8-tidy-imports==4.8.0
-mypy==0.990
+mypy==0.991
 pre-commit==2.20.0
 pytest==7.2.0
 pytest-asyncio==0.20.2
@@ -20,14 +20,14 @@ isort==5.10.1
 # For datetime mocking
 time-machine==2.8.2
 # fastapi testing
-httpx==0.23.0
+httpx==0.23.1
 
 # Convert jupyter notebooks to markdown documents
-nbconvert==7.2.4
+nbconvert==7.2.5
 
 # mypy types
 types-cachetools==5.2.1
 types-filelock==3.2.7
-types-requests==2.28.11.4
+types-requests==2.28.11.5
 types-tabulate==0.9.0.0
-types-python-dateutil==2.8.19.3
+types-python-dateutil==2.8.19.4

requirements.txt

@@ -1,8 +1,8 @@
-numpy==1.23.4
+numpy==1.23.5
 pandas==1.5.1
 pandas-ta==0.3.14b
 
-ccxt==2.1.75
+ccxt==2.1.96
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==38.0.1; platform_machine == 'armv7l'
 cryptography==38.0.3; platform_machine != 'armv7l'
@@ -30,7 +30,7 @@ py_find_1st==1.1.5
 # Load ticker files 30% faster
 python-rapidjson==1.9
 # Properly format api responses
-orjson==3.8.1
+orjson==3.8.2
 
 # Notify systemd
 sdnotify==0.3.2
@@ -38,7 +38,7 @@ sdnotify==0.3.2
 # API Server
 fastapi==0.87.0
 pydantic==1.10.2
-uvicorn==0.19.0
+uvicorn==0.20.0
 pyjwt==2.6.0
 aiofiles==22.1.0
 psutil==5.9.4

scripts/ws_client.py

@@ -199,6 +199,7 @@ async def create_client(
         host,
         port,
         token,
+        scheme='ws',
         name='default',
         protocol=ClientProtocol(),
         sleep_time=10,
@@ -211,13 +212,14 @@ async def create_client(
     :param host: The host
     :param port: The port
     :param token: The websocket auth token
+    :param scheme: `ws` for most connections, `wss` for ssl
     :param name: The name of the producer
     :param **kwargs: Any extra kwargs passed to websockets.connect
     """
 
     while 1:
         try:
-            websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}"
+            websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}"
             logger.info(f"Attempting to connect to {name} @ {host}:{port}")
 
             async with websockets.connect(websocket_url, **kwargs) as ws:
@@ -304,6 +306,7 @@ async def _main(args):
         producer['host'],
         producer['port'],
         producer['ws_token'],
+        'wss' if producer.get('secure', False) else 'ws',
         producer['name'],
         sleep_time=sleep_time,
         ping_timeout=ping_timeout,

setup.sh

@@ -83,7 +83,7 @@ function updateenv() {
     dev=$REPLY
     if [[ $REPLY =~ ^[Yy]$ ]]
     then
-        REQUIREMENTS_FREQAI="-r requirements-freqai.txt"
+        REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517"
         read -p "Do you also want dependencies for freqai-rl (~700mb additional space required) [y/N]? "
         dev=$REPLY
         if [[ $REPLY =~ ^[Yy]$ ]]

tests/exchange/test_exchange.py

@@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees(
     assert order1['fee']['rate'] == fee
 
 
-@pytest.mark.parametrize("side,startprice,endprice", [
+@pytest.mark.parametrize("side,price,filled", [
-    ("buy", 25.563, 25.566),
+    # order_book_l2_usd spread:
-    ("sell", 25.566, 25.563)
+    # best ask: 25.566
+    # best bid: 25.563
+    ("buy", 25.563, False),
+    ("buy", 25.566, True),
+    ("sell", 25.566, False),
+    ("sell", 25.563, True),
 ])
 @pytest.mark.parametrize("exchange_name", EXCHANGES)
-def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice,
+def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled,
                                          exchange_name, order_book_l2_usd):
     default_conf['dry_run'] = True
     exchange = get_patched_exchange(mocker, default_conf, id=exchange_name)
@@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
         ordertype='limit',
         side=side,
         amount=1,
-        rate=startprice,
+        rate=price,
         leverage=1.0
     )
     assert order_book_l2_usd.call_count == 1
@@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice,
     assert order["side"] == side
     assert order["type"] == "limit"
     assert order["symbol"] == "LTC/USDT"
+    assert order["average"] == price
+    assert order['status'] == 'open' if not filled else 'closed'
     order_book_l2_usd.reset_mock()
 
+    # fetch order again...
     order_closed = exchange.fetch_dry_run_order(order['id'])
-    assert order_book_l2_usd.call_count == 1
+    assert order_book_l2_usd.call_count == (1 if not filled else 0)
-    assert order_closed['status'] == 'open'
+    assert order_closed['status'] == ('open' if not filled else 'closed')
-    assert not order['fee']
+    assert order_closed['filled'] == (0 if not filled else 1)
-    assert order_closed['filled'] == 0
 
     order_book_l2_usd.reset_mock()
-    order_closed['price'] = endprice
-
-    order_closed = exchange.fetch_dry_run_order(order['id'])
-    assert order_closed['status'] == 'closed'
-    assert order['fee']
-    assert order_closed['filled'] == 1
-    assert order_closed['filled'] == order_closed['amount']
 
     # Empty orderbook test
     mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',

tests/rpc/test_rpc.py

@@ -1066,6 +1066,11 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open
     trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
     assert trade.stake_amount == 0.05
     assert trade.buy_tag == 'force_entry'
+    assert trade.open_order_id == 'mocked_limit_buy'
+
+    freqtradebot.strategy.position_adjustment_enable = True
+    with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'):
+        rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05)
 
     # Test not buying
     pair = 'XRP/BTC'