From 6a1655c047b88bba462677566f6943819ffc83a7 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:26:54 -0700 Subject: [PATCH 01/31] support ssl connections in emc --- freqtrade/constants.py | 1 + freqtrade/rpc/external_message_consumer.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 534d06fd4..cfac98ebd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -512,6 +512,7 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 65535 }, + 'secure': {'type': 'boolean', 'default': False}, 'ws_token': {'type': 'string'}, }, 'required': ['name', 'host', 'ws_token'] diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index b978407e4..d9aed7d52 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -31,6 +31,7 @@ class Producer(TypedDict): name: str host: str port: int + secure: bool ws_token: str @@ -180,7 +181,8 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + scheme = 'wss' if producer['secure'] else 'ws' + ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad async with websockets.connect( From 86e094e39b77e9c6ea8b8e88fa7a84339be151e0 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:33:42 -0700 Subject: [PATCH 02/31] update docs --- docs/producer-consumer.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/producer-consumer.md b/docs/producer-consumer.md index b69406edf..88e34d0d6 100644 --- a/docs/producer-consumer.md +++ b/docs/producer-consumer.md @@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect "name": "default", // This can be any name you'd like, default is "default" "host": "127.0.0.1", // The host from your producer's api_server config "port": 8080, // The port from your producer's api_server config + "secure": false, // Use a secure websockets connection, default false "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config } ], @@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.
**Datatype:** string | `producers.host` | **Required.** The hostname or IP address from your producer.
**Datatype:** string | `producers.port` | **Required.** The port matching the above host.
**Datatype:** string +| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.
**Datatype:** string | `producers.ws_token` | **Required.** `ws_token` as configured on the producer.
**Datatype:** string | | **Optional settings** | `wait_timeout` | Timeout until we ping again if no message is received.
*Defaults to `300`.*
**Datatype:** Integer - in seconds. From 1380ddd066c46784ea0ffd518f919ef8da7972f5 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:44:02 -0700 Subject: [PATCH 03/31] update ws client --- scripts/ws_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..70dbead14 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -199,6 +199,7 @@ async def create_client( host, port, token, + scheme='ws', name='default', protocol=ClientProtocol(), sleep_time=10, @@ -211,13 +212,14 @@ async def create_client( :param host: The host :param port: The port :param token: The websocket auth token + :param scheme: `ws` for most connections, `wss` for ssl :param name: The name of the producer :param **kwargs: Any extra kwargs passed to websockets.connect """ while 1: try: - websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" logger.info(f"Attempting to connect to {name} @ {host}:{port}") async with websockets.connect(websocket_url, **kwargs) as ws: @@ -304,6 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], + 'wss' if producer['secure'] else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, From a993cb512de6422d0c186e27c214502a5356c5eb Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 10:22:55 -0700 Subject: [PATCH 04/31] change to get call in ws_client --- scripts/ws_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 70dbead14..5d27f512e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -306,7 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], - 'wss' if producer['secure'] else 'ws', + 'wss' if producer.get('secure', False) else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, From 875e9ab447a6d3d165b13a64f95eb79f94daec74 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 11:59:03 -0700 Subject: [PATCH 05/31] change df serialization to avoid mem leak --- freqtrade/misc.py | 7 +++++-- freqtrade/rpc/api_server/ws/serializer.py | 7 ++++++- scripts/ws_client.py | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 49d33d46f..308f0b32d 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -262,7 +262,10 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str: :param dataframe: A pandas DataFrame :returns: A JSON string of the pandas DataFrame """ - return dataframe.to_json(orient='split') + # https://github.com/pandas-dev/pandas/issues/24889 + # https://github.com/pandas-dev/pandas/issues/40443 + # We need to convert to a dict to avoid mem leak + return dataframe.to_dict(orient='tight') def json_to_dataframe(data: str) -> pandas.DataFrame: @@ -271,7 +274,7 @@ def json_to_dataframe(data: str) -> pandas.DataFrame: :param data: A JSON string :returns: A pandas DataFrame from the JSON string """ - dataframe = pandas.read_json(data, orient='split') + dataframe = pandas.DataFrame.from_dict(data, orient='tight') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) diff --git a/freqtrade/rpc/api_server/ws/serializer.py b/freqtrade/rpc/api_server/ws/serializer.py index 6c402a100..8d06746f7 100644 --- a/freqtrade/rpc/api_server/ws/serializer.py +++ b/freqtrade/rpc/api_server/ws/serializer.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import orjson import rapidjson -from pandas import DataFrame +from pandas import DataFrame, Timestamp from freqtrade.misc import dataframe_to_json, json_to_dataframe from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy @@ -52,6 +52,11 @@ def _json_default(z): '__type__': 'dataframe', '__value__': dataframe_to_json(z) } + # Pandas returns a Timestamp object, we need to + # convert it to a timestamp int (with ms) for orjson + # to handle it + if isinstance(z, Timestamp): + return z.timestamp() * 1e3 raise TypeError diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..ff437e63e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -101,7 +101,7 @@ def json_deserialize(message): :param message: The message to deserialize """ def json_to_dataframe(data: str) -> pandas.DataFrame: - dataframe = pandas.read_json(data, orient='split') + dataframe = pandas.DataFrame.from_dict(data, orient='tight') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) From ce43fa5f431092bff5a23b1586f3381d1ba4cfac Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 12:03:11 -0700 Subject: [PATCH 06/31] small fix to websocketchannel and relay --- freqtrade/rpc/api_server/ws/channel.py | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/freqtrade/rpc/api_server/ws/channel.py b/freqtrade/rpc/api_server/ws/channel.py index 88b4db9ba..4eef738d4 100644 --- a/freqtrade/rpc/api_server/ws/channel.py +++ b/freqtrade/rpc/api_server/ws/channel.py @@ -77,21 +77,24 @@ class WebSocketChannel: # until self.drain_timeout for the relay to drain the outgoing queue # We can't use asyncio.wait_for here because the queue may have been created with a # different eventloop - start = time.time() - while self.queue.full(): - await asyncio.sleep(1) - if (time.time() - start) > self.drain_timeout: + if not self.is_closed(): + start = time.time() + while self.queue.full(): + await asyncio.sleep(1) + if (time.time() - start) > self.drain_timeout: + return False + + # If for some reason the queue is still full, just return False + try: + self.queue.put_nowait(data) + except asyncio.QueueFull: return False - # If for some reason the queue is still full, just return False - try: - self.queue.put_nowait(data) - except asyncio.QueueFull: + # If we got here everything is ok + return True + else: return False - # If we got here everything is ok - return True - async def recv(self): """ Receive data on the wrapped websocket @@ -109,14 +112,14 @@ class WebSocketChannel: Close the WebSocketChannel """ + self._closed.set() + self._relay_task.cancel() + try: await self.raw_websocket.close() except Exception: pass - self._closed.set() - self._relay_task.cancel() - def is_closed(self) -> bool: """ Closed flag From 4c7bb79c86d6167659e8dca643f7bf9cd55519af Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Nov 2022 13:59:29 +0100 Subject: [PATCH 07/31] Restore prior data transfer format --- freqtrade/misc.py | 20 +++++++++++++------- scripts/ws_client.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 308f0b32d..2d2c7513a 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Mapping, Union from typing.io import IO from urllib.parse import urlparse -import pandas +import orjson +import pandas as pd import rapidjson from freqtrade.constants import DECIMAL_PER_COIN_FALLBACK, DECIMALS_PER_COIN @@ -256,7 +257,7 @@ def parse_db_uri_for_logging(uri: str): return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') -def dataframe_to_json(dataframe: pandas.DataFrame) -> str: +def dataframe_to_json(dataframe: pd.DataFrame) -> str: """ Serialize a DataFrame for transmission over the wire using JSON :param dataframe: A pandas DataFrame @@ -265,23 +266,28 @@ def dataframe_to_json(dataframe: pandas.DataFrame) -> str: # https://github.com/pandas-dev/pandas/issues/24889 # https://github.com/pandas-dev/pandas/issues/40443 # We need to convert to a dict to avoid mem leak - return dataframe.to_dict(orient='tight') + def default(z): + if isinstance(z, pd.Timestamp): + return z.timestamp() * 1e3 + raise TypeError + + return str(orjson.dumps(dataframe.to_dict(orient='split'), default=default), 'utf-8') -def json_to_dataframe(data: str) -> pandas.DataFrame: +def json_to_dataframe(data: str) -> pd.DataFrame: """ Deserialize JSON into a DataFrame :param data: A JSON string :returns: A pandas DataFrame from the JSON string """ - dataframe = pandas.DataFrame.from_dict(data, orient='tight') + dataframe = pd.read_json(data, orient='split') if 'date' in dataframe.columns: - dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) + dataframe['date'] = pd.to_datetime(dataframe['date'], unit='ms', utc=True) return dataframe -def remove_entry_exit_signals(dataframe: pandas.DataFrame): +def remove_entry_exit_signals(dataframe: pd.DataFrame): """ Remove Entry and Exit signals from a DataFrame diff --git a/scripts/ws_client.py b/scripts/ws_client.py index ff437e63e..090039cde 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -101,7 +101,7 @@ def json_deserialize(message): :param message: The message to deserialize """ def json_to_dataframe(data: str) -> pandas.DataFrame: - dataframe = pandas.DataFrame.from_dict(data, orient='tight') + dataframe = pandas.read_json(data, orient='split') if 'date' in dataframe.columns: dataframe['date'] = pandas.to_datetime(dataframe['date'], unit='ms', utc=True) From 12cd83453c18b9962ba80f6a6a13434748917456 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 18 Nov 2022 14:03:56 +0100 Subject: [PATCH 08/31] Add warning when queue websocket queue becomes too full --- freqtrade/rpc/api_server/webserver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index 6464ae44e..ec4907e67 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -194,6 +194,9 @@ class ApiServer(RPCHandler): try: while True: logger.debug("Getting queue messages...") + if (qsize := async_queue.qsize()) > 20: + # If the queue becomes too big for too long, this may indicate a problem. + logger.warning(f"Queue size now {qsize}") # Get data from queue message: WSMessageSchemaType = await async_queue.get() logger.debug(f"Found message of type: {message.get('type')}") From b6a8e421f1b2ddb7112773c686867cc6debacf88 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Fri, 18 Nov 2022 09:43:39 -0700 Subject: [PATCH 09/31] remove redundant timestamp conversion in ws serializer --- freqtrade/rpc/api_server/ws/serializer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/freqtrade/rpc/api_server/ws/serializer.py b/freqtrade/rpc/api_server/ws/serializer.py index 8d06746f7..6c402a100 100644 --- a/freqtrade/rpc/api_server/ws/serializer.py +++ b/freqtrade/rpc/api_server/ws/serializer.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import orjson import rapidjson -from pandas import DataFrame, Timestamp +from pandas import DataFrame from freqtrade.misc import dataframe_to_json, json_to_dataframe from freqtrade.rpc.api_server.ws.proxy import WebSocketProxy @@ -52,11 +52,6 @@ def _json_default(z): '__type__': 'dataframe', '__value__': dataframe_to_json(z) } - # Pandas returns a Timestamp object, we need to - # convert it to a timestamp int (with ms) for orjson - # to handle it - if isinstance(z, Timestamp): - return z.timestamp() * 1e3 raise TypeError From 12b471c64b7d87a888224ad9de034e604b10fd4f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 20 Nov 2022 09:28:14 +0100 Subject: [PATCH 10/31] Prevent 2 parallel open orders through forceentry this leads to forgetting the prior order closes #7765 --- freqtrade/rpc/rpc.py | 3 +++ tests/rpc/test_rpc.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index a0824bcc1..1d3f36844 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -774,6 +774,9 @@ class RPC: is_short = trade.is_short if not self._freqtrade.strategy.position_adjustment_enable: raise RPCException(f'position for {pair} already open - id: {trade.id}') + if trade.open_order_id is not None: + raise RPCException(f'position for {pair} already open - id: {trade.id} ' + f'and has open order {trade.open_order_id}') else: if Trade.get_open_trade_count() >= self._config['max_open_trades']: raise RPCException("Maximum number of trades is reached.") diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index ef6c8b204..8828b6f33 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1066,6 +1066,11 @@ def test_rpc_force_entry(mocker, default_conf, ticker, fee, limit_buy_order_open trade = rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) assert trade.stake_amount == 0.05 assert trade.buy_tag == 'force_entry' + assert trade.open_order_id == 'mocked_limit_buy' + + freqtradebot.strategy.position_adjustment_enable = True + with pytest.raises(RPCException, match=r'position for LTC/BTC already open.*open order.*'): + rpc._rpc_force_entry(pair, 0.0001, order_type='limit', stake_amount=0.05) # Test not buying pair = 'XRP/BTC' From 106ac2ab4d76ab32ac25e8b77e429d7b22779bbc Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 20 Nov 2022 16:36:22 -0700 Subject: [PATCH 11/31] fix tests, change to get call --- freqtrade/rpc/external_message_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index d9aed7d52..6078efd07 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -181,7 +181,7 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - scheme = 'wss' if producer['secure'] else 'ws' + scheme = 'wss' if producer.get('secure', False) else 'ws' ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad From edb817e2e6254b9156b00e89e8144ba54f1c9644 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 20 Nov 2022 19:19:28 -0700 Subject: [PATCH 12/31] add tutorial for ssl in docs --- docs/rest-api.md | 169 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/docs/rest-api.md b/docs/rest-api.md index c7d762648..2087d9a94 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -458,3 +458,172 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. + + +### Using SSL/TLS + +SSL/TLS is used to provide security and encrypt network traffic. Freqtrade does not directly support SSL, but you can easily accomplish this with a reverse proxy such as Nginx or Traefik. Below are some steps to help you get started on setting one up for your bot. For the sake of simplicity, we will use a native installation of Nginx and certbot. + + +**Prerequisites** + +Before starting this tutorial, you will need a few things. + +- A Freqtrade bot set up and running +- A registered domain name, e.g. myftbot.com +- A DNS A record for the top level domain pointing to your server's public IP + +**Step 1: Installing Nginx and Certbot** + +Once you have all of the prerequisites, the first step is to get Nginx installed on your system. This tutorial assumes the use of Ubuntu 20.04, though you can find your linux distro's package commands via a search engine. First, update your local package index so that you have access to the most recent package listings then install Nginx: + +``` bash +> sudo apt update +> sudo apt install nginx +``` + +After accepting the installation, Nginx and any dependencies will be installed to your system and automatically started. You can check it is running with `systemd`: + +``` bash +> sudo systemctl status nginx + +● nginx.service - A high performance web server and a reverse proxy server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled; vendor preset: enabled) + Active: active (running) since Wed 2022-11-16 12:09:27 MST; 4 days ago + Docs: man:nginx(8) + Process: 1026 ExecStartPre=/usr/sbin/nginx -t -q -g daemon on; master_process on; (code=exited, status=0/SUCCESS) + Process: 1106 ExecStart=/usr/sbin/nginx -g daemon on; master_process on; (code=exited, status=0/SUCCESS) + Main PID: 1107 (nginx) + Tasks: 5 (limit: 6929) + Memory: 5.7M + CGroup: /system.slice/nginx.service + ├─1107 nginx: master process /usr/sbin/nginx -g daemon on; master_process on; + ├─1108 nginx: worker process + ├─1109 nginx: worker process + ├─1110 nginx: worker process + └─1111 nginx: worker process +``` + +Next you need to install certbot which will handle all of the certificate automation for your web server and domain. To install certbot it is required to have `snapd` on Ubuntu. If you haven't installed it yet, please review the instructions on [snapcraft's site for installation](https://snapcraft.io/docs/installing-snapd/). + +Once you are good to go, ensure your snapd version is up to date: + +``` bash +> sudo snap install core; sudo snap refresh core +``` + +If you have any Certbot packages installed via your package manager, you should remove them before installing Certbot: + +``` bash +> sudo apt remove certbot +``` + +Finally, install Certbot and prepare the Certbot command. + +``` bash +> sudo snap install --classic certbot +> sudo ln -s /snap/bin/certbot /usr/bin/certbot +``` + +**Step 2: Adjust the firewall** + +The next step is to allow HTTP and HTTPS traffic through your firewall. This is different for each depending on which firewall you use, and how you have it configured. In this example, we are using `ufw`. + +We'll start by enabling `ufw` if it isn't already: + +``` bash +> sudo ufw enable +``` + +You can list the application configurations that ufw knows how to work with + +``` bash +> sudo ufw app list + +Available applications: + CUPS + Nginx Full + Nginx HTTP + Nginx HTTPS +``` + +As you can see in the output, there are 3 profiles available for Nginx: + +- **Nginx Full**: This profile opens both port 80 (normal web traffic) and port 443 (SSL/TLS traffic) +- **Nginx HTTP**: This profile only opens port 80 (normal web traffic) +- **Nginx HTTPS**: This profile only opens port 443 (SSL/TLS traffic) + +We will configure the firewall to allow both port 80 and 443: + +``` bash +> sudo ufw allow 'Nginx Full' +``` + +You can verify the change by typing: + +``` bash +> sudo ufw status + +Status: active + +To Action From +-- ------ ---- +Nginx HTTPS ALLOW Anywhere +Nginx Full ALLOW Anywhere +Nginx HTTPS (v6) ALLOW Anywhere (v6) +Nginx Full (v6) ALLOW Anywhere (v6) +``` + +**Step 3: Configuring Nginx** + +Using your favorite editor, edit the default nginx configuration. In our case, it'll be under `/etc/nginx/conf.d/default.conf`: +``` bash +> sudo vim /etc/nginx/conf.d/default.conf +``` + +Add a section to your configuration like this: + +``` +server { + server_name myftbot.com; + location / { + proxy_pass http://localhost:8080; + } +} +``` + +Make sure to change `localhost` and `8080` to what you have set in your `api_server` configuration for your bot. + +Verify your nginx config file syntax and make sure there are no errors: +``` bash +> sudo nginx -t +``` + +Finally you can reload nginx to get the new configuration changes: + +``` bash +> sudo systemctl reload nginx +``` + +!!! Note + The `reload` command forces Nginx to read the new configuration without interrupting any connections. The `restart` command restarts the whole nginx service. + +**Step 4: Getting the certificates** + +Certbot already comes with an easy way to setup Nginx with SSL/TLS th automatically changes your configuration file with the required fields: + +``` bash +> sudo certbot --nginx +``` + +You will be prompted for some information such as your email (To receive updates about your certificates), the domain you pointed to the server, and agree to the TOS and optional newsletter. You can also set to redirect HTTP traffic to HTTPS, removing HTTP access. + +You can now test your SSL setup by using curl to make a request to your bot's Rest API: + +``` bash +> curl https://myftbot.com/api/v1/ping + +{'status': 'pong'} +``` + +If you see a pong response then everything is working and you have successfully set up SSL/TLS termination for your bot. From 0cb08024f1ef8e53ac3b10f2ae2fe53fa3e20219 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:01:59 +0000 Subject: [PATCH 13/31] Bump numpy from 1.23.4 to 1.23.5 Bumps [numpy](https://github.com/numpy/numpy) from 1.23.4 to 1.23.5. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.23.4...v1.23.5) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..7827a7be6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.23.4 +numpy==1.23.5 pandas==1.5.1 pandas-ta==0.3.14b From 5cce8f4f2d07fdcf2e1e46cbd723b3fbec5a75f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:00 +0000 Subject: [PATCH 14/31] Bump types-python-dateutil from 2.8.19.3 to 2.8.19.4 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.19.3 to 2.8.19.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..ee4a22387 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -29,4 +29,4 @@ types-cachetools==5.2.1 types-filelock==3.2.7 types-requests==2.28.11.4 types-tabulate==0.9.0.0 -types-python-dateutil==2.8.19.3 +types-python-dateutil==2.8.19.4 From 3f9dacc9bef4881ecc66a44d3fe73f346db1e6f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:15 +0000 Subject: [PATCH 15/31] Bump ccxt from 2.1.75 to 2.1.96 Bumps [ccxt](https://github.com/ccxt/ccxt) from 2.1.75 to 2.1.96. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/2.1.75...2.1.96) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..fa4767a43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.4 pandas==1.5.1 pandas-ta==0.3.14b -ccxt==2.1.75 +ccxt==2.1.96 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.3; platform_machine != 'armv7l' From ec15ef039845569dc000da052030b7764e549259 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:26 +0000 Subject: [PATCH 16/31] Bump nbconvert from 7.2.4 to 7.2.5 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.2.4 to 7.2.5. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md) - [Commits](https://github.com/jupyter/nbconvert/compare/v7.2.4...v7.2.5) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..da5719dfd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ time-machine==2.8.2 httpx==0.23.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.2.4 +nbconvert==7.2.5 # mypy types types-cachetools==5.2.1 From 844334a7ea9a05544faac65480658dc7d67a2074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:34 +0000 Subject: [PATCH 17/31] Bump uvicorn from 0.19.0 to 0.20.0 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.19.0 to 0.20.0. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.19.0...0.20.0) --- updated-dependencies: - dependency-name: uvicorn dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..036ca6335 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ sdnotify==0.3.2 # API Server fastapi==0.87.0 pydantic==1.10.2 -uvicorn==0.19.0 +uvicorn==0.20.0 pyjwt==2.6.0 aiofiles==22.1.0 psutil==5.9.4 From 8d1ee67ed438c08a418fc73e8b771ebf4e6d2f7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:38 +0000 Subject: [PATCH 18/31] Bump httpx from 0.23.0 to 0.23.1 Bumps [httpx](https://github.com/encode/httpx) from 0.23.0 to 0.23.1. - [Release notes](https://github.com/encode/httpx/releases) - [Changelog](https://github.com/encode/httpx/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/httpx/compare/0.23.0...0.23.1) --- updated-dependencies: - dependency-name: httpx dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..ec31df57c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ isort==5.10.1 # For datetime mocking time-machine==2.8.2 # fastapi testing -httpx==0.23.0 +httpx==0.23.1 # Convert jupyter notebooks to markdown documents nbconvert==7.2.4 From f09fb2374b4378c6e4b00bb26e7e07e62d2ce613 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:47 +0000 Subject: [PATCH 19/31] Bump orjson from 3.8.1 to 3.8.2 Bumps [orjson](https://github.com/ijl/orjson) from 3.8.1 to 3.8.2. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.1...3.8.2) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..03945c1bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.1 +orjson==3.8.2 # Notify systemd sdnotify==0.3.2 From beec9e2d1ad6956b88949dffdef4abcbd0689a6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 05:36:52 +0000 Subject: [PATCH 20/31] Bump mypy from 0.990 to 0.991 Bumps [mypy](https://github.com/python/mypy) from 0.990 to 0.991. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.990...v0.991) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 17fd99a0f..1eab3aa77 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 flake8==5.0.4 flake8-tidy-imports==4.8.0 -mypy==0.990 +mypy==0.991 pre-commit==2.20.0 pytest==7.2.0 pytest-asyncio==0.20.2 From 2df0d613daed590ef0c9ab756646014518851c12 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 06:58:59 +0100 Subject: [PATCH 21/31] Bump types-dateutil for precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4a8336b9..ac5f47b00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - types-filelock==3.2.7 - types-requests==2.28.11.4 - types-tabulate==0.9.0.0 - - types-python-dateutil==2.8.19.3 + - types-python-dateutil==2.8.19.4 # stages: [push] - repo: https://github.com/pycqa/isort From adc1174d2ee7ca2751bfc0b108d9dbb2679a0832 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:30:22 +0000 Subject: [PATCH 22/31] Bump types-requests from 2.28.11.4 to 2.28.11.5 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.11.4 to 2.28.11.5. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e1b755e7..ca76e5aee 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -27,6 +27,6 @@ nbconvert==7.2.5 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.11.4 +types-requests==2.28.11.5 types-tabulate==0.9.0.0 types-python-dateutil==2.8.19.4 From 7c00ef8a7619a59a6c1f49cc87e181417f309499 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 11:32:31 +0100 Subject: [PATCH 23/31] Bump pre-commit requests version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac5f47b00..ccf9d5098 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.11.4 + - types-requests==2.28.11.5 - types-tabulate==0.9.0.0 - types-python-dateutil==2.8.19.4 # stages: [push] From 0fa521704381748fee5bd953573ca3d34fbb8e0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 19:30:49 +0100 Subject: [PATCH 24/31] Improve protection setup lock_pair should be called when the order closes, not when the exit order is placed. it should also be called for stoploss orders, too. closes #7783 --- freqtrade/freqtradebot.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2e2638126..77b099d80 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin): trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, stoploss_order=True) - # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') self._notify_exit(trade, "stoploss", True) + self.handle_protections(trade.pair, trade.trade_direction) return True if trade.open_order_id or not trade.is_open: @@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin): trade.close_rate_requested = limit trade.exit_reason = exit_reason - if not sub_trade_amt: - # Lock pair for one candle to prevent immediate re-trading - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') - self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): @@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin): self._notify_enter(trade, order, fill=True, sub_trade=sub_trade) def handle_protections(self, pair: str, side: LongShort) -> None: + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock') prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } From 8cb2b4666d229a41320cf959c3eb6b10cf600d75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 20:42:07 +0100 Subject: [PATCH 25/31] Improve proxy docs closes #7769 --- docs/configuration.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ce4453561..98ac7c124 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Using proxy with Freqtrade To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values. +This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests. ``` bash export HTTP_PROXY="http://addr:port" @@ -672,17 +673,20 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` -#### Proxy just exchange requests +#### Proxy exchange requests -To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration. +To use a proxy for exchange connections - you can will have to define the proxies as part of the ccxt configuration. ``` json -"ccxt_config": { +{ + "exchange": { + "ccxt_config": { "aiohttp_proxy": "http://addr:port", "proxies": { - "http": "http://addr:port", - "https": "http://addr:port" + "http": "http://addr:port", + "https": "http://addr:port" }, + } } ``` From 86ff711525ba13cd88673686f90eed994e18e8b9 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Mon, 21 Nov 2022 12:52:18 -0700 Subject: [PATCH 26/31] update docs on reverse proxy --- docs/rest-api.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index 2087d9a94..beb9ac194 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,6 +389,23 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` +#### Reverse Proxy and Websockets + +There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. + +When using Nginx, the following configuration is required for WebSockets to work: +``` +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $connection_upgrade; +proxy_set_header Host $host; +``` + +To configure your reverse proxy, see it's documentation for proxying websockets. + +- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) +- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. @@ -459,7 +476,7 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. - + From 5a489ce71b58515d61653fd1f25e1d4f034204ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Nov 2022 10:46:38 +0100 Subject: [PATCH 27/31] Fix docs typo --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 98ac7c124..83b23425c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -675,7 +675,7 @@ freqtrade #### Proxy exchange requests -To use a proxy for exchange connections - you can will have to define the proxies as part of the ccxt configuration. +To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration. ``` json { From fff745fd83e63c5816801a000ac01ec279b6038e Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 22 Nov 2022 07:17:57 -0700 Subject: [PATCH 28/31] add map to nginx config --- docs/rest-api.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index beb9ac194..c7c41d571 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -393,12 +393,27 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. -When using Nginx, the following configuration is required for WebSockets to work: +When using Nginx, the following configuration is required for WebSockets to work (Note this configuration isn't complete, it's missing some information and can not be used as is): ``` -proxy_http_version 1.1; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $connection_upgrade; -proxy_set_header Host $host; +http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + ... + + server { + ... + + location / { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + } + } +} ``` To configure your reverse proxy, see it's documentation for proxying websockets. From bd05f85c26bc65228af6b3942bbd1b2f3955eb3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Nov 2022 18:11:18 +0100 Subject: [PATCH 29/31] Simplify ssl documentation --- docs/rest-api.md | 187 +++-------------------------------------------- 1 file changed, 12 insertions(+), 175 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index c7c41d571..62ad586dd 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,11 +389,12 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` -#### Reverse Proxy and Websockets +#### Reverse Proxy setup -There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. +When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is): + +Please make sure to replace `` (and the subsequent port) with the IP and Port matching your configuration/setup. -When using Nginx, the following configuration is required for WebSockets to work (Note this configuration isn't complete, it's missing some information and can not be used as is): ``` http { map $http_upgrade $connection_upgrade { @@ -401,13 +402,14 @@ http { '' close; } - ... + #... server { - ... + #... location / { proxy_http_version 1.1; + proxy_pass http://:8080; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header Host $host; @@ -416,11 +418,15 @@ http { } ``` -To configure your reverse proxy, see it's documentation for proxying websockets. +To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets. - **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) - **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) +!!! Tip "SSL certificates" + You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies. + While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel). + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. @@ -490,172 +496,3 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. - - From 335de760edd40b0d013698fca75724c1eb1a1f9f Mon Sep 17 00:00:00 2001 From: Emre Date: Wed, 23 Nov 2022 18:34:50 +0300 Subject: [PATCH 30/31] Enable --use-pep517 flag for freqai --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 1a4a285a3..fceab0074 100755 --- a/setup.sh +++ b/setup.sh @@ -82,7 +82,7 @@ function updateenv() { dev=$REPLY if [[ $REPLY =~ ^[Yy]$ ]] then - REQUIREMENTS_FREQAI="-r requirements-freqai.txt" + REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517" fi ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI} From c963fd720b01e111f46e1c7fb136bcb9e5621f75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Nov 2022 18:17:10 +0100 Subject: [PATCH 31/31] Slightly change test setup for dry_run_order_fill --- tests/exchange/test_exchange.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a719496e5..e61ad8532 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees( assert order1['fee']['rate'] == fee -@pytest.mark.parametrize("side,startprice,endprice", [ - ("buy", 25.563, 25.566), - ("sell", 25.566, 25.563) +@pytest.mark.parametrize("side,price,filled", [ + # order_book_l2_usd spread: + # best ask: 25.566 + # best bid: 25.563 + ("buy", 25.563, False), + ("buy", 25.566, True), + ("sell", 25.566, False), + ("sell", 25.563, True), ]) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled, exchange_name, order_book_l2_usd): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, ordertype='limit', side=side, amount=1, - rate=startprice, + rate=price, leverage=1.0 ) assert order_book_l2_usd.call_count == 1 @@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order["side"] == side assert order["type"] == "limit" assert order["symbol"] == "LTC/USDT" + assert order["average"] == price + assert order['status'] == 'open' if not filled else 'closed' order_book_l2_usd.reset_mock() + # fetch order again... order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_book_l2_usd.call_count == 1 - assert order_closed['status'] == 'open' - assert not order['fee'] - assert order_closed['filled'] == 0 + assert order_book_l2_usd.call_count == (1 if not filled else 0) + assert order_closed['status'] == ('open' if not filled else 'closed') + assert order_closed['filled'] == (0 if not filled else 1) order_book_l2_usd.reset_mock() - order_closed['price'] = endprice - - order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_closed['status'] == 'closed' - assert order['fee'] - assert order_closed['filled'] == 1 - assert order_closed['filled'] == order_closed['amount'] # Empty orderbook test mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',