From 6a1655c047b88bba462677566f6943819ffc83a7 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:26:54 -0700 Subject: [PATCH 01/25] support ssl connections in emc --- freqtrade/constants.py | 1 + freqtrade/rpc/external_message_consumer.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 534d06fd4..cfac98ebd 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -512,6 +512,7 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 65535 }, + 'secure': {'type': 'boolean', 'default': False}, 'ws_token': {'type': 'string'}, }, 'required': ['name', 'host', 'ws_token'] diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index b978407e4..d9aed7d52 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -31,6 +31,7 @@ class Producer(TypedDict): name: str host: str port: int + secure: bool ws_token: str @@ -180,7 +181,8 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - ws_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + scheme = 'wss' if producer['secure'] else 'ws' + ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad async with websockets.connect( From 86e094e39b77e9c6ea8b8e88fa7a84339be151e0 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:33:42 -0700 Subject: [PATCH 02/25] update docs --- docs/producer-consumer.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/producer-consumer.md b/docs/producer-consumer.md index b69406edf..88e34d0d6 100644 --- a/docs/producer-consumer.md +++ b/docs/producer-consumer.md @@ -21,6 +21,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect "name": "default", // This can be any name you'd like, default is "default" "host": "127.0.0.1", // The host from your producer's api_server config "port": 8080, // The port from your producer's api_server config + "secure": false, // Use a secure websockets connection, default false "ws_token": "sercet_Ws_t0ken" // The ws_token from your producer's api_server config } ], @@ -42,6 +43,7 @@ Enable subscribing to an instance by adding the `external_message_consumer` sect | `producers.name` | **Required.** Name of this producer. This name must be used in calls to `get_producer_pairs()` and `get_producer_df()` if more than one producer is used.
**Datatype:** string | `producers.host` | **Required.** The hostname or IP address from your producer.
**Datatype:** string | `producers.port` | **Required.** The port matching the above host.
**Datatype:** string +| `producers.secure` | **Optional.** Use ssl in websockets connection. Default False.
**Datatype:** string | `producers.ws_token` | **Required.** `ws_token` as configured on the producer.
**Datatype:** string | | **Optional settings** | `wait_timeout` | Timeout until we ping again if no message is received.
*Defaults to `300`.*
**Datatype:** Integer - in seconds. From 1380ddd066c46784ea0ffd518f919ef8da7972f5 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 15 Nov 2022 22:44:02 -0700 Subject: [PATCH 03/25] update ws client --- scripts/ws_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 090039cde..70dbead14 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -199,6 +199,7 @@ async def create_client( host, port, token, + scheme='ws', name='default', protocol=ClientProtocol(), sleep_time=10, @@ -211,13 +212,14 @@ async def create_client( :param host: The host :param port: The port :param token: The websocket auth token + :param scheme: `ws` for most connections, `wss` for ssl :param name: The name of the producer :param **kwargs: Any extra kwargs passed to websockets.connect """ while 1: try: - websocket_url = f"ws://{host}:{port}/api/v1/message/ws?token={token}" + websocket_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" logger.info(f"Attempting to connect to {name} @ {host}:{port}") async with websockets.connect(websocket_url, **kwargs) as ws: @@ -304,6 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], + 'wss' if producer['secure'] else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, From a993cb512de6422d0c186e27c214502a5356c5eb Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Thu, 17 Nov 2022 10:22:55 -0700 Subject: [PATCH 04/25] change to get call in ws_client --- scripts/ws_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ws_client.py b/scripts/ws_client.py index 70dbead14..5d27f512e 100644 --- a/scripts/ws_client.py +++ b/scripts/ws_client.py @@ -306,7 +306,7 @@ async def _main(args): producer['host'], producer['port'], producer['ws_token'], - 'wss' if producer['secure'] else 'ws', + 'wss' if producer.get('secure', False) else 'ws', producer['name'], sleep_time=sleep_time, ping_timeout=ping_timeout, From 106ac2ab4d76ab32ac25e8b77e429d7b22779bbc Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 20 Nov 2022 16:36:22 -0700 Subject: [PATCH 05/25] fix tests, change to get call --- freqtrade/rpc/external_message_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/external_message_consumer.py b/freqtrade/rpc/external_message_consumer.py index d9aed7d52..6078efd07 100644 --- a/freqtrade/rpc/external_message_consumer.py +++ b/freqtrade/rpc/external_message_consumer.py @@ -181,7 +181,7 @@ class ExternalMessageConsumer: host, port = producer['host'], producer['port'] token = producer['ws_token'] name = producer['name'] - scheme = 'wss' if producer['secure'] else 'ws' + scheme = 'wss' if producer.get('secure', False) else 'ws' ws_url = f"{scheme}://{host}:{port}/api/v1/message/ws?token={token}" # This will raise InvalidURI if the url is bad From edb817e2e6254b9156b00e89e8144ba54f1c9644 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Sun, 20 Nov 2022 19:19:28 -0700 Subject: [PATCH 06/25] add tutorial for ssl in docs --- docs/rest-api.md | 169 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) diff --git a/docs/rest-api.md b/docs/rest-api.md index c7d762648..2087d9a94 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -458,3 +458,172 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. + + +### Using SSL/TLS + +SSL/TLS is used to provide security and encrypt network traffic. Freqtrade does not directly support SSL, but you can easily accomplish this with a reverse proxy such as Nginx or Traefik. Below are some steps to help you get started on setting one up for your bot. For the sake of simplicity, we will use a native installation of Nginx and certbot. + + +**Prerequisites** + +Before starting this tutorial, you will need a few things. + +- A Freqtrade bot set up and running +- A registered domain name, e.g. myftbot.com +- A DNS A record for the top level domain pointing to your server's public IP + +**Step 1: Installing Nginx and Certbot** + +Once you have all of the prerequisites, the first step is to get Nginx installed on your system. This tutorial assumes the use of Ubuntu 20.04, though you can find your linux distro's package commands via a search engine. First, update your local package index so that you have access to the most recent package listings then install Nginx: + +``` bash +> sudo apt update +> sudo apt install nginx +``` + +After accepting the installation, Nginx and any dependencies will be installed to your system and automatically started. You can check it is running with `systemd`: + +``` bash +> sudo systemctl status nginx + +● nginx.service - A high performance web server and a reverse proxy server + Loaded: loaded (/lib/systemd/system/nginx.service; enabled; vendor preset: enabled) + Active: active (running) since Wed 2022-11-16 12:09:27 MST; 4 days ago + Docs: man:nginx(8) + Process: 1026 ExecStartPre=/usr/sbin/nginx -t -q -g daemon on; master_process on; (code=exited, status=0/SUCCESS) + Process: 1106 ExecStart=/usr/sbin/nginx -g daemon on; master_process on; (code=exited, status=0/SUCCESS) + Main PID: 1107 (nginx) + Tasks: 5 (limit: 6929) + Memory: 5.7M + CGroup: /system.slice/nginx.service + ├─1107 nginx: master process /usr/sbin/nginx -g daemon on; master_process on; + ├─1108 nginx: worker process + ├─1109 nginx: worker process + ├─1110 nginx: worker process + └─1111 nginx: worker process +``` + +Next you need to install certbot which will handle all of the certificate automation for your web server and domain. To install certbot it is required to have `snapd` on Ubuntu. If you haven't installed it yet, please review the instructions on [snapcraft's site for installation](https://snapcraft.io/docs/installing-snapd/). + +Once you are good to go, ensure your snapd version is up to date: + +``` bash +> sudo snap install core; sudo snap refresh core +``` + +If you have any Certbot packages installed via your package manager, you should remove them before installing Certbot: + +``` bash +> sudo apt remove certbot +``` + +Finally, install Certbot and prepare the Certbot command. + +``` bash +> sudo snap install --classic certbot +> sudo ln -s /snap/bin/certbot /usr/bin/certbot +``` + +**Step 2: Adjust the firewall** + +The next step is to allow HTTP and HTTPS traffic through your firewall. This is different for each depending on which firewall you use, and how you have it configured. In this example, we are using `ufw`. + +We'll start by enabling `ufw` if it isn't already: + +``` bash +> sudo ufw enable +``` + +You can list the application configurations that ufw knows how to work with + +``` bash +> sudo ufw app list + +Available applications: + CUPS + Nginx Full + Nginx HTTP + Nginx HTTPS +``` + +As you can see in the output, there are 3 profiles available for Nginx: + +- **Nginx Full**: This profile opens both port 80 (normal web traffic) and port 443 (SSL/TLS traffic) +- **Nginx HTTP**: This profile only opens port 80 (normal web traffic) +- **Nginx HTTPS**: This profile only opens port 443 (SSL/TLS traffic) + +We will configure the firewall to allow both port 80 and 443: + +``` bash +> sudo ufw allow 'Nginx Full' +``` + +You can verify the change by typing: + +``` bash +> sudo ufw status + +Status: active + +To Action From +-- ------ ---- +Nginx HTTPS ALLOW Anywhere +Nginx Full ALLOW Anywhere +Nginx HTTPS (v6) ALLOW Anywhere (v6) +Nginx Full (v6) ALLOW Anywhere (v6) +``` + +**Step 3: Configuring Nginx** + +Using your favorite editor, edit the default nginx configuration. In our case, it'll be under `/etc/nginx/conf.d/default.conf`: +``` bash +> sudo vim /etc/nginx/conf.d/default.conf +``` + +Add a section to your configuration like this: + +``` +server { + server_name myftbot.com; + location / { + proxy_pass http://localhost:8080; + } +} +``` + +Make sure to change `localhost` and `8080` to what you have set in your `api_server` configuration for your bot. + +Verify your nginx config file syntax and make sure there are no errors: +``` bash +> sudo nginx -t +``` + +Finally you can reload nginx to get the new configuration changes: + +``` bash +> sudo systemctl reload nginx +``` + +!!! Note + The `reload` command forces Nginx to read the new configuration without interrupting any connections. The `restart` command restarts the whole nginx service. + +**Step 4: Getting the certificates** + +Certbot already comes with an easy way to setup Nginx with SSL/TLS th automatically changes your configuration file with the required fields: + +``` bash +> sudo certbot --nginx +``` + +You will be prompted for some information such as your email (To receive updates about your certificates), the domain you pointed to the server, and agree to the TOS and optional newsletter. You can also set to redirect HTTP traffic to HTTPS, removing HTTP access. + +You can now test your SSL setup by using curl to make a request to your bot's Rest API: + +``` bash +> curl https://myftbot.com/api/v1/ping + +{'status': 'pong'} +``` + +If you see a pong response then everything is working and you have successfully set up SSL/TLS termination for your bot. From 0cb08024f1ef8e53ac3b10f2ae2fe53fa3e20219 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:01:59 +0000 Subject: [PATCH 07/25] Bump numpy from 1.23.4 to 1.23.5 Bumps [numpy](https://github.com/numpy/numpy) from 1.23.4 to 1.23.5. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.23.4...v1.23.5) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..7827a7be6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.23.4 +numpy==1.23.5 pandas==1.5.1 pandas-ta==0.3.14b From 5cce8f4f2d07fdcf2e1e46cbd723b3fbec5a75f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:00 +0000 Subject: [PATCH 08/25] Bump types-python-dateutil from 2.8.19.3 to 2.8.19.4 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.19.3 to 2.8.19.4. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..ee4a22387 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -29,4 +29,4 @@ types-cachetools==5.2.1 types-filelock==3.2.7 types-requests==2.28.11.4 types-tabulate==0.9.0.0 -types-python-dateutil==2.8.19.3 +types-python-dateutil==2.8.19.4 From 3f9dacc9bef4881ecc66a44d3fe73f346db1e6f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:15 +0000 Subject: [PATCH 09/25] Bump ccxt from 2.1.75 to 2.1.96 Bumps [ccxt](https://github.com/ccxt/ccxt) from 2.1.75 to 2.1.96. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/2.1.75...2.1.96) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..fa4767a43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.23.4 pandas==1.5.1 pandas-ta==0.3.14b -ccxt==2.1.75 +ccxt==2.1.96 # Pin cryptography for now due to rust build errors with piwheels cryptography==38.0.1; platform_machine == 'armv7l' cryptography==38.0.3; platform_machine != 'armv7l' From ec15ef039845569dc000da052030b7764e549259 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:26 +0000 Subject: [PATCH 10/25] Bump nbconvert from 7.2.4 to 7.2.5 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.2.4 to 7.2.5. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md) - [Commits](https://github.com/jupyter/nbconvert/compare/v7.2.4...v7.2.5) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..da5719dfd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ time-machine==2.8.2 httpx==0.23.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.2.4 +nbconvert==7.2.5 # mypy types types-cachetools==5.2.1 From 844334a7ea9a05544faac65480658dc7d67a2074 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:34 +0000 Subject: [PATCH 11/25] Bump uvicorn from 0.19.0 to 0.20.0 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.19.0 to 0.20.0. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.19.0...0.20.0) --- updated-dependencies: - dependency-name: uvicorn dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..036ca6335 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ sdnotify==0.3.2 # API Server fastapi==0.87.0 pydantic==1.10.2 -uvicorn==0.19.0 +uvicorn==0.20.0 pyjwt==2.6.0 aiofiles==22.1.0 psutil==5.9.4 From 8d1ee67ed438c08a418fc73e8b771ebf4e6d2f7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:38 +0000 Subject: [PATCH 12/25] Bump httpx from 0.23.0 to 0.23.1 Bumps [httpx](https://github.com/encode/httpx) from 0.23.0 to 0.23.1. - [Release notes](https://github.com/encode/httpx/releases) - [Changelog](https://github.com/encode/httpx/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/httpx/compare/0.23.0...0.23.1) --- updated-dependencies: - dependency-name: httpx dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e4d42538..ec31df57c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,7 +19,7 @@ isort==5.10.1 # For datetime mocking time-machine==2.8.2 # fastapi testing -httpx==0.23.0 +httpx==0.23.1 # Convert jupyter notebooks to markdown documents nbconvert==7.2.4 From f09fb2374b4378c6e4b00bb26e7e07e62d2ce613 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 03:02:47 +0000 Subject: [PATCH 13/25] Bump orjson from 3.8.1 to 3.8.2 Bumps [orjson](https://github.com/ijl/orjson) from 3.8.1 to 3.8.2. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.1...3.8.2) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ec8b5ce7c..03945c1bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.1 +orjson==3.8.2 # Notify systemd sdnotify==0.3.2 From beec9e2d1ad6956b88949dffdef4abcbd0689a6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 05:36:52 +0000 Subject: [PATCH 14/25] Bump mypy from 0.990 to 0.991 Bumps [mypy](https://github.com/python/mypy) from 0.990 to 0.991. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.990...v0.991) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 17fd99a0f..1eab3aa77 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 flake8==5.0.4 flake8-tidy-imports==4.8.0 -mypy==0.990 +mypy==0.991 pre-commit==2.20.0 pytest==7.2.0 pytest-asyncio==0.20.2 From 2df0d613daed590ef0c9ab756646014518851c12 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 06:58:59 +0100 Subject: [PATCH 15/25] Bump types-dateutil for precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4a8336b9..ac5f47b00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - types-filelock==3.2.7 - types-requests==2.28.11.4 - types-tabulate==0.9.0.0 - - types-python-dateutil==2.8.19.3 + - types-python-dateutil==2.8.19.4 # stages: [push] - repo: https://github.com/pycqa/isort From adc1174d2ee7ca2751bfc0b108d9dbb2679a0832 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:30:22 +0000 Subject: [PATCH 16/25] Bump types-requests from 2.28.11.4 to 2.28.11.5 Bumps [types-requests](https://github.com/python/typeshed) from 2.28.11.4 to 2.28.11.5. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1e1b755e7..ca76e5aee 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -27,6 +27,6 @@ nbconvert==7.2.5 # mypy types types-cachetools==5.2.1 types-filelock==3.2.7 -types-requests==2.28.11.4 +types-requests==2.28.11.5 types-tabulate==0.9.0.0 types-python-dateutil==2.8.19.4 From 7c00ef8a7619a59a6c1f49cc87e181417f309499 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 11:32:31 +0100 Subject: [PATCH 17/25] Bump pre-commit requests version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac5f47b00..ccf9d5098 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.2.1 - types-filelock==3.2.7 - - types-requests==2.28.11.4 + - types-requests==2.28.11.5 - types-tabulate==0.9.0.0 - types-python-dateutil==2.8.19.4 # stages: [push] From 0fa521704381748fee5bd953573ca3d34fbb8e0b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 19:30:49 +0100 Subject: [PATCH 18/25] Improve protection setup lock_pair should be called when the order closes, not when the exit order is placed. it should also be called for stoploss orders, too. closes #7783 --- freqtrade/freqtradebot.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2e2638126..77b099d80 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1133,10 +1133,8 @@ class FreqtradeBot(LoggingMixin): trade.exit_reason = ExitType.STOPLOSS_ON_EXCHANGE.value self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, stoploss_order=True) - # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') self._notify_exit(trade, "stoploss", True) + self.handle_protections(trade.pair, trade.trade_direction) return True if trade.open_order_id or not trade.is_open: @@ -1595,11 +1593,6 @@ class FreqtradeBot(LoggingMixin): trade.close_rate_requested = limit trade.exit_reason = exit_reason - if not sub_trade_amt: - # Lock pair for one candle to prevent immediate re-trading - self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), - reason='Auto lock') - self._notify_exit(trade, order_type, sub_trade=bool(sub_trade_amt), order=order_obj) # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): @@ -1809,6 +1802,8 @@ class FreqtradeBot(LoggingMixin): self._notify_enter(trade, order, fill=True, sub_trade=sub_trade) def handle_protections(self, pair: str, side: LongShort) -> None: + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(pair, datetime.now(timezone.utc), reason='Auto lock') prot_trig = self.protections.stop_per_pair(pair, side=side) if prot_trig: msg = {'type': RPCMessageType.PROTECTION_TRIGGER, } From 8cb2b4666d229a41320cf959c3eb6b10cf600d75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 21 Nov 2022 20:42:07 +0100 Subject: [PATCH 19/25] Improve proxy docs closes #7769 --- docs/configuration.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ce4453561..98ac7c124 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -665,6 +665,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d ### Using proxy with Freqtrade To use a proxy with freqtrade, export your proxy settings using the variables `"HTTP_PROXY"` and `"HTTPS_PROXY"` set to the appropriate values. +This will have the proxy settings applied to everything (telegram, coingecko, ...) except exchange requests. ``` bash export HTTP_PROXY="http://addr:port" @@ -672,17 +673,20 @@ export HTTPS_PROXY="http://addr:port" freqtrade ``` -#### Proxy just exchange requests +#### Proxy exchange requests -To use a proxy just for exchange connections (skips/ignores telegram and coingecko) - you can also define the proxies as part of the ccxt configuration. +To use a proxy for exchange connections - you can will have to define the proxies as part of the ccxt configuration. ``` json -"ccxt_config": { +{ + "exchange": { + "ccxt_config": { "aiohttp_proxy": "http://addr:port", "proxies": { - "http": "http://addr:port", - "https": "http://addr:port" + "http": "http://addr:port", + "https": "http://addr:port" }, + } } ``` From 86ff711525ba13cd88673686f90eed994e18e8b9 Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Mon, 21 Nov 2022 12:52:18 -0700 Subject: [PATCH 20/25] update docs on reverse proxy --- docs/rest-api.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index 2087d9a94..beb9ac194 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,6 +389,23 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` +#### Reverse Proxy and Websockets + +There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. + +When using Nginx, the following configuration is required for WebSockets to work: +``` +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $connection_upgrade; +proxy_set_header Host $host; +``` + +To configure your reverse proxy, see it's documentation for proxying websockets. + +- **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) +- **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. @@ -459,7 +476,7 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. - + From 5a489ce71b58515d61653fd1f25e1d4f034204ce Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Nov 2022 10:46:38 +0100 Subject: [PATCH 21/25] Fix docs typo --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 98ac7c124..83b23425c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -675,7 +675,7 @@ freqtrade #### Proxy exchange requests -To use a proxy for exchange connections - you can will have to define the proxies as part of the ccxt configuration. +To use a proxy for exchange connections - you will have to define the proxies as part of the ccxt configuration. ``` json { From fff745fd83e63c5816801a000ac01ec279b6038e Mon Sep 17 00:00:00 2001 From: Timothy Pogue Date: Tue, 22 Nov 2022 07:17:57 -0700 Subject: [PATCH 22/25] add map to nginx config --- docs/rest-api.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index beb9ac194..c7c41d571 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -393,12 +393,27 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. -When using Nginx, the following configuration is required for WebSockets to work: +When using Nginx, the following configuration is required for WebSockets to work (Note this configuration isn't complete, it's missing some information and can not be used as is): ``` -proxy_http_version 1.1; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $connection_upgrade; -proxy_set_header Host $host; +http { + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + ... + + server { + ... + + location / { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + } + } +} ``` To configure your reverse proxy, see it's documentation for proxying websockets. From bd05f85c26bc65228af6b3942bbd1b2f3955eb3e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 22 Nov 2022 18:11:18 +0100 Subject: [PATCH 23/25] Simplify ssl documentation --- docs/rest-api.md | 187 +++-------------------------------------------- 1 file changed, 12 insertions(+), 175 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index c7c41d571..62ad586dd 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -389,11 +389,12 @@ Now anytime those types of RPC messages are sent in the bot, you will receive th } ``` -#### Reverse Proxy and Websockets +#### Reverse Proxy setup -There are some quirks when using a reverse proxy with the message websocket endpoint. The message websocket endpoint keeps a long-running connection open between the Rest API and the client. It's built on top of HTTP and uses the HTTP Upgrade mechanism to change from HTTP to WebSockets during connection. There are some challenges that a reverse proxy faces when supporting WebSockets, such as WebSockets are a hop-by-hop protocol, so when a proxy intercepts an Upgrade request from the client it needs to send it's own Upgrade request to the server, including appropriate headers. Also, since these connections are long lived, the proxy needs to allow these connections to remain open. +When using [Nginx](https://nginx.org/en/docs/), the following configuration is required for WebSockets to work (Note this configuration is incomplete, it's missing some information and can not be used as is): + +Please make sure to replace `` (and the subsequent port) with the IP and Port matching your configuration/setup. -When using Nginx, the following configuration is required for WebSockets to work (Note this configuration isn't complete, it's missing some information and can not be used as is): ``` http { map $http_upgrade $connection_upgrade { @@ -401,13 +402,14 @@ http { '' close; } - ... + #... server { - ... + #... location / { proxy_http_version 1.1; + proxy_pass http://:8080; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header Host $host; @@ -416,11 +418,15 @@ http { } ``` -To configure your reverse proxy, see it's documentation for proxying websockets. +To properly configure your reverse proxy (securely), please consult it's documentation for proxying websockets. - **Traefik**: Traefik supports websockets out of the box, see the [documentation](https://doc.traefik.io/traefik/) - **Caddy**: Caddy v2 supports websockets out of the box, see the [documentation](https://caddyserver.com/docs/v2-upgrade#proxy) +!!! Tip "SSL certificates" + You can use tools like certbot to setup ssl certificates to access your bot's UI through encrypted connection by using any fo the above reverse proxies. + While this will protect your data in transit, we do not recommend to run the freqtrade API outside of your private network (VPN, SSH tunnel). + ### OpenAPI interface To enable the builtin openAPI interface (Swagger UI), specify `"enable_openapi": true` in the api_server configuration. @@ -490,172 +496,3 @@ The correct configuration for this case is `http://localhost:8080` - the main pa !!! Note We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. - - From 335de760edd40b0d013698fca75724c1eb1a1f9f Mon Sep 17 00:00:00 2001 From: Emre Date: Wed, 23 Nov 2022 18:34:50 +0300 Subject: [PATCH 24/25] Enable --use-pep517 flag for freqai --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 1a4a285a3..fceab0074 100755 --- a/setup.sh +++ b/setup.sh @@ -82,7 +82,7 @@ function updateenv() { dev=$REPLY if [[ $REPLY =~ ^[Yy]$ ]] then - REQUIREMENTS_FREQAI="-r requirements-freqai.txt" + REQUIREMENTS_FREQAI="-r requirements-freqai.txt --use-pep517" fi ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} ${REQUIREMENTS_FREQAI} From c963fd720b01e111f46e1c7fb136bcb9e5621f75 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 23 Nov 2022 18:17:10 +0100 Subject: [PATCH 25/25] Slightly change test setup for dry_run_order_fill --- tests/exchange/test_exchange.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index a719496e5..e61ad8532 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1207,12 +1207,17 @@ def test_create_dry_run_order_fees( assert order1['fee']['rate'] == fee -@pytest.mark.parametrize("side,startprice,endprice", [ - ("buy", 25.563, 25.566), - ("sell", 25.566, 25.563) +@pytest.mark.parametrize("side,price,filled", [ + # order_book_l2_usd spread: + # best ask: 25.566 + # best bid: 25.563 + ("buy", 25.563, False), + ("buy", 25.566, True), + ("sell", 25.566, False), + ("sell", 25.563, True), ]) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, endprice, +def test_create_dry_run_order_limit_fill(default_conf, mocker, side, price, filled, exchange_name, order_book_l2_usd): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -1226,7 +1231,7 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, ordertype='limit', side=side, amount=1, - rate=startprice, + rate=price, leverage=1.0 ) assert order_book_l2_usd.call_count == 1 @@ -1235,22 +1240,17 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order["side"] == side assert order["type"] == "limit" assert order["symbol"] == "LTC/USDT" + assert order["average"] == price + assert order['status'] == 'open' if not filled else 'closed' order_book_l2_usd.reset_mock() + # fetch order again... order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_book_l2_usd.call_count == 1 - assert order_closed['status'] == 'open' - assert not order['fee'] - assert order_closed['filled'] == 0 + assert order_book_l2_usd.call_count == (1 if not filled else 0) + assert order_closed['status'] == ('open' if not filled else 'closed') + assert order_closed['filled'] == (0 if not filled else 1) order_book_l2_usd.reset_mock() - order_closed['price'] = endprice - - order_closed = exchange.fetch_dry_run_order(order['id']) - assert order_closed['status'] == 'closed' - assert order['fee'] - assert order_closed['filled'] == 1 - assert order_closed['filled'] == order_closed['amount'] # Empty orderbook test mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',