From 027e0234430a6e9fb794c13b2a73aa73f529decd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 Mar 2023 18:00:06 +0100 Subject: [PATCH 01/28] Stop from open with leverage --- freqtrade/strategy/strategy_helper.py | 13 ++++++++----- tests/strategy/test_strategy_helpers.py | 18 +++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index aa753a829..3ba1850b3 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -86,7 +86,8 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, def stoploss_from_open( open_relative_stop: float, current_profit: float, - is_short: bool = False + is_short: bool = False, + leverage: float = 1.0 ) -> float: """ @@ -102,21 +103,23 @@ def stoploss_from_open( :param open_relative_stop: Desired stop loss percentage relative to open price :param current_profit: The current profit percentage :param is_short: When true, perform the calculation for short instead of long + :param leverage: Leverage to use for the calculation :return: Stop loss value relative to current price """ # formula is undefined for current_profit -1 (longs) or 1 (shorts), return maximum value - if (current_profit == -1 and not is_short) or (is_short and current_profit == 1): + _current_profit = current_profit / leverage + if (_current_profit == -1 and not is_short) or (is_short and _current_profit == 1): return 1 if is_short is True: - stoploss = -1 + ((1 - open_relative_stop) / (1 - current_profit)) + stoploss = -1 + ((1 - open_relative_stop / leverage) / (1 - _current_profit)) else: - stoploss = 1 - ((1 + open_relative_stop) / (1 + current_profit)) + stoploss = 1 - ((1 + open_relative_stop / leverage) / (1 + _current_profit)) # negative stoploss values indicate the requested stop price is higher/lower # (long/short) than the current price - return max(stoploss, 0.0) + return max(stoploss * leverage, 0.0) def stoploss_from_absolute(stop_rate: float, current_rate: float, is_short: bool = False) -> float: diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index cb79ac171..a55580780 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -177,26 +177,30 @@ def test_stoploss_from_open(side, profitrange): ("long", 0.1, 0.2, 1, 0.08333333), ("long", 0.1, 0.5, 1, 0.266666666), ("long", 0.1, 5, 1, 0.816666666), # 500% profit, set stoploss to 10% above open price + ("long", 0, 5, 10, 3.3333333), # 500% profit, set stoploss break even + ("long", 0.1, 5, 10, 3.26666666), # 500% profit, set stoploss to 10% above open price + ("long", -0.1, 5, 10, 3.3999999), # 500% profit, set stoploss to 10% belowopen price ("short", 0, 0.1, 1, 0.1111111), ("short", -0.1, 0.1, 1, 0.2222222), ("short", 0.1, 0.2, 1, 0.125), ("short", 0.1, 1, 1, 1), + ("short", -0.01, 5, 10, 10.01999999), # 500% profit at 10x ]) def test_stoploss_from_open_leverage(side, rel_stop, curr_profit, leverage, expected): - stoploss = stoploss_from_open(rel_stop, curr_profit, side == 'short') + stoploss = stoploss_from_open(rel_stop, curr_profit, side == 'short', leverage) assert pytest.approx(stoploss) == expected open_rate = 100 if stoploss != 1: if side == 'long': - current_rate = open_rate * (1 + curr_profit) - stop = current_rate * (1 - stoploss) - assert pytest.approx(stop) == open_rate * (1 + rel_stop) + current_rate = open_rate * (1 + curr_profit / leverage) + stop = current_rate * (1 - stoploss / leverage) + assert pytest.approx(stop) == open_rate * (1 + rel_stop / leverage) else: - current_rate = open_rate * (1 - curr_profit) - stop = current_rate * (1 + stoploss) - assert pytest.approx(stop) == open_rate * (1 - rel_stop) + current_rate = open_rate * (1 - curr_profit / leverage) + stop = current_rate * (1 + stoploss / leverage) + assert pytest.approx(stop) == open_rate * (1 - rel_stop / leverage) def test_stoploss_from_absolute(): From d80760d20c9b663f64bcd530ed2e4c84d7a7ab2e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 14:16:53 +0100 Subject: [PATCH 02/28] bump ccxt to 2.8.98 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6b1c888b8..a6b3ddd51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.2 pandas==1.5.3 pandas-ta==0.3.14b -ccxt==2.8.54 +ccxt==2.8.98 cryptography==39.0.1 aiohttp==3.8.4 SQLAlchemy==2.0.4 From 0bdd238d7ff2ebf748b705f0cb239767925eeade Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:56:37 +0000 Subject: [PATCH 03/28] Bump orjson from 3.8.6 to 3.8.7 Bumps [orjson](https://github.com/ijl/orjson) from 3.8.6 to 3.8.7. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.8.6...3.8.7) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6b3ddd51..8f5c2f46d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.9 # Properly format api responses -orjson==3.8.6 +orjson==3.8.7 # Notify systemd sdnotify==0.3.2 From f4c17be8dec5dd176a73fa5fb07d752e79443087 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:56:44 +0000 Subject: [PATCH 04/28] Bump ruff from 0.0.253 to 0.0.254 Bumps [ruff](https://github.com/charliermarsh/ruff) from 0.0.253 to 0.0.254. - [Release notes](https://github.com/charliermarsh/ruff/releases) - [Changelog](https://github.com/charliermarsh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/charliermarsh/ruff/compare/v0.0.253...v0.0.254) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2ba004f8d..4c009fc42 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.0.253 +ruff==0.0.254 mypy==1.0.1 pre-commit==3.1.1 pytest==7.2.1 From 8484427cf879a4dae7aee02a504811d6c7e99b75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:56:54 +0000 Subject: [PATCH 05/28] Bump cryptography from 39.0.1 to 39.0.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 39.0.1 to 39.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/39.0.1...39.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6b3ddd51..1f22bc033 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ pandas==1.5.3 pandas-ta==0.3.14b ccxt==2.8.98 -cryptography==39.0.1 +cryptography==39.0.2 aiohttp==3.8.4 SQLAlchemy==2.0.4 python-telegram-bot==13.15 From 57969f8b0164579df9f5bd1b80504598a2b84e70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:56:58 +0000 Subject: [PATCH 06/28] Bump prompt-toolkit from 3.0.37 to 3.0.38 Bumps [prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit) from 3.0.37 to 3.0.38. - [Release notes](https://github.com/prompt-toolkit/python-prompt-toolkit/releases) - [Changelog](https://github.com/prompt-toolkit/python-prompt-toolkit/blob/master/CHANGELOG) - [Commits](https://github.com/prompt-toolkit/python-prompt-toolkit/compare/3.0.37...3.0.38) --- updated-dependencies: - dependency-name: prompt-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6b3ddd51..efd858f45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ psutil==5.9.4 colorama==0.4.6 # Building config files interactively questionary==1.10.0 -prompt-toolkit==3.0.37 +prompt-toolkit==3.0.38 # Extensions to datetime library python-dateutil==2.8.2 From d1d9e25c2e8707edbbc905f53e7722bfe5b5af4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:57:03 +0000 Subject: [PATCH 07/28] Bump mkdocs-material from 9.0.15 to 9.1.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.0.15 to 9.1.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.0.15...9.1.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 065411018..2ca11dabf 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.3.7 mkdocs==1.4.2 -mkdocs-material==9.0.15 +mkdocs-material==9.1.1 mdx_truly_sane_lists==1.3 pymdown-extensions==9.9.2 jinja2==3.1.2 From 48e16f6aba7a4d2ed33958a473144e62e3cac32d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:57:18 +0000 Subject: [PATCH 08/28] Bump sqlalchemy from 2.0.4 to 2.0.5.post1 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 2.0.4 to 2.0.5.post1. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6b3ddd51..6eeecfde6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ pandas-ta==0.3.14b ccxt==2.8.98 cryptography==39.0.1 aiohttp==3.8.4 -SQLAlchemy==2.0.4 +SQLAlchemy==2.0.5.post1 python-telegram-bot==13.15 arrow==1.2.3 cachetools==4.2.2 From a57b033745558bc6e61de9704dfab3e2e5f20e1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:57:27 +0000 Subject: [PATCH 09/28] Bump types-python-dateutil from 2.8.19.9 to 2.8.19.10 Bumps [types-python-dateutil](https://github.com/python/typeshed) from 2.8.19.9 to 2.8.19.10. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-python-dateutil dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2ba004f8d..a945ffc63 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -29,4 +29,4 @@ types-cachetools==5.3.0.4 types-filelock==3.2.7 types-requests==2.28.11.15 types-tabulate==0.9.0.1 -types-python-dateutil==2.8.19.9 +types-python-dateutil==2.8.19.10 From 9750e9ca4ef54d11f4525b79ea3ff9664b4a43e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Mar 2023 06:32:33 +0100 Subject: [PATCH 10/28] pre-commit python-dateutil --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 565eb96f7..402e5641d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - types-filelock==3.2.7 - types-requests==2.28.11.15 - types-tabulate==0.9.0.1 - - types-python-dateutil==2.8.19.9 + - types-python-dateutil==2.8.19.10 - SQLAlchemy==2.0.4 # stages: [push] From 25fd4a04d6460ed2a00af144d7fcb291669a1943 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Mar 2023 06:34:37 +0100 Subject: [PATCH 11/28] Update sqlalchemy QueryPropertyDescriptor to match latest version --- .pre-commit-config.yaml | 2 +- freqtrade/persistence/pairlock.py | 5 ++--- freqtrade/persistence/trade_model.py | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 565eb96f7..895916c15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - types-requests==2.28.11.15 - types-tabulate==0.9.0.1 - types-python-dateutil==2.8.19.9 - - SQLAlchemy==2.0.4 + - SQLAlchemy==2.0.5.post1 # stages: [push] - repo: https://github.com/pycqa/isort diff --git a/freqtrade/persistence/pairlock.py b/freqtrade/persistence/pairlock.py index a6d1eeaf0..1e5699145 100644 --- a/freqtrade/persistence/pairlock.py +++ b/freqtrade/persistence/pairlock.py @@ -2,8 +2,7 @@ from datetime import datetime, timezone from typing import Any, ClassVar, Dict, Optional from sqlalchemy import String, or_ -from sqlalchemy.orm import Mapped, Query, mapped_column -from sqlalchemy.orm.scoping import _QueryDescriptorType +from sqlalchemy.orm import Mapped, Query, QueryPropertyDescriptor, mapped_column from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.persistence.base import ModelBase, SessionType @@ -14,7 +13,7 @@ class PairLock(ModelBase): Pair Locks database model. """ __tablename__ = 'pairlocks' - query: ClassVar[_QueryDescriptorType] + query: ClassVar[QueryPropertyDescriptor] _session: ClassVar[SessionType] id: Mapped[int] = mapped_column(primary_key=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 21fe80819..8e8a414c8 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -8,8 +8,8 @@ from math import isclose from typing import Any, ClassVar, Dict, List, Optional, cast from sqlalchemy import Enum, Float, ForeignKey, Integer, String, UniqueConstraint, desc, func -from sqlalchemy.orm import Mapped, Query, lazyload, mapped_column, relationship -from sqlalchemy.orm.scoping import _QueryDescriptorType +from sqlalchemy.orm import (Mapped, Query, QueryPropertyDescriptor, lazyload, mapped_column, + relationship) from freqtrade.constants import (DATETIME_PRINT_FORMAT, MATH_CLOSE_PREC, NON_OPEN_EXCHANGE_STATES, BuySell, LongShort) @@ -36,7 +36,7 @@ class Order(ModelBase): Mirrors CCXT Order structure """ __tablename__ = 'orders' - query: ClassVar[_QueryDescriptorType] + query: ClassVar[QueryPropertyDescriptor] _session: ClassVar[SessionType] # Uniqueness should be ensured over pair, order_id @@ -1181,7 +1181,7 @@ class Trade(ModelBase, LocalTrade): Note: Fields must be aligned with LocalTrade class """ __tablename__ = 'trades' - query: ClassVar[_QueryDescriptorType] + query: ClassVar[QueryPropertyDescriptor] _session: ClassVar[SessionType] use_db: bool = True From 0fe72510d5cbaa8389070cf75e1e4cedf880008d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 05:36:16 +0000 Subject: [PATCH 12/28] Bump pymdown-extensions from 9.9.2 to 9.10 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 9.9.2 to 9.10. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/9.9.2...9.10) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 2ca11dabf..1b9a1f9b7 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,5 +2,5 @@ markdown==3.3.7 mkdocs==1.4.2 mkdocs-material==9.1.1 mdx_truly_sane_lists==1.3 -pymdown-extensions==9.9.2 +pymdown-extensions==9.10 jinja2==3.1.2 From de015a2d7e23e4d006e1cd84c7e12e1cd2378dad Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 18:04:30 +0100 Subject: [PATCH 13/28] Improve telegram message formatting --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 1a96b1671..e8a8a941f 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -510,14 +510,14 @@ class Telegram(RPCHandler): if prev_avg_price: minus_on_entry = (cur_entry_average - prev_avg_price) / prev_avg_price - lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg profit") + lines.append(f"*{wording} #{order_nr}:* at {minus_on_entry:.2%} avg Profit") if is_open: lines.append("({})".format(cur_entry_datetime .humanize(granularity=["day", "hour", "minute"]))) lines.append(f"*Amount:* {cur_entry_amount} " f"({round_coin_value(order['cost'], quote_currency)})") lines.append(f"*Average {wording} Price:* {cur_entry_average} " - f"({price_to_1st_entry:.2%} from 1st entry rate)") + f"({price_to_1st_entry:.2%} from 1st entry Rate)") lines.append(f"*Order filled:* {order['order_filled_date']}") # TODO: is this really useful? From 11eea9b4e1cef0a9bfceb27b37cfe8b7a10cc89f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 18:05:42 +0100 Subject: [PATCH 14/28] Fix formatting for /status Realized profit --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e8a8a941f..7452005db 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -602,7 +602,7 @@ class Telegram(RPCHandler): if r['is_open']: if r.get('realized_profit'): lines.append( - "*Realized Profit:* `{realized_profit_r} {realized_profit_ratio:.2%}`") + "*Realized Profit:* `{realized_profit_ratio:.2%} ({realized_profit_r})`") lines.append("*Total Profit:* `{total_profit_abs_r}` ") if (r['stop_loss_abs'] != r['initial_stop_loss_abs'] From ca789b3282926c3e40f2be1e905034be001c2abb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 18:11:10 +0100 Subject: [PATCH 15/28] /status - whitespace --- freqtrade/rpc/telegram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 7452005db..2e87eabc9 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -594,7 +594,7 @@ class Telegram(RPCHandler): "*Close Rate:* `{close_rate:.8f}`" if r['close_rate'] else "", "*Open Date:* `{open_date}`", "*Close Date:* `{close_date}`" if r['close_date'] else "", - "*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "", + "\n*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "", ("*Unrealized Profit:* " if r['is_open'] else "*Close Profit: *") + "`{profit_ratio:.2%}` `({profit_abs_r})`", ]) @@ -605,6 +605,8 @@ class Telegram(RPCHandler): "*Realized Profit:* `{realized_profit_ratio:.2%} ({realized_profit_r})`") lines.append("*Total Profit:* `{total_profit_abs_r}` ") + # Append empty line to improve readability + lines.append(" ") if (r['stop_loss_abs'] != r['initial_stop_loss_abs'] and r['initial_stop_loss_ratio'] is not None): # Adding initial stoploss only if it is different from stoploss From fff08f737f29471ada06a336c4b0aefe0ec6c621 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 19:17:39 +0100 Subject: [PATCH 16/28] /status msg - improve formatting further --- freqtrade/rpc/telegram.py | 11 ++++++++--- tests/rpc/test_rpc_telegram.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2e87eabc9..19027f4d5 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -569,6 +569,8 @@ class Telegram(RPCHandler): and not o['ft_order_side'] == 'stoploss']) r['exit_reason'] = r.get('exit_reason', "") r['stake_amount_r'] = round_coin_value(r['stake_amount'], r['quote_currency']) + r['max_stake_amount_r'] = round_coin_value( + r['max_stake_amount'] or r['stake_amount'], r['quote_currency']) r['profit_abs_r'] = round_coin_value(r['profit_abs'], r['quote_currency']) r['realized_profit_r'] = round_coin_value(r['realized_profit'], r['quote_currency']) r['total_profit_abs_r'] = round_coin_value( @@ -580,21 +582,24 @@ class Telegram(RPCHandler): f"*Direction:* {'`Short`' if r.get('is_short') else '`Long`'}" + " ` ({leverage}x)`" if r.get('leverage') else "", "*Amount:* `{amount} ({stake_amount_r})`", + "*Total invested:* `{max_stake_amount_r}`" if position_adjust else "", "*Enter Tag:* `{enter_tag}`" if r['enter_tag'] else "", "*Exit Reason:* `{exit_reason}`" if r['exit_reason'] else "", ] if position_adjust: max_buy_str = (f"/{max_entries + 1}" if (max_entries > 0) else "") - lines.append("*Number of Entries:* `{num_entries}" + max_buy_str + "`") - lines.append("*Number of Exits:* `{num_exits}`") + lines.extend([ + "*Number of Entries:* `{num_entries}" + max_buy_str + "`", + "*Number of Exits:* `{num_exits}`" + ]) lines.extend([ "*Open Rate:* `{open_rate:.8f}`", "*Close Rate:* `{close_rate:.8f}`" if r['close_rate'] else "", "*Open Date:* `{open_date}`", "*Close Date:* `{close_date}`" if r['close_date'] else "", - "\n*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "", + " \n*Current Rate:* `{current_rate:.8f}`" if r['is_open'] else "", ("*Unrealized Profit:* " if r['is_open'] else "*Close Profit: *") + "`{profit_ratio:.2%}` `({profit_abs_r})`", ]) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 69d0f805d..1dc255b3e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -198,6 +198,7 @@ def test_telegram_status(default_conf, update, mocker) -> None: 'current_rate': 1.098e-05, 'amount': 90.99181074, 'stake_amount': 90.99181074, + 'max_stake_amount': 90.99181074, 'buy_tag': None, 'enter_tag': None, 'close_profit_ratio': None, From 9d285e3dc04e1b4c2c0ae1d01a17198f4f0cac86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 19:35:06 +0100 Subject: [PATCH 17/28] Add total_profit_ratio to telegram output part of #8234 --- freqtrade/rpc/rpc.py | 6 ++++++ freqtrade/rpc/telegram.py | 7 ++++--- tests/rpc/test_rpc.py | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 8692c477f..4b3f6209e 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -192,6 +192,11 @@ class RPC: current_profit = trade.close_profit or 0.0 current_profit_abs = trade.close_profit_abs or 0.0 total_profit_abs = trade.realized_profit + current_profit_abs + total_profit_ratio = 0.0 + if trade.max_stake_amount: + total_profit_ratio = ( + (total_profit_abs / trade.max_stake_amount) * trade.leverage + ) # Calculate fiat profit if not isnan(current_profit_abs) and self._fiat_converter: @@ -224,6 +229,7 @@ class RPC: total_profit_abs=total_profit_abs, total_profit_fiat=total_profit_fiat, + total_profit_ratio=total_profit_ratio, stoploss_current_dist=stoploss_current_dist, stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8), stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2), diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 19027f4d5..ced1bb5ca 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -606,9 +606,10 @@ class Telegram(RPCHandler): if r['is_open']: if r.get('realized_profit'): - lines.append( - "*Realized Profit:* `{realized_profit_ratio:.2%} ({realized_profit_r})`") - lines.append("*Total Profit:* `{total_profit_abs_r}` ") + lines.extend([ + "*Realized Profit:* `{realized_profit_ratio:.2%} ({realized_profit_r})`" + "*Total Profit:* `{total_profit_ratio:.2%} ({total_profit_abs_r})`" + ]) # Append empty line to improve readability lines.append(" ") diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index cd72da763..cf7d2bdeb 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -79,6 +79,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'realized_profit_ratio': None, 'total_profit_abs': -4.09e-06, 'total_profit_fiat': ANY, + 'total_profit_ratio': ANY, 'exchange': 'binance', 'leverage': 1.0, 'interest_rate': 0.0, @@ -185,6 +186,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_pct': ANY, 'profit_abs': ANY, 'total_profit_abs': ANY, + 'total_profit_ratio': ANY, 'current_rate': ANY, }) assert results[0] == response_norate From cab1b750b35f4d356d64206c2b20aa2cb338f167 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Mar 2023 19:45:04 +0100 Subject: [PATCH 18/28] Improve test accuracy --- freqtrade/rpc/rpc.py | 2 +- tests/rpc/test_rpc.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 4b3f6209e..c68ed2d48 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -192,7 +192,7 @@ class RPC: current_profit = trade.close_profit or 0.0 current_profit_abs = trade.close_profit_abs or 0.0 total_profit_abs = trade.realized_profit + current_profit_abs - total_profit_ratio = 0.0 + total_profit_ratio: Optional[float] = None if trade.max_stake_amount: total_profit_ratio = ( (total_profit_abs / trade.max_stake_amount) * trade.leverage diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index cf7d2bdeb..1a1802c68 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -50,7 +50,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'amount': 91.07468123, 'amount_requested': 91.07468124, 'stake_amount': 0.001, - 'max_stake_amount': ANY, + 'max_stake_amount': None, 'trade_duration': None, 'trade_duration_s': None, 'close_profit': None, @@ -79,7 +79,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'realized_profit_ratio': None, 'total_profit_abs': -4.09e-06, 'total_profit_fiat': ANY, - 'total_profit_ratio': ANY, + 'total_profit_ratio': None, 'exchange': 'binance', 'leverage': 1.0, 'interest_rate': 0.0, @@ -169,6 +169,10 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: results = rpc._rpc_trade_status() response = deepcopy(gen_response) + response.update({ + 'max_stake_amount': 0.001, + 'total_profit_ratio': pytest.approx(-0.00409), + }) assert results[0] == response mocker.patch(f'{EXMS}.get_rate', @@ -182,6 +186,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_current_dist': ANY, 'stoploss_current_dist_ratio': ANY, 'stoploss_current_dist_pct': ANY, + 'max_stake_amount': 0.001, 'profit_ratio': ANY, 'profit_pct': ANY, 'profit_abs': ANY, From c4a80e33ea3e647fab91efd4ace1c8de3f4afc6b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Mar 2023 07:01:17 +0100 Subject: [PATCH 19/28] Fix missing newline in telegram /status --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ced1bb5ca..30aa55359 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -607,7 +607,7 @@ class Telegram(RPCHandler): if r['is_open']: if r.get('realized_profit'): lines.extend([ - "*Realized Profit:* `{realized_profit_ratio:.2%} ({realized_profit_r})`" + "*Realized Profit:* `{realized_profit_ratio:.2%} ({realized_profit_r})`", "*Total Profit:* `{total_profit_ratio:.2%} ({total_profit_abs_r})`" ]) From d779d60812f29553d029182e7b8b7907c2d5fea0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Mar 2023 07:10:02 +0100 Subject: [PATCH 20/28] Expose total_profit_ratio through API --- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/api_server/api_v1.py | 3 ++- tests/rpc/test_rpc_apiserver.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index a751179b2..064a509fd 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -286,6 +286,7 @@ class OpenTradeSchema(TradeSchema): current_rate: float total_profit_abs: float total_profit_fiat: Optional[float] + total_profit_ratio: Optional[float] open_order: Optional[str] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index f6bab3624..8ea70bb69 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -42,7 +42,8 @@ logger = logging.getLogger(__name__) # 2.22: Add FreqAI to backtesting # 2.23: Allow plot config request in webserver mode # 2.24: Add cancel_open_order endpoint -API_VERSION = 2.24 +# 2.25: Add several profit values to /status endpoint +API_VERSION = 2.25 # Public API, requires no auth. router_public = APIRouter() diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e140a43f1..9c2c3ee3a 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1012,6 +1012,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets, is_short, 'profit_fiat': ANY, 'total_profit_abs': ANY, 'total_profit_fiat': ANY, + 'total_profit_ratio': ANY, 'realized_profit': 0.0, 'realized_profit_ratio': None, 'current_rate': current_rate, From 85e64cd1215b4ef2e0d323fe4be08db90d1d884c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 07:21:23 +0000 Subject: [PATCH 21/28] Bump ccxt from 2.8.98 to 2.9.4 Bumps [ccxt](https://github.com/ccxt/ccxt) from 2.8.98 to 2.9.4. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/2.8.98...2.9.4) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a702507f9..c972ae1d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.24.2 pandas==1.5.3 pandas-ta==0.3.14b -ccxt==2.8.98 +ccxt==2.9.4 cryptography==39.0.2 aiohttp==3.8.4 SQLAlchemy==2.0.5.post1 From d9dc831772e67b3b04f26c9d160152c19940f5b9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Tue, 7 Mar 2023 11:33:54 +0100 Subject: [PATCH 22/28] allow user to drop ohlc from features in RL --- docs/freqai-parameter-table.md | 1 + docs/freqai-reinforcement-learning.md | 4 +++- freqtrade/constants.py | 1 + .../freqai/RL/BaseReinforcementLearningModel.py | 16 +++++++++------- tests/freqai/conftest.py | 4 +++- tests/freqai/test_freqai_interface.py | 8 +------- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/freqai-parameter-table.md b/docs/freqai-parameter-table.md index 275062a33..f67ea8541 100644 --- a/docs/freqai-parameter-table.md +++ b/docs/freqai-parameter-table.md @@ -84,6 +84,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the | `add_state_info` | Tell FreqAI to include state information in the feature set for training and inferencing. The current state variables include trade duration, current profit, trade position. This is only available in dry/live runs, and is automatically switched to false for backtesting.
**Datatype:** bool.
Default: `False`. | `net_arch` | Network architecture which is well described in [`stable_baselines3` doc](https://stable-baselines3.readthedocs.io/en/master/guide/custom_policy.html#examples). In summary: `[, dict(vf=[], pi=[])]`. By default this is set to `[128, 128]`, which defines 2 shared hidden layers with 128 units each. | `randomize_starting_position` | Randomize the starting point of each episode to avoid overfitting.
**Datatype:** bool.
Default: `False`. +| `drop_ohlc_from_features` | Do not include the normalized ohlc data in the feature set passed to the agent during training (ohlc will still be used for driving the environment in all cases)
**Datatype:** Boolean.
**Default:** `False` ### Additional parameters diff --git a/docs/freqai-reinforcement-learning.md b/docs/freqai-reinforcement-learning.md index 3810aec4e..04ca42a5d 100644 --- a/docs/freqai-reinforcement-learning.md +++ b/docs/freqai-reinforcement-learning.md @@ -176,9 +176,11 @@ As you begin to modify the strategy and the prediction model, you will quickly r factor = 100 + pair = self.pair.replace(':', '') + # you can use feature values from dataframe # Assumes the shifted RSI indicator has been generated in the strategy. - rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{self.pair}_" + rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{pair}_" f"{self.config['timeframe']}"].iloc[self._current_tick] # reward agent for entering trades diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 1727da92e..46e9b5cd4 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -588,6 +588,7 @@ CONF_SCHEMA = { "rl_config": { "type": "object", "properties": { + "drop_ohlc_from_features": {"type": "boolean", "default": False}, "train_cycles": {"type": "integer"}, "max_trade_duration_candles": {"type": "integer"}, "add_state_info": {"type": "boolean", "default": False}, diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index a8ef69394..bac717f9f 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -114,6 +114,7 @@ class BaseReinforcementLearningModel(IFreqaiModel): # normalize all data based on train_dataset only prices_train, prices_test = self.build_ohlc_price_dataframes(dk.data_dictionary, pair, dk) + data_dictionary = dk.normalize_data(data_dictionary) # data cleaning/analysis @@ -148,12 +149,8 @@ class BaseReinforcementLearningModel(IFreqaiModel): env_info = self.pack_env_dict(dk.pair) - self.train_env = self.MyRLEnv(df=train_df, - prices=prices_train, - **env_info) - self.eval_env = Monitor(self.MyRLEnv(df=test_df, - prices=prices_test, - **env_info)) + self.train_env = self.MyRLEnv(df=train_df, prices=prices_train, **env_info) + self.eval_env = Monitor(self.MyRLEnv(df=test_df, prices=prices_test, **env_info)) self.eval_callback = EvalCallback(self.eval_env, deterministic=True, render=False, eval_freq=len(train_df), best_model_save_path=str(dk.data_path)) @@ -285,7 +282,6 @@ class BaseReinforcementLearningModel(IFreqaiModel): train_df = data_dictionary["train_features"] test_df = data_dictionary["test_features"] - # %-raw_volume_gen_shift-2_ETH/USDT_1h # price data for model training and evaluation tf = self.config['timeframe'] rename_dict = {'%-raw_open': 'open', '%-raw_low': 'low', @@ -318,6 +314,12 @@ class BaseReinforcementLearningModel(IFreqaiModel): prices_test.rename(columns=rename_dict, inplace=True) prices_test.reset_index(drop=True) + if self.rl_config["drop_ohlc_from_features"]: + train_df.drop(rename_dict.keys(), axis=1, inplace=True) + test_df.drop(rename_dict.keys(), axis=1, inplace=True) + feature_list = dk.training_features_list + feature_list = [e for e in feature_list if e not in rename_dict.keys()] + return prices_train, prices_test def load_model_from_disk(self, dk: FreqaiDataKitchen) -> Any: diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 68e7ea49a..e140ee80b 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -78,7 +78,9 @@ def make_rl_config(conf): "rr": 1, "profit_aim": 0.02, "win_reward_factor": 2 - }} + }, + "drop_ohlc_from_features": False + } return conf diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index f8bee3659..3b370aea4 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -68,13 +68,6 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, freqai_conf['freqai']['feature_parameters'].update({"shuffle_after_split": shuffle}) freqai_conf['freqai']['feature_parameters'].update({"buffer_train_data_candles": buffer}) - if 'ReinforcementLearner' in model: - model_save_ext = 'zip' - freqai_conf = make_rl_config(freqai_conf) - # test the RL guardrails - freqai_conf['freqai']['feature_parameters'].update({"use_SVM_to_remove_outliers": True}) - freqai_conf['freqai']['data_split_parameters'].update({'shuffle': True}) - if 'ReinforcementLearner' in model: model_save_ext = 'zip' freqai_conf = make_rl_config(freqai_conf) @@ -84,6 +77,7 @@ def test_extract_data_and_train_model_Standard(mocker, freqai_conf, model, pca, if 'test_3ac' in model or 'test_4ac' in model: freqai_conf["freqaimodel_path"] = str(Path(__file__).parents[1] / "freqai" / "test_models") + freqai_conf["freqai"]["rl_config"]["drop_ohlc_from_features"] = True strategy = get_patched_freqai_strategy(mocker, freqai_conf) exchange = get_patched_exchange(mocker, freqai_conf) From 2c7ae756f5b4576c7d393d09a6d57563d17be440 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Mar 2023 07:05:59 +0100 Subject: [PATCH 23/28] Improve mock behavior --- tests/optimize/test_backtest_detail.py | 2 +- tests/test_freqtradebot.py | 40 +++++++++++++------------- tests/test_integration.py | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index ae06fca1d..2cb42c003 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -924,7 +924,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data: BTContainer) mocker.patch(f"{EXMS}.get_fee", return_value=0.0) mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=0.00001) mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf')) - mocker.patch('freqtrade.exchange.binance.Binance.get_max_leverage', return_value=100) + mocker.patch(f"{EXMS}.get_max_leverage", return_value=100) patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 5e9cca0f8..06832589c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1068,7 +1068,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf_usdt, limit_order, is_sho mocker.patch(f'{EXMS}.get_trades_for_order', return_value=[]) stoploss = MagicMock(return_value={'id': 13434334}) - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', stoploss) + mocker.patch(f'{EXMS}.create_stoploss', stoploss) freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -1263,7 +1263,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf_usdt, fee, caplog, get_fee=fee, ) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, fetch_stoploss_order=MagicMock(return_value={'status': 'canceled', 'id': 100}), create_stoploss=MagicMock(side_effect=ExchangeError()), ) @@ -1307,7 +1307,7 @@ def test_create_stoploss_order_invalid_order( get_fee=fee, ) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, fetch_order=MagicMock(return_value={'status': 'canceled'}), create_stoploss=MagicMock(side_effect=InvalidOrderException()), ) @@ -1360,7 +1360,7 @@ def test_create_stoploss_order_insufficient_funds( fetch_order=MagicMock(return_value={'status': 'canceled'}), ) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, create_stoploss=MagicMock(side_effect=InsufficientFundsError()), ) patch_get_signal(freqtrade, enter_short=is_short, enter_long=not is_short) @@ -1410,7 +1410,7 @@ def test_handle_stoploss_on_exchange_trailing( get_fee=fee, ) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, create_stoploss=stoploss, stoploss_adjust=MagicMock(return_value=True), ) @@ -1453,7 +1453,7 @@ def test_handle_stoploss_on_exchange_trailing( } }) - mocker.patch('freqtrade.exchange.binance.Binance.fetch_stoploss_order', stoploss_order_hanging) + mocker.patch(f'{EXMS}.fetch_stoploss_order', stoploss_order_hanging) # stoploss initially at 5% assert freqtrade.handle_trade(trade) is False @@ -1471,8 +1471,8 @@ def test_handle_stoploss_on_exchange_trailing( cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock(return_value={'id': 'so1'}) - mocker.patch('freqtrade.exchange.binance.Binance.cancel_stoploss_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', stoploss_order_mock) + mocker.patch(f'{EXMS}.cancel_stoploss_order', cancel_order_mock) + mocker.patch(f'{EXMS}.create_stoploss', stoploss_order_mock) # stoploss should not be updated as the interval is 60 seconds assert freqtrade.handle_trade(trade) is False @@ -1535,7 +1535,7 @@ def test_handle_stoploss_on_exchange_trailing_error( get_fee=fee, ) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, create_stoploss=stoploss, stoploss_adjust=MagicMock(return_value=True), ) @@ -1573,9 +1573,9 @@ def test_handle_stoploss_on_exchange_trailing_error( 'stopPrice': '0.1' } } - mocker.patch('freqtrade.exchange.binance.Binance.cancel_stoploss_order', + mocker.patch(f'{EXMS}.cancel_stoploss_order', side_effect=InvalidOrderException()) - mocker.patch('freqtrade.exchange.binance.Binance.fetch_stoploss_order', + mocker.patch(f'{EXMS}.fetch_stoploss_order', return_value=stoploss_order_hanging) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/USDT.*", caplog) @@ -1586,8 +1586,8 @@ def test_handle_stoploss_on_exchange_trailing_error( # Fail creating stoploss order trade.stoploss_last_update = arrow.utcnow().shift(minutes=-601).datetime caplog.clear() - cancel_mock = mocker.patch('freqtrade.exchange.binance.Binance.cancel_stoploss_order') - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', side_effect=ExchangeError()) + cancel_mock = mocker.patch(f'{EXMS}.cancel_stoploss_order') + mocker.patch(f'{EXMS}.create_stoploss', side_effect=ExchangeError()) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/USDT\..*", caplog) @@ -1604,7 +1604,7 @@ def test_stoploss_on_exchange_price_rounding( stoploss_mock = MagicMock(return_value={'id': '13434334'}) adjust_mock = MagicMock(return_value=False) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, create_stoploss=stoploss_mock, stoploss_adjust=adjust_mock, price_to_precision=price_mock, @@ -1643,7 +1643,7 @@ def test_handle_stoploss_on_exchange_custom_stop( get_fee=fee, ) mocker.patch.multiple( - 'freqtrade.exchange.binance.Binance', + EXMS, create_stoploss=stoploss, stoploss_adjust=MagicMock(return_value=True), ) @@ -1686,7 +1686,7 @@ def test_handle_stoploss_on_exchange_custom_stop( } }) - mocker.patch('freqtrade.exchange.binance.Binance.fetch_stoploss_order', stoploss_order_hanging) + mocker.patch(f'{EXMS}.fetch_stoploss_order', stoploss_order_hanging) assert freqtrade.handle_trade(trade) is False assert freqtrade.handle_stoploss_on_exchange(trade) is False @@ -1703,8 +1703,8 @@ def test_handle_stoploss_on_exchange_custom_stop( cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock(return_value={'id': 'so1'}) - mocker.patch('freqtrade.exchange.binance.Binance.cancel_stoploss_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', stoploss_order_mock) + mocker.patch(f'{EXMS}.cancel_stoploss_order', cancel_order_mock) + mocker.patch(f'{EXMS}.create_stoploss', stoploss_order_mock) # stoploss should not be updated as the interval is 60 seconds assert freqtrade.handle_trade(trade) is False @@ -1821,7 +1821,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, limit_orde cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() mocker.patch(f'{EXMS}.cancel_stoploss_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', stoploss_order_mock) + mocker.patch(f'{EXMS}.create_stoploss', stoploss_order_mock) # price goes down 5% mocker.patch(f'{EXMS}.fetch_ticker', MagicMock(return_value={ @@ -3660,7 +3660,7 @@ def test_may_execute_trade_exit_after_stoploss_on_exchange_hit( } }) - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', stoploss) + mocker.patch(f'{EXMS}.create_stoploss', stoploss) freqtrade = FreqtradeBot(default_conf_usdt) freqtrade.strategy.order_types['stoploss_on_exchange'] = True diff --git a/tests/test_integration.py b/tests/test_integration.py index a3dd8d935..4c57c5669 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -56,9 +56,9 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, [ExitCheckTuple(exit_type=ExitType.EXIT_SIGNAL)]] ) cancel_order_mock = MagicMock() - mocker.patch('freqtrade.exchange.binance.Binance.create_stoploss', stoploss) mocker.patch.multiple( EXMS, + create_stoploss=stoploss, fetch_ticker=ticker, get_fee=fee, amount_to_precision=lambda s, x, y: y, From 29d337fa02be2aa6d79e8be49ea2d5b16c9f9cb9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 8 Mar 2023 11:26:28 +0100 Subject: [PATCH 24/28] ensure ohlc is dropped from both train and predict --- .../RL/BaseReinforcementLearningModel.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index bac717f9f..c528d8910 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -235,6 +235,9 @@ class BaseReinforcementLearningModel(IFreqaiModel): filtered_dataframe, _ = dk.filter_features( unfiltered_df, dk.training_features_list, training_filter=False ) + + filtered_dataframe = self.drop_ohlc_from_df(filtered_dataframe, dk) + filtered_dataframe = dk.normalize_data_from_metadata(filtered_dataframe) dk.data_dictionary["prediction_features"] = filtered_dataframe @@ -314,14 +317,24 @@ class BaseReinforcementLearningModel(IFreqaiModel): prices_test.rename(columns=rename_dict, inplace=True) prices_test.reset_index(drop=True) - if self.rl_config["drop_ohlc_from_features"]: - train_df.drop(rename_dict.keys(), axis=1, inplace=True) - test_df.drop(rename_dict.keys(), axis=1, inplace=True) - feature_list = dk.training_features_list - feature_list = [e for e in feature_list if e not in rename_dict.keys()] + train_df = self.drop_ohlc_from_df(train_df, dk) + test_df = self.drop_ohlc_from_df(test_df, dk) return prices_train, prices_test + def drop_ohlc_from_df(self, df: DataFrame, dk: FreqaiDataKitchen): + """ + Given a dataframe, drop the ohlc data + """ + drop_list = ['%-raw_open', '%-raw_low', '%-raw_high', '%-raw_close'] + + if self.rl_config["drop_ohlc_from_features"]: + df.drop(drop_list, axis=1, inplace=True) + feature_list = dk.training_features_list + feature_list = [e for e in feature_list if e not in drop_list] + + return df + def load_model_from_disk(self, dk: FreqaiDataKitchen) -> Any: """ Can be used by user if they are trying to limit_ram_usage *and* From 85e345fc4839292a88fee0e101e34551aa30e6ec Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Wed, 8 Mar 2023 19:29:39 +0100 Subject: [PATCH 25/28] Update BaseReinforcementLearningModel.py --- freqtrade/freqai/RL/BaseReinforcementLearningModel.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index c528d8910..acdcf3135 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -330,8 +330,6 @@ class BaseReinforcementLearningModel(IFreqaiModel): if self.rl_config["drop_ohlc_from_features"]: df.drop(drop_list, axis=1, inplace=True) - feature_list = dk.training_features_list - feature_list = [e for e in feature_list if e not in drop_list] return df From 0318486bee485dc2951871d0b9c23909b1865a9f Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Mar 2023 19:35:26 +0100 Subject: [PATCH 26/28] Update stoploss_from_open documentation for leverage adjustment --- docs/strategy-callbacks.md | 6 +++--- docs/strategy-customization.md | 6 ++++-- freqtrade/strategy/strategy_helper.py | 7 ++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 81366c66e..f1cdc9f3b 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -316,11 +316,11 @@ class AwesomeStrategy(IStrategy): # evaluate highest to lowest, so that highest possible stop is used if current_profit > 0.40: - return stoploss_from_open(0.25, current_profit, is_short=trade.is_short) + return stoploss_from_open(0.25, current_profit, is_short=trade.is_short, leverage=trade.leverage) elif current_profit > 0.25: - return stoploss_from_open(0.15, current_profit, is_short=trade.is_short) + return stoploss_from_open(0.15, current_profit, is_short=trade.is_short, leverage=trade.leverage) elif current_profit > 0.20: - return stoploss_from_open(0.07, current_profit, is_short=trade.is_short) + return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage) # return maximum stoploss value, keeping current stoploss price unchanged return 1 diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 3519a80cd..8ab0b1464 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -881,7 +881,7 @@ All columns of the informative dataframe will be available on the returning data ### *stoploss_from_open()* -Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the open price instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired percentage above the open price. +Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the entry point instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired trade profit above the entry point. ??? Example "Returning a stoploss relative to the open price from the custom stoploss function" @@ -889,6 +889,8 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit, False)` which will return `0.1157024793`. 11.57% below $121 is $107, which is the same as 7% above $100. + This function will consider leverage - so at 10x leverage, the actual stoploss would be 0.7% above $100 (0.7% * 10x = 7%). + ``` python @@ -907,7 +909,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati # once the profit has risen above 10%, keep the stoploss at 7% above the open price if current_profit > 0.10: - return stoploss_from_open(0.07, current_profit, is_short=trade.is_short) + return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage) return 1 diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 3ba1850b3..27ebe7e69 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -90,17 +90,18 @@ def stoploss_from_open( leverage: float = 1.0 ) -> float: """ - - Given the current profit, and a desired stop loss value relative to the open price, + Given the current profit, and a desired stop loss value relative to the trade entry price, return a stop loss value that is relative to the current price, and which can be returned from `custom_stoploss`. The requested stop can be positive for a stop above the open price, or negative for a stop below the open price. The return value is always >= 0. + `open_relative_stop` will be considered as adjusted for leverage if leverage is provided.. Returns 0 if the resulting stop price would be above/below (longs/shorts) the current price - :param open_relative_stop: Desired stop loss percentage relative to open price + :param open_relative_stop: Desired stop loss percentage, relative to the open price, + adjusted for leverage :param current_profit: The current profit percentage :param is_short: When true, perform the calculation for short instead of long :param leverage: Leverage to use for the calculation From d10ee0979a0fd9349b5ad1e2d6ba03e3c3d3104f Mon Sep 17 00:00:00 2001 From: robcaulk Date: Wed, 8 Mar 2023 19:37:11 +0100 Subject: [PATCH 27/28] ensure training_features_list is updated properly --- freqtrade/freqai/RL/BaseReinforcementLearningModel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py index acdcf3135..e10880f46 100644 --- a/freqtrade/freqai/RL/BaseReinforcementLearningModel.py +++ b/freqtrade/freqai/RL/BaseReinforcementLearningModel.py @@ -330,6 +330,8 @@ class BaseReinforcementLearningModel(IFreqaiModel): if self.rl_config["drop_ohlc_from_features"]: df.drop(drop_list, axis=1, inplace=True) + feature_list = dk.training_features_list + dk.training_features_list = [e for e in feature_list if e not in drop_list] return df From d3a3ddbc614609a63a0fbea827d635b3673f59aa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Mar 2023 19:42:43 +0100 Subject: [PATCH 28/28] Check if exchang provides bid/ask via fetch_tickers - and fail with spread filter if it doesn't. closes #8286 --- freqtrade/exchange/exchange.py | 1 + freqtrade/exchange/gate.py | 1 + freqtrade/plugins/pairlist/SpreadFilter.py | 7 +++++++ tests/plugins/test_pairlist.py | 6 ++++++ 4 files changed, 15 insertions(+) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index cdbda1506..c0e07c6d7 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -69,6 +69,7 @@ class Exchange: # Check https://github.com/ccxt/ccxt/issues/10767 for removal of ohlcv_volume_currency "ohlcv_volume_currency": "base", # "base" or "quote" "tickers_have_quoteVolume": True, + "tickers_have_bid_ask": True, # bid / ask empty for fetch_tickers "tickers_have_price": True, "trades_pagination": "time", # Possible are "time" or "id" "trades_pagination_arg": "since", diff --git a/freqtrade/exchange/gate.py b/freqtrade/exchange/gate.py index 80ed4088a..03b568460 100644 --- a/freqtrade/exchange/gate.py +++ b/freqtrade/exchange/gate.py @@ -32,6 +32,7 @@ class Gate(Exchange): _ft_has_futures: Dict = { "needs_trading_fees": True, + "tickers_have_bid_ask": False, "fee_cost_in_contracts": False, # Set explicitly to false for clarity "order_props_in_contracts": ['amount', 'filled', 'remaining'], "stop_price_type_field": "price_type", diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 207328d08..d47b68568 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -5,6 +5,7 @@ import logging from typing import Any, Dict, Optional from freqtrade.constants import Config +from freqtrade.exceptions import OperationalException from freqtrade.exchange.types import Ticker from freqtrade.plugins.pairlist.IPairList import IPairList @@ -22,6 +23,12 @@ class SpreadFilter(IPairList): self._max_spread_ratio = pairlistconfig.get('max_spread_ratio', 0.005) self._enabled = self._max_spread_ratio != 0 + if not self._exchange.get_option('tickers_have_bid_ask'): + raise OperationalException( + f"{self.name} requires exchange to have bid/ask data for tickers, " + "which is not available for the selected exchange / trading mode." + ) + @property def needstickers(self) -> bool: """ diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 40a3871d7..18ee365e2 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -828,6 +828,12 @@ def test_pair_whitelist_not_supported_Spread(mocker, default_conf, tickers) -> N match=r'Exchange does not support fetchTickers, .*'): get_patched_freqtradebot(mocker, default_conf) + mocker.patch(f'{EXMS}.exchange_has', MagicMock(return_value=True)) + mocker.patch(f'{EXMS}.get_option', MagicMock(return_value=False)) + with pytest.raises(OperationalException, + match=r'.*requires exchange to have bid/ask data'): + get_patched_freqtradebot(mocker, default_conf) + @pytest.mark.parametrize("pairlist", TESTABLE_PAIRLISTS) def test_pairlist_class(mocker, whitelist_conf, markets, pairlist):