From c499bb051f4753f20c9daef9660932c2b610ecd7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Jun 2022 19:41:17 +0200 Subject: [PATCH 01/22] Allow empty unfilledtimeout for webserver mode --- freqtrade/rpc/api_server/api_schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index f21334bc6..a31c74c2e 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -166,7 +166,7 @@ class ShowConfig(BaseModel): trailing_stop_positive: Optional[float] trailing_stop_positive_offset: Optional[float] trailing_only_offset_is_reached: Optional[bool] - unfilledtimeout: UnfilledTimeout + unfilledtimeout: Optional[UnfilledTimeout] # Empty in webserver mode order_types: Optional[OrderTypes] use_custom_stoploss: Optional[bool] timeframe: Optional[str] From f709222943fcc2807561ae4a8fc7bcb9a8d6c66c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 29 Apr 2022 06:53:30 +0200 Subject: [PATCH 02/22] Properly close out orders in backtesting --- freqtrade/optimize/backtesting.py | 1 + freqtrade/persistence/trade_model.py | 1 + 2 files changed, 2 insertions(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index aebaecaca..8fe5f509e 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -1094,6 +1094,7 @@ class Backtesting: # 5. Process exit orders. order = trade.select_order(trade.exit_side, is_open=True) if order and self._get_order_filled(order.price, row): + order.close_bt_order(current_time, trade) trade.open_order_id = None trade.close_date = current_time trade.close(order.price, show_msg=False) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 45a16bfbd..7b475d618 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -166,6 +166,7 @@ class Order(_DECL_BASE): def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): self.order_filled_date = close_date self.filled = self.amount + self.remaining = 0 self.status = 'closed' self.ft_is_open = False if (self.ft_order_side == trade.entry_side From c0ff554d5be871098cd10424fdd579322b5370df Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 May 2022 20:12:05 +0200 Subject: [PATCH 03/22] Cleanup old, left open dry-run orders --- freqtrade/persistence/migrations.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 53e35d9da..b0fdf0412 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -247,6 +247,35 @@ def set_sqlite_to_wal(engine): connection.execute(text("PRAGMA journal_mode=wal")) +def fix_old_dry_orders(engine): + with engine.begin() as connection: + connection.execute( + text( + """ + update orders + set ft_is_open = 0 + where ft_is_open = 1 and (ft_trade_id, order_id) not in ( + select id, stoploss_order_id from trades where stoploss_order_id is not null + ) and ft_order_side = 'stoploss' + and order_id like 'dry_%' + """ + ) + ) + connection.execute( + text( + """ + update orders + set ft_is_open = 0 + where ft_is_open = 1 + and (ft_trade_id, order_id) not in ( + select id, open_order_id from trades where open_order_id is not null + ) and ft_order_side != 'stoploss' + and order_id like 'dry_%' + """ + ) + ) + + def check_migrate(engine, decl_base, previous_tables) -> None: """ Checks if migration is necessary and migrates if necessary @@ -288,3 +317,4 @@ def check_migrate(engine, decl_base, previous_tables) -> None: "start with a fresh database.") set_sqlite_to_wal(engine) + fix_old_dry_orders(engine) From 8369d5bedd25f7679b060fd075be2eb061623ebe Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 24 May 2022 20:31:45 +0200 Subject: [PATCH 04/22] Include open orders in json responses --- freqtrade/persistence/trade_model.py | 17 ++++++++++++++++- freqtrade/rpc/telegram.py | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 7b475d618..ded616f8a 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -395,7 +395,7 @@ class LocalTrade(): ) def to_json(self) -> Dict[str, Any]: - filled_orders = self.select_filled_orders() + filled_orders = self.select_filled_or_open_orders() orders = [order.to_json(self.entry_side) for order in filled_orders] return { @@ -898,6 +898,21 @@ class LocalTrade(): (o.filled or 0) > 0 and o.status in NON_OPEN_EXCHANGE_STATES] + def select_filled_or_open_orders(self) -> List['Order']: + """ + Finds filled or open orders + :param order_side: Side of the order (either 'buy', 'sell', or None) + :return: array of Order objects + """ + return [o for o in self.orders if + ( + o.ft_is_open is False + and (o.filled or 0) > 0 + and o.status in NON_OPEN_EXCHANGE_STATES + ) + or (o.ft_is_open is True and o.status is not None) + ] + @property def nr_of_successful_entries(self) -> int: """ diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4a274002e..e456b1eef 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -396,7 +396,7 @@ class Telegram(RPCHandler): first_avg = filled_orders[0]["safe_price"] for x, order in enumerate(filled_orders): - if not order['ft_is_entry']: + if not order['ft_is_entry'] or order['is_open'] is True: continue cur_entry_datetime = arrow.get(order["order_filled_date"]) cur_entry_amount = order["amount"] From 79107fd062e9e60f78c467367b7c34cc68f5b6c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 May 2022 07:11:43 +0200 Subject: [PATCH 05/22] Add minimal order object serialization --- freqtrade/data/btanalysis.py | 2 +- freqtrade/persistence/trade_model.py | 44 +++++++++++++++------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index fef432576..0b466241f 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -337,7 +337,7 @@ def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: :param trades: List of trade objects :return: Dataframe with BT_DATA_COLUMNS """ - df = pd.DataFrame.from_records([t.to_json() for t in trades], columns=BT_DATA_COLUMNS) + df = pd.DataFrame.from_records([t.to_json(True) for t in trades], columns=BT_DATA_COLUMNS) if len(df) > 0: df.loc[:, 'close_date'] = pd.to_datetime(df['close_date'], utc=True) df.loc[:, 'open_date'] = pd.to_datetime(df['open_date'], utc=True) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index ded616f8a..0be9d22c1 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -137,31 +137,35 @@ class Order(_DECL_BASE): 'info': {}, } - def to_json(self, entry_side: str) -> Dict[str, Any]: - return { - 'pair': self.ft_pair, - 'order_id': self.order_id, - 'status': self.status, + def to_json(self, entry_side: str, minified: bool = False) -> Dict[str, Any]: + resp = { 'amount': self.amount, - 'average': round(self.average, 8) if self.average else 0, 'safe_price': self.safe_price, - 'cost': self.cost if self.cost else 0, - 'filled': self.filled, 'ft_order_side': self.ft_order_side, - 'is_open': self.ft_is_open, - 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_date else None, - 'order_timestamp': int(self.order_date.replace( - tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, - 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) - if self.order_filled_date else None, 'order_filled_timestamp': int(self.order_filled_date.replace( tzinfo=timezone.utc).timestamp() * 1000) if self.order_filled_date else None, - 'order_type': self.order_type, - 'price': self.price, 'ft_is_entry': self.ft_order_side == entry_side, - 'remaining': self.remaining, } + if not minified: + resp.update({ + 'pair': self.ft_pair, + 'order_id': self.order_id, + 'status': self.status, + 'average': round(self.average, 8) if self.average else 0, + 'cost': self.cost if self.cost else 0, + 'filled': self.filled, + 'is_open': self.ft_is_open, + 'order_date': self.order_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_date else None, + 'order_timestamp': int(self.order_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.order_date else None, + 'order_filled_date': self.order_filled_date.strftime(DATETIME_PRINT_FORMAT) + if self.order_filled_date else None, + 'order_type': self.order_type, + 'price': self.price, + 'remaining': self.remaining, + }) + return resp def close_bt_order(self, close_date: datetime, trade: 'LocalTrade'): self.order_filled_date = close_date @@ -394,9 +398,9 @@ class LocalTrade(): f'open_rate={self.open_rate:.8f}, open_since={open_since})' ) - def to_json(self) -> Dict[str, Any]: + def to_json(self, minified: bool = False) -> Dict[str, Any]: filled_orders = self.select_filled_or_open_orders() - orders = [order.to_json(self.entry_side) for order in filled_orders] + orders = [order.to_json(self.entry_side, minified) for order in filled_orders] return { 'trade_id': self.id, From 786bc3616352a650d4107a539aad84f7c32d1714 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:44 +0000 Subject: [PATCH 06/22] Bump orjson from 3.6.8 to 3.7.1 Bumps [orjson](https://github.com/ijl/orjson) from 3.6.8 to 3.7.1. - [Release notes](https://github.com/ijl/orjson/releases) - [Changelog](https://github.com/ijl/orjson/blob/master/CHANGELOG.md) - [Commits](https://github.com/ijl/orjson/compare/3.6.8...3.7.1) --- updated-dependencies: - dependency-name: orjson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..21d80571f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ py_find_1st==1.1.5 # Load ticker files 30% faster python-rapidjson==1.6 # Properly format api responses -orjson==3.6.8 +orjson==3.7.1 # Notify systemd sdnotify==0.3.2 From 04cb49b7e404fb0ab29245e769296f7c5ec17d41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:48 +0000 Subject: [PATCH 07/22] Bump filelock from 3.7.0 to 3.7.1 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.7.0 to 3.7.1. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.7.0...3.7.1) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b8762214a..94e59ec15 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,5 +5,5 @@ scipy==1.8.1 scikit-learn==1.1.1 scikit-optimize==0.9.0 -filelock==3.7.0 +filelock==3.7.1 progressbar2==4.0.0 From 6547f3aadb96f60e9d5a42b0008a55de1c47e75c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:52 +0000 Subject: [PATCH 08/22] Bump mkdocs-material from 8.2.16 to 8.3.2 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 8.2.16 to 8.3.2. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/8.2.16...8.3.2) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index e7ca17c34..f351151ab 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.3.0 -mkdocs-material==8.2.16 +mkdocs-material==8.3.2 mdx_truly_sane_lists==1.2 pymdown-extensions==9.4 jinja2==3.1.2 From 35316ec06841b9b2638ab6068c6d58aa3c15991a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:55 +0000 Subject: [PATCH 09/22] Bump jsonschema from 4.5.1 to 4.6.0 Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.5.1 to 4.6.0. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.5.1...v4.6.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..d0b662d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.2.2 cachetools==4.2.2 requests==2.27.1 urllib3==1.26.9 -jsonschema==4.5.1 +jsonschema==4.6.0 TA-Lib==0.4.24 technical==1.3.0 tabulate==0.8.9 From 963dc0221caa59b8cf4cb2309cab6735f8be6161 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:01:59 +0000 Subject: [PATCH 10/22] Bump types-requests from 2.27.29 to 2.27.30 Bumps [types-requests](https://github.com/python/typeshed) from 2.27.29 to 2.27.30. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-requests dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6a7e15870..4eb157aae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,6 +24,6 @@ nbconvert==6.5.0 # mypy types types-cachetools==5.0.1 types-filelock==3.2.6 -types-requests==2.27.29 +types-requests==2.27.30 types-tabulate==0.8.9 types-python-dateutil==2.8.17 From 4affa75ff5103e95fdbc6c59fa457423326fdc74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:02:07 +0000 Subject: [PATCH 11/22] Bump sqlalchemy from 1.4.36 to 1.4.37 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.36 to 1.4.37. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..717577480 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.84.39 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 -SQLAlchemy==1.4.36 +SQLAlchemy==1.4.37 python-telegram-bot==13.12 arrow==1.2.2 cachetools==4.2.2 From 05922e9ebc0c7911f0bc75e50b5a57dc6a0cc29d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 03:02:15 +0000 Subject: [PATCH 12/22] Bump ccxt from 1.84.39 to 1.84.97 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.84.39 to 1.84.97. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.84.39...1.84.97) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a7dbaf57c..432ff976d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.84.39 +ccxt==1.84.97 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 99f6c75c40dc95073ec81a03c82e775d87753667 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 10:22:19 +0200 Subject: [PATCH 13/22] Bump types-requests precommit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95a1d5002..685d789ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: additional_dependencies: - types-cachetools==5.0.1 - types-filelock==3.2.6 - - types-requests==2.27.29 + - types-requests==2.27.30 - types-tabulate==0.8.9 - types-python-dateutil==2.8.17 # stages: [push] From ea9b68baddeb76f2581660d13ff11b797f4a6b00 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 10:50:48 +0200 Subject: [PATCH 14/22] Add updating freqtrade to updating desc --- docs/updating.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/updating.md b/docs/updating.md index 1839edc4c..8dc7279a4 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -32,4 +32,8 @@ Please ensure that you're also updating dependencies - otherwise things might br ``` bash git pull pip install -U -r requirements.txt +pip install -e . + +# Ensure freqUI is at the latest version +freqtrade install-ui ``` From 82c5a6b29dc1c45e0e542d2caace0fb2d87dad68 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 10:57:33 +0200 Subject: [PATCH 15/22] Update CI to use concurrency --- .github/workflows/ci.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2e420e8e..c3ed6d80d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ on: schedule: - cron: '0 5 * * 4' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_linux: @@ -296,17 +300,17 @@ jobs: details: Freqtrade doc test failed! webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} - cleanup-prior-runs: - permissions: - actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it - contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch - runs-on: ubuntu-20.04 - steps: - - name: Cleanup previous runs on this branch - uses: rokroskar/workflow-run-cleanup-action@v0.3.3 - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + # cleanup-prior-runs: + # permissions: + # actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it + # contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch + # runs-on: ubuntu-20.04 + # steps: + # - name: Cleanup previous runs on this branch + # uses: rokroskar/workflow-run-cleanup-action@v0.3.3 + # if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" + # env: + # GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: From 0b806af48756bcb5190fadc1cd50cd9b2ff32b3d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 26 May 2022 07:17:22 +0200 Subject: [PATCH 16/22] Add orders column to btresult --- freqtrade/data/btanalysis.py | 4 +++- freqtrade/optimize/optimize_reports.py | 4 ---- tests/data/test_btanalysis.py | 4 ++-- tests/optimize/test_backtesting.py | 17 +++++++++++++++++ .../test_backtesting_adjust_position.py | 7 ++++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 0b466241f..9e38f6833 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -26,7 +26,7 @@ BT_DATA_COLUMNS = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'profit_ratio', 'profit_abs', 'exit_reason', 'initial_stop_loss_abs', 'initial_stop_loss_ratio', 'stop_loss_abs', 'stop_loss_ratio', 'min_rate', 'max_rate', 'is_open', 'enter_tag', - 'is_short' + 'is_short', 'open_timestamp', 'close_timestamp', 'orders' ] @@ -283,6 +283,8 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non if 'enter_tag' not in df.columns: df['enter_tag'] = df['buy_tag'] df = df.drop(['buy_tag'], axis=1) + if 'orders' not in df.columns: + df.loc[:, 'orders'] = None else: # old format - only with lists. diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 93336fa3f..e3dd17411 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union -from numpy import int64 from pandas import DataFrame, to_datetime from tabulate import tabulate @@ -417,9 +416,6 @@ def generate_strategy_stats(pairlist: List[str], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - if not results.empty: - results['open_timestamp'] = results['open_date'].view(int64) // 1e6 - results['close_timestamp'] = results['close_date'].view(int64) // 1e6 backtest_days = (max_date - min_date).days or 1 strat_stats = { diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 4157bd899..977140ebb 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -85,7 +85,7 @@ def test_load_backtest_data_new_format(testdatadir): filename = testdatadir / "backtest_results/backtest-result_new.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert set(bt_data.columns) == set(BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) + assert set(bt_data.columns) == set(BT_DATA_COLUMNS) assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -110,7 +110,7 @@ def test_load_backtest_data_multi(testdatadir): bt_data = load_backtest_data(filename, strategy=strategy) assert isinstance(bt_data, DataFrame) assert set(bt_data.columns) == set( - BT_DATA_COLUMNS + ['close_timestamp', 'open_timestamp']) + BT_DATA_COLUMNS) assert len(bt_data) == 179 # Test loading from string (must yield same result) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f169e0a35..6912184aa 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -795,10 +795,27 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'is_open': [False, False], 'enter_tag': [None, None], "is_short": [False, False], + 'open_timestamp': [1517251200000, 1517283000000], + 'close_timestamp': [1517265300000, 1517285400000], + 'orders': [ + [ + {'amount': 0.00957442, 'safe_price': 0.104445, 'ft_order_side': 'buy', + 'order_filled_timestamp': 1517251200000, 'ft_is_entry': True}, + {'amount': 0.00957442, 'safe_price': 0.10496853383458644, 'ft_order_side': 'sell', + 'order_filled_timestamp': 1517265300000, 'ft_is_entry': False} + ], [ + {'amount': 0.0097064, 'safe_price': 0.10302485, 'ft_order_side': 'buy', + 'order_filled_timestamp': 1517283000000, 'ft_is_entry': True}, + {'amount': 0.0097064, 'safe_price': 0.10354126528822055, 'ft_order_side': 'sell', + 'order_filled_timestamp': 1517285400000, 'ft_is_entry': False} + ] + ] }) pd.testing.assert_frame_equal(results, expected) + assert 'orders' in results.columns data_pair = processed[pair] for _, t in results.iterrows(): + assert len(t['orders']) == 2 ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate assert ln is not None diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 94505e3ce..fca9c01b2 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -70,9 +70,14 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'is_open': [False, False], 'enter_tag': [None, None], 'is_short': [False, False], + 'open_timestamp': [1517251200000, 1517283000000], + 'close_timestamp': [1517265300000, 1517285400000], }) - pd.testing.assert_frame_equal(results, expected) + pd.testing.assert_frame_equal(results.drop(columns=['orders']), expected) data_pair = processed[pair] + assert len(results.iloc[0]['orders']) == 6 + assert len(results.iloc[1]['orders']) == 2 + for _, t in results.iterrows(): ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate From 057be50941c25fb493b90086dabc7997987b7f05 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Jun 2022 11:11:47 +0200 Subject: [PATCH 17/22] Remove old concurrency method --- .github/workflows/ci.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3ed6d80d..bbe0bcf6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -300,18 +300,6 @@ jobs: details: Freqtrade doc test failed! webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} - # cleanup-prior-runs: - # permissions: - # actions: write # for rokroskar/workflow-run-cleanup-action to obtain workflow name & cancel it - # contents: read # for rokroskar/workflow-run-cleanup-action to obtain branch - # runs-on: ubuntu-20.04 - # steps: - # - name: Cleanup previous runs on this branch - # uses: rokroskar/workflow-run-cleanup-action@v0.3.3 - # if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" - # env: - # GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: needs: [ build_linux, build_macos, build_windows, docs_check, mypy_version_check ] From 9534d6cca177de8aee7edd330fc8103e8d07e4bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Jun 2022 07:03:40 +0200 Subject: [PATCH 18/22] Cancel orders which can no longer be found after several days --- freqtrade/freqtradebot.py | 11 ++++++++++- tests/test_freqtradebot.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 95eb911cf..d96c63bcc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -4,7 +4,7 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() import copy import logging import traceback -from datetime import datetime, time, timezone +from datetime import datetime, time, timedelta, timezone from math import isclose from threading import Lock from typing import Any, Dict, List, Optional, Tuple @@ -302,6 +302,15 @@ class FreqtradeBot(LoggingMixin): self.update_trade_state(order.trade, order.order_id, fo, stoploss_order=(order.ft_order_side == 'stoploss')) + except InvalidOrderException as e: + logger.warning(f"Error updating Order {order.order_id} due to {e}.") + if order.order_date_utc - timedelta(days=5) < datetime.now(timezone.utc): + logger.warning( + "Order is older than 5 days. Assuming order was fully cancelled.") + fo = order.to_ccxt_object() + fo['status'] = 'canceled' + self.handle_timedout_order(fo, order.trade) + except ExchangeError as e: logger.warning(f"Error updating Order {order.order_id} due to {e}") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 0e4f9db99..cd7459cbe 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4802,10 +4802,19 @@ def test_startup_update_open_orders(mocker, default_conf_usdt, fee, caplog, is_s assert len(Order.get_open_orders()) == 2 caplog.clear() - mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=InvalidOrderException) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=ExchangeError) freqtrade.startup_update_open_orders() assert log_has_re(r"Error updating Order .*", caplog) + mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=InvalidOrderException) + hto_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_timedout_order') + # Orders which are no longer found after X days should be assumed as canceled. + freqtrade.startup_update_open_orders() + assert log_has_re(r"Order is older than \d days.*", caplog) + assert hto_mock.call_count == 2 + assert hto_mock.call_args_list[0][0][0]['status'] == 'canceled' + assert hto_mock.call_args_list[1][0][0]['status'] == 'canceled' + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize("is_short", [False, True]) From 381d64833d30ee10684e0633826a473d4f873197 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Jun 2022 21:05:31 +0200 Subject: [PATCH 19/22] version-bump ccxt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index acaecd872..05d5a10db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.22.4 pandas==1.4.2 pandas-ta==0.3.14b -ccxt==1.84.97 +ccxt==1.85.57 # Pin cryptography for now due to rust build errors with piwheels cryptography==37.0.2 aiohttp==3.8.1 From 7eacb847b05c53f7db80016885303be654cfb64b Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Jun 2022 20:21:45 +0200 Subject: [PATCH 20/22] Fix backtesting bug when order is not replaced --- freqtrade/optimize/backtesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 8fe5f509e..1aad8520a 100755 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -966,6 +966,7 @@ class Backtesting: return False else: del trade.orders[trade.orders.index(order)] + trade.open_order_id = None self.canceled_entry_orders += 1 # place new order if result was not None From d265b8adb621f93cee91d9fdea85a52f9d425171 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jun 2022 03:01:48 +0000 Subject: [PATCH 21/22] Bump python from 3.10.4-slim-bullseye to 3.10.5-slim-bullseye Bumps python from 3.10.4-slim-bullseye to 3.10.5-slim-bullseye. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5f7b52265..5138ecec9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.4-slim-bullseye as base +FROM python:3.10.5-slim-bullseye as base # Setup env ENV LANG C.UTF-8 From c550cd8b0d2b8559f22a91c87e01c7afd1b00dd2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Jun 2022 07:04:46 +0200 Subject: [PATCH 22/22] Simplify query in freqtradebot --- freqtrade/freqtradebot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d96c63bcc..fdccc2f8a 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -227,7 +227,7 @@ class FreqtradeBot(LoggingMixin): Notify the user when the bot is stopped (not reloaded) and there are still open trades active. """ - open_trades = Trade.get_trades([Trade.is_open.is_(True)]).all() + open_trades = Trade.get_open_trades() if len(open_trades) != 0 and self.state != State.RELOAD_CONFIG: msg = {