From 75bc5809a9b18bf1c1b97ce922046fe764be62f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 20:02:51 +0100 Subject: [PATCH 1/7] Better handle backtest errors --- freqtrade/rpc/api_server/api_backtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index ce71467ca..77de33994 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -118,7 +118,7 @@ async def api_start_backtest( # noqa: C901 logger.info("Backtest finished.") - except (OperationalException, DependencyException) as e: + except (Exception, OperationalException, DependencyException) as e: logger.exception(f"Backtesting caused an error: {e}") pass finally: From e6766b9b82805d0673155b7da7fc8eca121ec21c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 20:22:59 +0100 Subject: [PATCH 2/7] Add bt-error to UI backtest method. --- freqtrade/rpc/api_server/api_backtest.py | 10 ++++++++++ freqtrade/rpc/api_server/webserver.py | 1 + tests/rpc/test_rpc_apiserver.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index 77de33994..d9d7a27f1 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -29,6 +29,7 @@ router = APIRouter() async def api_start_backtest( # noqa: C901 bt_settings: BacktestRequest, background_tasks: BackgroundTasks, config=Depends(get_config), ws_mode=Depends(is_webserver_mode)): + ApiServer._bt['bt_error'] = None """Start backtesting if not done so already""" if ApiServer._bgtask_running: raise RPCException('Bot Background task already running') @@ -120,6 +121,7 @@ async def api_start_backtest( # noqa: C901 except (Exception, OperationalException, DependencyException) as e: logger.exception(f"Backtesting caused an error: {e}") + ApiServer._bt['bt_error'] = str(e) pass finally: ApiServer._bgtask_running = False @@ -162,6 +164,14 @@ def api_get_backtest(ws_mode=Depends(is_webserver_mode)): "progress": 0, "status_msg": "Backtest not yet executed" } + if ApiServer._bt['bt_error']: + return { + "status": "error", + "running": False, + "step": "", + "progress": 0, + "status_msg": f"Backtest failed with {ApiServer._bt['bt_error']}" + } return { "status": "ended", diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index b3ef794d8..b53662451 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -41,6 +41,7 @@ class ApiServer(RPCHandler): 'data': None, 'timerange': None, 'last_config': {}, + 'bt_error': None, } _has_rpc: bool = False _bgtask_running: bool = False diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 94b210c76..43d9abb78 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1737,9 +1737,15 @@ def test_api_backtesting(botclient, mocker, fee, caplog, tmpdir): data['stake_amount'] = 101 mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', - side_effect=DependencyException()) + side_effect=DependencyException('DeadBeef')) rc = client_post(client, f"{BASE_URI}/backtest", data=data) - assert log_has("Backtesting caused an error: ", caplog) + assert log_has("Backtesting caused an error: DeadBeef", caplog) + + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'error' + assert 'Backtest failed' in result['status_msg'] # Delete backtesting to avoid leakage since the backtest-object may stick around. rc = client_delete(client, f"{BASE_URI}/backtest") From 2bc9413be11288d0b42c0c354cb2d988eca5af22 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 20:58:24 +0100 Subject: [PATCH 3/7] Fix minor stylistic errors --- freqtrade/__main__.py | 0 freqtrade/commands/analyze_commands.py | 0 freqtrade/commands/hyperopt_commands.py | 0 freqtrade/data/entryexitanalysis.py | 0 freqtrade/optimize/hyperopt_tools.py | 0 freqtrade/vendor/qtpylib/indicators.py | 1 - freqtrade/worker.py | 0 scripts/ws_client.py | 0 tests/data/test_entryexitanalysis.py | 0 tests/exchange/test_ccxt_compat.py | 2 +- tests/strategy/test_interface.py | 4 ++-- 11 files changed, 3 insertions(+), 4 deletions(-) mode change 100644 => 100755 freqtrade/__main__.py mode change 100755 => 100644 freqtrade/commands/analyze_commands.py mode change 100755 => 100644 freqtrade/commands/hyperopt_commands.py mode change 100755 => 100644 freqtrade/data/entryexitanalysis.py mode change 100755 => 100644 freqtrade/optimize/hyperopt_tools.py mode change 100755 => 100644 freqtrade/worker.py mode change 100644 => 100755 scripts/ws_client.py mode change 100755 => 100644 tests/data/test_entryexitanalysis.py diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py old mode 100644 new mode 100755 diff --git a/freqtrade/commands/analyze_commands.py b/freqtrade/commands/analyze_commands.py old mode 100755 new mode 100644 diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py old mode 100755 new mode 100644 diff --git a/freqtrade/data/entryexitanalysis.py b/freqtrade/data/entryexitanalysis.py old mode 100755 new mode 100644 diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py old mode 100755 new mode 100644 diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index 4f14ae13c..3da4f038d 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # # QTPyLib: Quantitative Trading Python Library diff --git a/freqtrade/worker.py b/freqtrade/worker.py old mode 100755 new mode 100644 diff --git a/scripts/ws_client.py b/scripts/ws_client.py old mode 100644 new mode 100755 diff --git a/tests/data/test_entryexitanalysis.py b/tests/data/test_entryexitanalysis.py old mode 100755 new mode 100644 diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index f1d240f9f..bbeb56c6a 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -309,7 +309,7 @@ def exchange(request, exchange_conf): @pytest.fixture(params=EXCHANGES, scope="class") def exchange_futures(request, exchange_conf, class_mocker): - if not EXCHANGES[request.param].get('futures') is True: + if EXCHANGES[request.param].get('futures') is not True: yield None, request.param else: exchange_conf = set_test_proxy( diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index fe562907a..0b30d2059 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -214,12 +214,12 @@ def test_ignore_expired_candle(default_conf): current_time = latest_date + timedelta(seconds=30 + 300) - assert not strategy.ignore_expired_candle( + assert strategy.ignore_expired_candle( latest_date=latest_date, current_time=current_time, timeframe_seconds=300, enter=True - ) is True + ) is not True def test_assert_df_raise(mocker, caplog, ohlcv_history): From 549a0e1c447df9fe75ee834dad58d83ee4cd8bb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 21:06:07 +0100 Subject: [PATCH 4/7] Add ruff linting - initial configuration --- .github/workflows/ci.yml | 12 ++++++++++++ .pre-commit-config.yaml | 6 ++++++ pyproject.toml | 14 ++++++++++++++ requirements-dev.txt | 1 + 4 files changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc8ac3b6..4cff9ef7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,10 @@ jobs: run: | isort --check . + - name: Ruff - linting + run: | + ruff check . + - name: Mypy run: | mypy freqtrade scripts tests @@ -194,6 +198,10 @@ jobs: run: | isort --check . + - name: Ruff - linting + run: | + ruff check . + - name: Mypy run: | mypy freqtrade scripts @@ -252,6 +260,10 @@ jobs: run: | flake8 + - name: Ruff - linting + run: | + ruff check . + - name: Mypy run: | mypy freqtrade scripts tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57ce81b8c..58f526ce9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,12 @@ repos: name: isort (python) # stages: [push] + - repo: https://github.com/charliermarsh/ruff-pre-commit + # Ruff version. + rev: 'v0.0.251' + hooks: + - id: ruff + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 82d4ceaf8..7da2a2200 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,17 @@ exclude = [ "build_helpers/*.py", ] ignore = ["freqtrade/vendor/**"] + + +[tool.ruff] +line-length = 100 +extend-exclude = [".env"] +extend-select = [ + "TID", + "EXE", + "YTT", + # "DTZ", + # "RSE", + # "TCH", + # "PTH", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 32b7cfcc5..287cb8ae9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,6 +9,7 @@ coveralls==3.3.1 flake8==6.0.0 flake8-tidy-imports==4.8.0 +ruff==0.0.251 mypy==1.0.1 pre-commit==3.0.4 pytest==7.2.1 From b4ea37d59862ef5769468a0749c5f4e4ee4254da Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Feb 2023 21:08:17 +0100 Subject: [PATCH 5/7] Remove flake8 in favor of ruff --- .github/workflows/ci.yml | 12 ------------ CONTRIBUTING.md | 11 ++++++----- docs/developer.md | 2 +- environment.yml | 3 +-- requirements-dev.txt | 2 -- setup.py | 2 -- 6 files changed, 8 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cff9ef7a..e00b9040b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,10 +90,6 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 6 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Flake8 - run: | - flake8 - - name: Sort imports (isort) run: | isort --check . @@ -190,10 +186,6 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Flake8 - run: | - flake8 - - name: Sort imports (isort) run: | isort --check . @@ -256,10 +248,6 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Flake8 - run: | - flake8 - - name: Ruff - linting run: | ruff check . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4e0bc024..040aae39c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,16 +45,17 @@ pytest tests/test_.py::test_ ### 2. Test if your code is PEP8 compliant -#### Run Flake8 +#### Run Ruff ```bash -flake8 freqtrade tests scripts +ruff . ``` -We receive a lot of code that fails the `flake8` checks. +We receive a lot of code that fails the `ruff` checks. To help with that, we encourage you to install the git pre-commit -hook that will warn you when you try to commit code that fails these checks. -Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using-hooks.html). +hook that will warn you when you try to commit code that fails these checks. + +you can manually run pre-commit with `pre-commit run -a`. ##### Additional styles applied diff --git a/docs/developer.md b/docs/developer.md index 0546c20e9..1bc75551f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -24,7 +24,7 @@ This will spin up a local server (usually on port 8000) so you can see if everyt To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ". Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`. -This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`. +This will install all required tools for development, including `pytest`, `ruff`, `mypy`, and `coveralls`. Then install the git hook scripts by running `pre-commit install`, so your changes will be verified locally before committing. This avoids a lot of waiting for CI already, as some basic formatting checks are done locally on your machine. diff --git a/environment.yml b/environment.yml index 5b039e7f7..171e7c3da 100644 --- a/environment.yml +++ b/environment.yml @@ -41,7 +41,6 @@ dependencies: # 2/4 req dev - coveralls - - flake8 - mypy - pytest - pytest-asyncio @@ -70,6 +69,6 @@ dependencies: - tables - pytest-random-order - ccxt - - flake8-tidy-imports + - ruff - -e . # - python-rapidjso diff --git a/requirements-dev.txt b/requirements-dev.txt index 287cb8ae9..c23447694 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,8 +7,6 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -flake8==6.0.0 -flake8-tidy-imports==4.8.0 ruff==0.0.251 mypy==1.0.1 pre-commit==3.0.4 diff --git a/setup.py b/setup.py index 30aacc3f2..edd7b243b 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,6 @@ hdf5 = [ develop = [ 'coveralls', - 'flake8', - 'flake8-tidy-imports', 'mypy', 'pytest', 'pytest-asyncio', From bf968a9fd878cfdb37fe7b0a5865f82767921cf6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Feb 2023 06:51:03 +0100 Subject: [PATCH 6/7] Use actions as documented --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e00b9040b..17c0efd6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,9 +94,9 @@ jobs: run: | isort --check . - - name: Ruff - linting + - name: Run Ruff run: | - ruff check . + ruff check --format=github . - name: Mypy run: | @@ -190,9 +190,9 @@ jobs: run: | isort --check . - - name: Ruff - linting + - name: Run Ruff run: | - ruff check . + ruff check --format=github . - name: Mypy run: | @@ -248,9 +248,9 @@ jobs: freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - - name: Ruff - linting + - name: Run Ruff run: | - ruff check . + ruff check --format=github . - name: Mypy run: | From 6b829d839b93d5a793a7ff70bfea632ce6a05655 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Feb 2023 07:12:54 +0100 Subject: [PATCH 7/7] Improve ruff config --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7da2a2200..8a7750731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,11 +62,11 @@ ignore = ["freqtrade/vendor/**"] line-length = 100 extend-exclude = [".env"] extend-select = [ - "TID", - "EXE", - "YTT", - # "DTZ", - # "RSE", - # "TCH", - # "PTH", + "TID", # flake8-tidy-imports + # "EXE", # flake8-executable + "YTT", # flake8-2020 + # "DTZ", # flake8-datetimez + # "RSE", # flake8-raise + # "TCH", # flake8-type-checking + # "PTH", # flake8-use-pathlib ]