diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 44ff606b4..4d1f28a0d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,17 @@ updates: schedule: interval: daily open-pull-requests-limit: 10 + - package-ecosystem: pip directory: "/" schedule: interval: weekly open-pull-requests-limit: 10 target-branch: develop + +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 + target-branch: develop diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 228a60389..6b7d7cc29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,6 @@ name: Freqtrade CI on: push: branches: - - master - stable - develop tags: @@ -20,7 +19,7 @@ jobs: strategy: matrix: os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -39,7 +38,7 @@ jobs: - name: pip cache (linux) uses: actions/cache@v2 - if: startsWith(matrix.os, 'ubuntu') + if: runner.os == 'Linux' with: path: ~/.cache/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -50,8 +49,9 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - *nix + if: runner.os == 'Linux' run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH export TA_LIBRARY_PATH=${HOME}/dependencies/lib export TA_INCLUDE_PATH=${HOME}/dependencies/include @@ -69,7 +69,7 @@ jobs: if: matrix.python-version == '3.9' - name: Coveralls - if: (startsWith(matrix.os, 'ubuntu-20') && matrix.python-version == '3.8') + if: (runner.os == 'Linux' && matrix.python-version == '3.8') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu @@ -101,23 +101,20 @@ jobs: run: | mypy freqtrade scripts - - name: Slack Notification - uses: lazy-actions/slatify@v3.0.0 + - name: Discord notification + uses: rjstone/discord-webhook-notify@v1 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: - type: ${{ job.status }} - job_name: '*Freqtrade CI ${{ matrix.os }}*' - mention: 'here' - mention_if: 'failure' - channel: '#notifications' - url: ${{ secrets.SLACK_WEBHOOK }} + severity: error + details: Freqtrade CI failed on ${{ matrix.os }} + webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} build_macos: runs-on: ${{ matrix.os }} strategy: matrix: os: [ macos-latest ] - python-version: [3.7, 3.8, 3.9] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -136,7 +133,7 @@ jobs: - name: pip cache (macOS) uses: actions/cache@v2 - if: startsWith(matrix.os, 'macOS') + if: runner.os == 'macOS' with: path: ~/Library/Caches/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -147,10 +144,11 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - macOS + if: runner.os == 'macOS' run: | brew update brew install hdf5 c-blosc - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH export TA_LIBRARY_PATH=${HOME}/dependencies/lib export TA_INCLUDE_PATH=${HOME}/dependencies/include @@ -162,7 +160,7 @@ jobs: pytest --random-order --cov=freqtrade --cov-config=.coveragerc - name: Coveralls - if: (startsWith(matrix.os, 'ubuntu-20') && matrix.python-version == '3.8') + if: (runner.os == 'Linux' && matrix.python-version == '3.8') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu @@ -194,17 +192,13 @@ jobs: run: | mypy freqtrade scripts - - name: Slack Notification - uses: lazy-actions/slatify@v3.0.0 + - name: Discord notification + uses: rjstone/discord-webhook-notify@v1 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: - type: ${{ job.status }} - job_name: '*Freqtrade CI ${{ matrix.os }}*' - mention: 'here' - mention_if: 'failure' - channel: '#notifications' - url: ${{ secrets.SLACK_WEBHOOK }} - + severity: error + details: Test Succeeded! + webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} build_windows: @@ -212,7 +206,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: [3.7, 3.8] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 @@ -224,7 +218,6 @@ jobs: - name: Pip cache (Windows) uses: actions/cache@preview - if: startsWith(runner.os, 'Windows') with: path: ~\AppData\Local\pip\Cache key: ${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -257,16 +250,13 @@ jobs: run: | mypy freqtrade scripts - - name: Slack Notification - uses: lazy-actions/slatify@v3.0.0 + - name: Discord notification + uses: rjstone/discord-webhook-notify@v1 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: - type: ${{ job.status }} - job_name: '*Freqtrade CI windows*' - mention: 'here' - mention_if: 'failure' - channel: '#notifications' - url: ${{ secrets.SLACK_WEBHOOK }} + severity: error + details: Test Failed + webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} docs_check: runs-on: ubuntu-20.04 @@ -288,14 +278,13 @@ jobs: pip install mkdocs mkdocs build - - name: Slack Notification - uses: lazy-actions/slatify@v3.0.0 + - name: Discord notification + uses: rjstone/discord-webhook-notify@v1 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: - type: ${{ job.status }} - job_name: '*Freqtrade Docs*' - channel: '#notifications' - url: ${{ secrets.SLACK_WEBHOOK }} + severity: error + details: Freqtrade doc test failed! + webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} cleanup-prior-runs: runs-on: ubuntu-20.04 @@ -306,7 +295,7 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - # Notify on slack only once - when CI completes (and after deploy) in case it's successfull + # Notify only once - when CI completes (and after deploy) in case it's successfull notify-complete: needs: [ build_linux, build_macos, build_windows, docs_check ] runs-on: ubuntu-20.04 @@ -320,14 +309,13 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Slack Notification - uses: lazy-actions/slatify@v3.0.0 + - name: Discord notification + uses: rjstone/discord-webhook-notify@v1 if: always() && steps.check.outputs.has-permission && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: - type: ${{ job.status }} - job_name: '*Freqtrade CI*' - channel: '#notifications' - url: ${{ secrets.SLACK_WEBHOOK }} + severity: info + details: Test Completed! + webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} deploy: needs: [ build_linux, build_macos, build_windows, docs_check ] @@ -385,7 +373,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: crazy-max/ghaction-docker-buildx@v1 + uses: crazy-max/ghaction-docker-buildx@v3.3.1 with: buildx-version: latest qemu-version: latest @@ -400,17 +388,13 @@ jobs: run: | build_helpers/publish_docker_multi.sh - - - name: Slack Notification - uses: lazy-actions/slatify@v3.0.0 + - name: Discord notification + uses: rjstone/discord-webhook-notify@v1 if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: - type: ${{ job.status }} - job_name: '*Freqtrade CI Deploy*' - mention: 'here' - mention_if: 'failure' - channel: '#notifications' - url: ${{ secrets.SLACK_WEBHOOK }} + severity: info + details: Deploy Succeeded! + webhookUrl: ${{ secrets.DISCORD_WEBHOOK }} deploy_arm: diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 95e69be2a..95a6e82e2 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v2.1.0 + uses: peter-evans/dockerhub-description@v2.4.3 env: DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKERHUB_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 15c174bfe..000000000 --- a/.travis.yml +++ /dev/null @@ -1,55 +0,0 @@ -os: -- linux -dist: bionic -language: python -python: -- 3.8 -services: - - docker -env: - global: - - IMAGE_NAME=freqtradeorg/freqtrade -install: -- cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies; cd .. -- export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH -- export TA_LIBRARY_PATH=${HOME}/dependencies/lib -- export TA_INCLUDE_PATH=${HOME}/dependencies/include -- pip install -r requirements-dev.txt -- pip install -e . -jobs: - - include: - - stage: tests - script: - - pytest --random-order --cov=freqtrade --cov-config=.coveragerc - # Allow failure for coveralls - # - coveralls || true - name: pytest - - script: - - cp config_examples/config_bittrex.example.json config.json - - freqtrade create-userdir --userdir user_data - - freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: backtest - - script: - - cp config_examples/config_bittrex.example.json config.json - - freqtrade create-userdir --userdir user_data - - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily - name: hyperopt - - script: flake8 - name: flake8 - - script: - # Test Documentation boxes - - # !!! : is not allowed! - # !!! "title" - Title needs to be quoted! - - grep -Er '^!{3}\s\S+:|^!{3}\s\S+\s[^"]' docs/*; test $? -ne 0 - name: doc syntax - - script: mypy freqtrade scripts - name: mypy - -notifications: - slack: - secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= -cache: - pip: True - directories: - - $HOME/dependencies diff --git a/Dockerfile b/Dockerfile index 8f5b85698..1d283e5c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.9-slim-bullseye as base +FROM python:3.10.0-slim-bullseye as base # Setup env ENV LANG C.UTF-8 diff --git a/README.md b/README.md index 9882bce02..3a7d42fe9 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements -- [Python 3.7.x](http://docs.python-guide.org/en/latest/starting/installation/) +- [Python >= 3.7](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) diff --git a/build_helpers/TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl deleted file mode 100644 index bccfd090f..000000000 Binary files a/build_helpers/TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.21-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.21-cp38-cp38-win_amd64.whl deleted file mode 100644 index 67b41bf99..000000000 Binary files a/build_helpers/TA_Lib-0.4.21-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.21-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.21-cp39-cp39-win_amd64.whl deleted file mode 100644 index da9d74558..000000000 Binary files a/build_helpers/TA_Lib-0.4.21-cp39-cp39-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.23-cp310-cp310-win_amd64.whl b/build_helpers/TA_Lib-0.4.23-cp310-cp310-win_amd64.whl new file mode 100644 index 000000000..35d35d956 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.23-cp310-cp310-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.23-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.23-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..b06b09a7d Binary files /dev/null and b/build_helpers/TA_Lib-0.4.23-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.23-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.23-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..b51557cac Binary files /dev/null and b/build_helpers/TA_Lib-0.4.23-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.23-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.23-cp39-cp39-win_amd64.whl new file mode 100644 index 000000000..eb5453f34 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.23-cp39-cp39-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index ec38ea212..c982b3a05 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -1,19 +1,21 @@ # Downloads don't work automatically, since the URL is regenerated via javascript. # Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib -python -m pip install --upgrade pip +python -m pip install --upgrade pip wheel $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" if ($pyv -eq '3.7') { - pip install build_helpers\TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.23-cp37-cp37m-win_amd64.whl } if ($pyv -eq '3.8') { - pip install build_helpers\TA_Lib-0.4.21-cp38-cp38-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.23-cp38-cp38-win_amd64.whl } if ($pyv -eq '3.9') { - pip install build_helpers\TA_Lib-0.4.21-cp39-cp39-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.23-cp39-cp39-win_amd64.whl +} +if ($pyv -eq '3.10') { + pip install build_helpers\TA_Lib-0.4.23-cp310-cp310-win_amd64.whl } - pip install -r requirements-dev.txt pip install -e . diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index f5a52ff49..9ac31bf16 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -13,7 +13,7 @@ A sample of this can be found below, which is identical to the Default Hyperopt ``` python from datetime import datetime -from typing import Dict +from typing import Any, Dict from pandas import DataFrame diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 02b0307e5..93a2025ed 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -176,12 +176,15 @@ Log messages are send to `syslog` with the `user` facility. So you can see them On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfile syslog` or `--logfile journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. For `rsyslog` the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add + ``` if $programname startswith "freqtrade" then -/var/log/freqtrade.log ``` + to one of the rsyslog configuration files, for example at the end of the `/etc/rsyslog.d/50-default.conf`. For `syslog` (`rsyslog`), the reduction mode can be switched on. This will reduce the number of repeating messages. For instance, multiple bot Heartbeat messages will be reduced to a single message when nothing else happens with the bot. To achieve this, set in `/etc/rsyslog.conf`: + ``` # Filter duplicated messages $RepeatedMsgReduction on diff --git a/docs/assets/plot-profit.png b/docs/assets/plot-profit.png index 88d69a2d4..e9fe6c341 100644 Binary files a/docs/assets/plot-profit.png and b/docs/assets/plot-profit.png differ diff --git a/docs/backtesting.md b/docs/backtesting.md index a49e4700a..ad62c84b3 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -484,8 +484,8 @@ Since backtesting lacks some detailed information about what happens within a ca - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies - Sell-reason does not explain if a trade was positive or negative, just what triggered the sell (this can look odd if negative ROI values are used) - Evaluation sequence (if multiple signals happen on the same candle) - - ROI (if not stoploss) - Sell-signal + - ROI (if not stoploss) - Stoploss Taking these assumptions, backtesting tries to mirror real trading as closely as possible. However, backtesting will **never** replace running a strategy in dry-run mode. diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 80443a0bf..0b9f7b67c 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -56,7 +56,11 @@ This loop will be repeated again and again until the bot is stopped. * Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair). * Loops per candle simulating entry and exit points. * Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy). + * Call `custom_entry_price()` (if implemented in the strategy) to determine entry price (Prices are moved to be within the opening candle). + * Determine stake size by calling the `custom_stake_amount()` callback. * Call `custom_stoploss()` and `custom_sell()` to find custom exit points. + * For sells based on sell-signal and custom-sell: Call `custom_exit_price()` to determine exit price (Prices are moved to be within the closing candle). + * Generate backtest report output !!! Note diff --git a/docs/configuration.md b/docs/configuration.md index c4689f0a6..f18f947e7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -126,14 +126,16 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.uid` | API uid to use for the exchange. Only required when you are in production mode and for exchanges that use uid for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Supports regex pairs as `.*/BTC`. Not used by VolumePairList. [More information](plugins.md#pairlists-and-pairlist-handlers).
**Datatype:** List | `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting. [More information](plugins.md#pairlists-and-pairlist-handlers).
**Datatype:** List -| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict +| `exchange.ccxt_config` | Additional CCXT parameters passed to both ccxt instances (sync and async). This is usually the correct place for additional ccxt configurations. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation). Please avoid adding exchange secrets here (use the dedicated fields instead), as they may be contained in logs.
**Datatype:** Dict | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer | `exchange.skip_pair_validation` | Skip pairlist validation on startup.
*Defaults to `false`
**Datatype:** Boolean | `exchange.skip_open_order_update` | Skips open order updates on startup should the exchange cause problems. Only relevant in live conditions.
*Defaults to `false`
**Datatype:** Boolean +| `exchange.unknown_fee_rate` | Fallback value to use when calculating trading fees. This can be useful for exchanges which have fees in non-tradable currencies. The value provided here will be multiplied with the "fee cost".
*Defaults to `None`
**Datatype:** float | `exchange.log_responses` | Log relevant exchange responses. For debug mode only - use with care.
*Defaults to `false`
**Datatype:** Boolean | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean diff --git a/docs/developer.md b/docs/developer.md index b69a70aa3..ee4bac5c2 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -324,9 +324,8 @@ jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to markdown freqtrade This documents some decisions taken for the CI Pipeline. * CI runs on all OS variants, Linux (ubuntu), macOS and Windows. -* Docker images are build for the branches `stable` and `develop`. +* Docker images are build for the branches `stable` and `develop`, and are built as multiarch builds, supporting multiple platforms via the same tag. * Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`. -* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`. * Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of. * Full docker image rebuilds are run once a week via schedule. * Deployments run on ubuntu. diff --git a/docs/exchanges.md b/docs/exchanges.md index 3883e0b1d..374a6b8cc 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -199,6 +199,11 @@ OKEX requires a passphrase for each api key, you will therefore need to add this !!! Warning OKEX only provides 100 candles per api call. Therefore, the strategy will only have a pretty low amount of data available in backtesting mode. +## Gate.io + +Gate.io allows the use of `POINT` to pay for fees. As this is not a tradable currency (no regular market available), automatic fee calculations will fail (and default to a fee of 0). +The configuration parameter `exchange.unknown_fee_rate` can be used to specify the exchange rate between Point and the stake currency. Obviously, changing the stake-currency will also require changes to this value. + ## All exchanges Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 29e20a32f..031397719 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -196,7 +196,7 @@ Trade count is used as a tie breaker. You can use the `minutes` parameter to only consider performance of the past X minutes (rolling window). Not defining this parameter (or setting it to 0) will use all-time performance. -The optional `min_profit` parameter defines the minimum profit a pair must have to be considered. +The optional `min_profit` (as ratio -> a setting of `0.01` corresponds to 1%) parameter defines the minimum profit a pair must have to be considered. Pairs below this level will be filtered out. Using this parameter without `minutes` is highly discouraged, as it can lead to an empty pairlist without a way to recover. @@ -206,7 +206,7 @@ Using this parameter without `minutes` is highly discouraged, as it can lead to { "method": "PerformanceFilter", "minutes": 1440, // rolling 24h - "min_profit": 0.01 + "min_profit": 0.01 // minimal profit 1% } ], ``` @@ -260,7 +260,7 @@ Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. !!! Tip - You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. + You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. ShuffleFilter will automatically detect runmodes and apply the `seed` only for backtesting modes - if a `seed` value is set. #### SpreadFilter diff --git a/docs/installation.md b/docs/installation.md index ee7ffe55d..c67eff60b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -36,6 +36,10 @@ The easiest way to install and run Freqtrade is to clone the bot Github reposito These requirements apply to both [Script Installation](#script-installation) and [Manual Installation](#manual-installation). +!!! Note "ARM64 systems" + If you are running an ARM64 system (like a MacOS M1 or an Oracle VM), please use [docker](docker_quickstart.md) to run freqtrade. + While native installation is possible with some manual effort, this is not supported at the moment. + ### Install guide * [Python >= 3.7.x](http://docs.python-guide.org/en/latest/starting/installation/) @@ -416,16 +420,3 @@ open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10 ``` If this file is inexistent, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. - -### MacOS installation error with python 3.9 - -When using python 3.9 on macOS, it's currently necessary to install some os-level modules to allow dependencies to compile. -The errors you'll see happen during installation and are related to the installation of `tables` or `blosc`. - -You can install the necessary libraries with the following command: - -```bash -brew install hdf5 c-blosc -``` - -After this, please run the installation (script) again. diff --git a/docs/plotting.md b/docs/plotting.md index b2d7654f6..315dbc236 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -283,6 +283,8 @@ The `plot-profit` subcommand shows an interactive graph with three plots: * The summarized profit made by backtesting. Note that this is not the real-world profit, but more of an estimate. * Profit for each individual pair. +* Parallelism of trades. +* Underwater (Periods of drawdown). The first graph is good to get a grip of how the overall market progresses. @@ -292,6 +294,8 @@ This graph will also highlight the start (and end) of the Max drawdown period. The third graph can be useful to spot outliers, events in pairs that cause profit spikes. +The forth graph can help you analyze trade parallelism, showing how often max_open_trades have been maxed out. + Possible options for the `freqtrade plot-profit` subcommand: ``` diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 772919436..0dccfa17a 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.3 -mkdocs-material==7.3.6 +mkdocs-material==8.1.4 mdx_truly_sane_lists==1.2 pymdown-extensions==9.1 diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 573d184ff..4cc607883 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -127,6 +127,21 @@ The provided exit-tag is then used as sell-reason - and shown as such in backtes !!! Note `sell_reason` is limited to 100 characters, remaining data will be truncated. +## Strategy version + +You can implement custom strategy versioning by using the "version" method, and returning the version you would like this strategy to have. + +``` python +def version(self) -> str: + """ + Returns version of the strategy. + """ + return "1.1" +``` + +!!! Note + You should make sure to implement proper version control (like a git repository) alongside this, as freqtrade will not keep historic versions of your strategy, so it's up to the user to be able to eventually roll back to a prior version of the strategy. + ## Derived strategies The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: diff --git a/docs/strategy-callbacks.md b/docs/strategy-callbacks.md index 7a7756652..11032433d 100644 --- a/docs/strategy-callbacks.md +++ b/docs/strategy-callbacks.md @@ -387,8 +387,10 @@ class AwesomeStrategy(IStrategy): **Example**: If the new_entryprice is 97, the proposed_rate is 100 and the `custom_price_max_distance_ratio` is set to 2%, The retained valid custom entry price will be 98, which is 2% below the current (proposed) rate. -!!! Warning "No backtesting support" - Custom entry-prices are currently not supported during backtesting. +!!! Warning "Backtesting" + While Custom prices are supported in backtesting (starting with 2021.12), prices will be moved to within the candle's high/low prices. + This behavior is currently being tested, and might be changed at a later point. + `custom_exit_price()` is only called for sells of type Sell_signal and Custom sell. All other sell-types will use regular backtesting prices. ## Custom order timeout rules diff --git a/docs/webhook-config.md b/docs/webhook-config.md index ec944cb50..c93f9aac8 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -50,7 +50,7 @@ Sample configuration (tested using IFTTT). The url in `webhook.url` should point to the correct url for your webhook. If you're using [IFTTT](https://ifttt.com) (as shown in the sample above) please insert your event and key to the url. -You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use `"format": "form"` or `"format": "json"` respectively. Example configuration for Mattermost Cloud integration: +You can set the POST body format to Form-Encoded (default), JSON-Encoded, or raw data. Use `"format": "form"`, `"format": "json"`, or `"format": "raw"` respectively. Example configuration for Mattermost Cloud integration: ```json "webhook": { @@ -63,7 +63,36 @@ You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use }, ``` -The result would be POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. +The result would be a POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. + +When using the Form-Encoded or JSON-Encoded configuration you can configure any number of payload values, and both the key and value will be ouput in the POST request. However, when using the raw data format you can only configure one value and it **must** be named `"data"`. In this instance the data key will not be output in the POST request, only the value. For example: + +```json + "webhook": { + "enabled": true, + "url": "https://", + "format": "raw", + "webhookstatus": { + "data": "Status: {status}" + } + }, +``` + +The result would be a POST request with e.g. `Status: running` body and `Content-Type: text/plain` header. + +Optional parameters are available to enable automatic retries for webhook messages. The `webhook.retries` parameter can be set for the maximum number of retries the webhook request should attempt if it is unsuccessful (i.e. HTTP response status is not 200). By default this is set to `0` which is disabled. An additional `webhook.retry_delay` parameter can be set to specify the time in seconds between retry attempts. By default this is set to `0.1` (i.e. 100ms). Note that increasing the number of retries or retry delay may slow down the trader if there are connectivity issues with the webhook. Example configuration for retries: + +```json + "webhook": { + "enabled": true, + "url": "https://", + "retries": 3, + "retry_delay": 0.2, + "webhookstatus": { + "status": "Status: {status}" + } + }, +``` Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called. @@ -75,11 +104,13 @@ Possible parameters are: * `trade_id` * `exchange` * `pair` -* `limit` +* ~~`limit` # Deprecated - should no longer be used.~~ +* `open_rate` * `amount` * `open_date` * `stake_amount` * `stake_currency` +* `base_currency` * `fiat_currency` * `order_type` * `current_rate` @@ -98,6 +129,7 @@ Possible parameters are: * `open_date` * `stake_amount` * `stake_currency` +* `base_currency` * `fiat_currency` * `order_type` * `current_rate` @@ -116,7 +148,10 @@ Possible parameters are: * `open_date` * `stake_amount` * `stake_currency` +* `base_currency` * `fiat_currency` +* `order_type` +* `current_rate` * `buy_tag` ### Webhooksell @@ -134,6 +169,7 @@ Possible parameters are: * `profit_amount` * `profit_ratio` * `stake_currency` +* `base_currency` * `fiat_currency` * `sell_reason` * `order_type` @@ -156,6 +192,7 @@ Possible parameters are: * `profit_amount` * `profit_ratio` * `stake_currency` +* `base_currency` * `fiat_currency` * `sell_reason` * `order_type` @@ -178,6 +215,7 @@ Possible parameters are: * `profit_amount` * `profit_ratio` * `stake_currency` +* `base_currency` * `fiat_currency` * `sell_reason` * `order_type` diff --git a/docs/windows_installation.md b/docs/windows_installation.md index 2db0ae913..6f51dbf8f 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -23,9 +23,9 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which need to be downloaded and installed using `pip install TA_Lib-0.4.21-cp38-cp38-win_amd64.whl` (make sure to use the version matching your python version). +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which need to be downloaded and installed using `pip install TA_Lib-0.4.23-cp38-cp38-win_amd64.whl` (make sure to use the version matching your python version). -Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows. +Freqtrade provides these dependencies for the latest 3 Python versions (3.7, 3.8, 3.9 and 3.10) and for 64bit Windows. Other versions must be downloaded from the above link. ``` powershell diff --git a/freqtrade/configuration/PeriodicCache.py b/freqtrade/configuration/PeriodicCache.py index 25c0c47f3..64fff668e 100644 --- a/freqtrade/configuration/PeriodicCache.py +++ b/freqtrade/configuration/PeriodicCache.py @@ -1,6 +1,6 @@ from datetime import datetime, timezone -from cachetools.ttl import TTLCache +from cachetools import TTLCache class PeriodicCache(TTLCache): diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e775e39fc..f15759ea5 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -50,6 +50,8 @@ USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] +WEBHOOK_FORMAT_OPTIONS = ['form', 'json', 'raw'] + ENV_VAR_PREFIX = 'FREQTRADE__' NON_OPEN_EXCHANGE_STATES = ('cancelled', 'canceled', 'closed', 'expired') @@ -312,10 +314,16 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'enabled': {'type': 'boolean'}, + 'url': {'type': 'string'}, + 'format': {'type': 'string', 'enum': WEBHOOK_FORMAT_OPTIONS, 'default': 'form'}, + 'retries': {'type': 'integer', 'minimum': 0}, + 'retry_delay': {'type': 'number', 'minimum': 0}, 'webhookbuy': {'type': 'object'}, 'webhookbuycancel': {'type': 'object'}, + 'webhookbuyfill': {'type': 'object'}, 'webhooksell': {'type': 'object'}, 'webhooksellcancel': {'type': 'object'}, + 'webhooksellfill': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, }, }, @@ -387,6 +395,7 @@ CONF_SCHEMA = { }, 'uniqueItems': True }, + 'unknown_fee_rate': {'type': 'number'}, 'outdated_offset': {'type': 'integer', 'minimum': 1}, 'markets_refresh_interval': {'type': 'integer'}, 'ccxt_config': {'type': 'object'}, diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 7d97661c4..56c16f966 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -325,6 +325,7 @@ def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], :param column: Column in the original dataframes to use :return: DataFrame with the column renamed to the dict key, and a column named mean, containing the mean of all pairs. + :raise: ValueError if no data is provided. """ df_comb = pd.concat([data[pair].set_index('date').rename( {column: pair}, axis=1)[pair] for pair in data], axis=1) @@ -360,6 +361,36 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, return df +def _calc_drawdown_series(profit_results: pd.DataFrame, *, date_col: str, value_col: str + ) -> pd.DataFrame: + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + max_drawdown_df['date'] = profit_results.loc[:, date_col] + return max_drawdown_df + + +def calculate_underwater(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_ratio' + ): + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, + high and low time and high and low value. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) + + return max_drawdown_df + + def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', value_col: str = 'profit_ratio' ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float]: @@ -375,10 +406,7 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date' if len(trades) == 0: raise ValueError("Trade dataframe empty.") profit_results = trades.sort_values(date_col).reset_index(drop=True) - max_drawdown_df = pd.DataFrame() - max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() - max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() - max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + max_drawdown_df = _calc_drawdown_series(profit_results, date_col=date_col, value_col=value_col) idxmin = max_drawdown_df['drawdown'].idxmin() if idxmin == 0: diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index dd60530aa..49fac99ea 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -6,7 +6,6 @@ from typing import List, Optional import numpy as np import pandas as pd -from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, ListPairsWithTimeframes, TradeList) @@ -61,10 +60,10 @@ class HDF5DataHandler(IDataHandler): filename = self._pair_data_filename(self._datadir, pair, timeframe) - ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc') - ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date']) - - ds.close() + _data.loc[:, self._columns].to_hdf( + filename, key, mode='a', complevel=9, complib='blosc', + format='table', data_columns=['date'] + ) def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame: @@ -99,19 +98,6 @@ class HDF5DataHandler(IDataHandler): 'low': 'float', 'close': 'float', 'volume': 'float'}) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str) -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :param timeframe: Timeframe (e.g. "5m") - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_data_filename(self._datadir, pair, timeframe) - if filename.exists(): - filename.unlink() - return True - return False - def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: """ Append data to existing data structures @@ -142,11 +128,11 @@ class HDF5DataHandler(IDataHandler): """ key = self._pair_trades_key(pair) - ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair), - mode='a', complevel=9, complib='blosc') - ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS), - format='table', data_columns=['timestamp']) - ds.close() + pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS).to_hdf( + self._pair_trades_filename(self._datadir, pair), key, + mode='a', complevel=9, complib='blosc', + format='table', data_columns=['timestamp'] + ) def trades_append(self, pair: str, data: TradeList): """ @@ -180,17 +166,9 @@ class HDF5DataHandler(IDataHandler): trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None}) return trades.values.tolist() - def trades_purge(self, pair: str) -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_trades_filename(self._datadir, pair) - if filename.exists(): - filename.unlink() - return True - return False + @classmethod + def _get_file_extension(cls): + return "h5" @classmethod def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str: @@ -199,15 +177,3 @@ class HDF5DataHandler(IDataHandler): @classmethod def _pair_trades_key(cls, pair: str) -> str: return f"{pair}/trades" - - @classmethod - def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5') - return filename - - @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-trades.h5') - return filename diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 05052b2d7..cb02f98e3 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -12,6 +12,7 @@ from typing import List, Optional, Type from pandas import DataFrame +from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, TradeList from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe @@ -26,6 +27,13 @@ class IDataHandler(ABC): def __init__(self, datadir: Path) -> None: self._datadir = datadir + @classmethod + def _get_file_extension(cls) -> str: + """ + Get file extension for this particular datahandler + """ + raise NotImplementedError() + @abstractclassmethod def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: """ @@ -70,7 +78,6 @@ class IDataHandler(ABC): :return: DataFrame with ohlcv data, or empty DataFrame """ - @abstractmethod def ohlcv_purge(self, pair: str, timeframe: str) -> bool: """ Remove data for this pair @@ -78,6 +85,11 @@ class IDataHandler(ABC): :param timeframe: Timeframe (e.g. "5m") :return: True when deleted, false if file did not exist. """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False @abstractmethod def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: @@ -123,13 +135,17 @@ class IDataHandler(ABC): :return: List of trades """ - @abstractmethod def trades_purge(self, pair: str) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. :return: True when deleted, false if file did not exist. """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: """ @@ -141,6 +157,18 @@ class IDataHandler(ABC): """ return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + return filename + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') + return filename + def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, @@ -173,7 +201,7 @@ class IDataHandler(ABC): enddate = pairdf.iloc[-1]['date'] if timerange_startup: - self._validate_pairdata(pair, pairdf, timerange_startup) + self._validate_pairdata(pair, pairdf, timeframe, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): return pairdf @@ -200,7 +228,7 @@ class IDataHandler(ABC): return True return False - def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): + def _validate_pairdata(self, pair, pairdata: DataFrame, timeframe: str, timerange: TimeRange): """ Validates pairdata for missing data at start end end and logs warnings. :param pairdata: Dataframe to validate @@ -210,12 +238,12 @@ class IDataHandler(ABC): if timerange.starttype == 'date': start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) if pairdata.iloc[0]['date'] > start: - logger.warning(f"Missing data at start for pair {pair}, " + logger.warning(f"Missing data at start for pair {pair} at {timeframe}, " f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}") if timerange.stoptype == 'date': stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) if pairdata.iloc[-1]['date'] < stop: - logger.warning(f"Missing data at end for pair {pair}, " + logger.warning(f"Missing data at end for pair {pair} at {timeframe}, " f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 24d6e814b..ccefc8356 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -174,34 +174,10 @@ class JsonDataHandler(IDataHandler): pass return tradesdata - def trades_purge(self, pair: str) -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_trades_filename(self._datadir, pair) - if filename.exists(): - filename.unlink() - return True - return False - - @classmethod - def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') - return filename - @classmethod def _get_file_extension(cls): return "json.gz" if cls._use_zip else "json" - @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: - pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') - return filename - class JsonGzDataHandler(JsonDataHandler): diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index d803baf31..eab483db3 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 from freqtrade.enums.backteststate import BacktestState +from freqtrade.enums.ordertypevalue import OrderTypeValues from freqtrade.enums.rpcmessagetype import RPCMessageType from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode from freqtrade.enums.selltype import SellType diff --git a/freqtrade/enums/ordertypevalue.py b/freqtrade/enums/ordertypevalue.py new file mode 100644 index 000000000..9bb716171 --- /dev/null +++ b/freqtrade/enums/ordertypevalue.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class OrderTypeValues(str, Enum): + limit = 'limit' + market = 'market' diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 614e8ad68..b356a8147 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -5,6 +5,7 @@ from freqtrade.exchange.exchange import Exchange # isort: on from freqtrade.exchange.bibox import Bibox from freqtrade.exchange.binance import Binance +from freqtrade.exchange.bitpanda import Bitpanda from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.coinbasepro import Coinbasepro diff --git a/freqtrade/exchange/bitpanda.py b/freqtrade/exchange/bitpanda.py new file mode 100644 index 000000000..4cac35ce8 --- /dev/null +++ b/freqtrade/exchange/bitpanda.py @@ -0,0 +1,37 @@ +""" Bitpanda exchange subclass """ +import logging +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bitpanda(Exchange): + """ + Bitpanda exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + """ + + def get_trades_for_order(self, order_id: str, pair: str, since: datetime, + params: Optional[Dict] = None) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ + params = {'to': int(datetime.now(timezone.utc).timestamp() * 1000)} + return super().get_trades_for_order(order_id, pair, since, params) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index a4c827e07..3916ee8f7 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -4,9 +4,20 @@ import time from functools import wraps from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError +from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) +__logging_mixin = None + + +def _get_logging_mixin(): + # Logging-mixin to cache kucoin responses + # Only to be used in retrier + global __logging_mixin + if not __logging_mixin: + __logging_mixin = LoggingMixin(logger) + return __logging_mixin # Maximum default retry count. @@ -72,28 +83,33 @@ def calculate_backoff(retrycount, max_retries): def retrier_async(f): async def wrapper(*args, **kwargs): count = kwargs.pop('count', API_RETRY_COUNT) + kucoin = args[0].name == "Kucoin" # Check if the exchange is KuCoin. try: return await f(*args, **kwargs) except TemporaryError as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) + msg = f'{f.__name__}() returned exception: "{ex}". ' if count > 0: - logger.warning('retrying %s() still for %s times', f.__name__, count) + msg += f'Retrying still for {count} times.' count -= 1 - kwargs.update({'count': count}) + kwargs['count'] = count if isinstance(ex, DDosProtection): - if "kucoin" in str(ex) and "429000" in str(ex): + if kucoin and "429000" in str(ex): # Temporary fix for 429000 error on kucoin # see https://github.com/freqtrade/freqtrade/issues/5700 for details. - logger.warning( + _get_logging_mixin().log_once( f"Kucoin 429 error, avoid triggering DDosProtection backoff delay. " - f"{count} tries left before giving up") + f"{count} tries left before giving up", logmethod=logger.warning) + # Reset msg to avoid logging too many times. + msg = '' else: backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT) logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}") await asyncio.sleep(backoff_delay) + if msg: + logger.warning(msg) return await wrapper(*args, **kwargs) else: - logger.warning('Giving up retrying: %s()', f.__name__) + logger.warning(msg + 'Giving up.') raise ex return wrapper @@ -106,9 +122,9 @@ def retrier(_func=None, retries=API_RETRY_COUNT): try: return f(*args, **kwargs) except (TemporaryError, RetryableOrderError) as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) + msg = f'{f.__name__}() returned exception: "{ex}". ' if count > 0: - logger.warning('retrying %s() still for %s times', f.__name__, count) + logger.warning(msg + f'Retrying still for {count} times.') count -= 1 kwargs.update({'count': count}) if isinstance(ex, (DDosProtection, RetryableOrderError)): @@ -118,7 +134,7 @@ def retrier(_func=None, retries=API_RETRY_COUNT): time.sleep(backoff_delay) return wrapper(*args, **kwargs) else: - logger.warning('Giving up retrying: %s()', f.__name__) + logger.warning(msg + 'Giving up.') raise ex return wrapper # Support both @retrier and @retrier(retries=2) syntax diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 5fa852eb0..8bd9db9f6 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -83,6 +83,8 @@ class Exchange: self._api: ccxt.Exchange = None self._api_async: ccxt_async.Exchange = None self._markets: Dict = {} + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) self._config.update(config) @@ -170,8 +172,10 @@ class Exchange: def close(self): logger.debug("Exchange object destroyed, closing async loop") - if self._api_async and inspect.iscoroutinefunction(self._api_async.close): - asyncio.get_event_loop().run_until_complete(self._api_async.close()) + if (self._api_async and inspect.iscoroutinefunction(self._api_async.close) + and self._api_async.session): + logger.info("Closing async ccxt session.") + self.loop.run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, ccxt_kwargs: Dict = {}) -> ccxt.Exchange: @@ -326,7 +330,7 @@ class Exchange: def _load_async_markets(self, reload: bool = False) -> None: try: if self._api_async: - asyncio.get_event_loop().run_until_complete( + self.loop.run_until_complete( self._api_async.load_markets(reload=reload)) except (asyncio.TimeoutError, ccxt.BaseError) as e: @@ -685,16 +689,20 @@ class Exchange: if not self.exchange_has('fetchL2OrderBook'): return True ob = self.fetch_l2_order_book(pair, 1) - if side == 'buy': - price = ob['asks'][0][0] - logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") - if limit >= price: - return True - else: - price = ob['bids'][0][0] - logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") - if limit <= price: - return True + try: + if side == 'buy': + price = ob['asks'][0][0] + logger.debug(f"{pair} checking dry buy-order: price={price}, limit={limit}") + if limit >= price: + return True + else: + price = ob['bids'][0][0] + logger.debug(f"{pair} checking dry sell-order: price={price}, limit={limit}") + if limit <= price: + return True + except IndexError: + # Ignore empty orderbooks when filling - can be filled with the next iteration. + pass return False def check_dry_limit_order_filled(self, order: Dict[str, Any]) -> Dict[str, Any]: @@ -1087,7 +1095,8 @@ class Exchange: # Fee handling @retrier - def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + def get_trades_for_order(self, order_id: str, pair: str, since: datetime, + params: Optional[Dict] = None) -> List: """ Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. The "since" argument passed in is coming from the database and is in UTC, @@ -1111,8 +1120,10 @@ class Exchange: try: # Allow 5s offset to catch slight time offsets (discovered in #1185) # since needs to be int in milliseconds + _params = params if params else {} my_trades = self._api.fetch_my_trades( - pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000), + params=_params) matched_trades = [trade for trade in my_trades if trade['order'] == order_id] self._log_exchange_response('get_trades_for_order', matched_trades) @@ -1190,9 +1201,11 @@ class Exchange: tick = self.fetch_ticker(comb) fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') - return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) except ExchangeError: - return None + fee_to_quote_rate = self._config['exchange'].get('unknown_fee_rate', None) + if not fee_to_quote_rate: + return None + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: """ @@ -1218,7 +1231,7 @@ class Exchange: :param since_ms: Timestamp in milliseconds to get history from :return: List with candle (OHLCV) data """ - pair, timeframe, data = asyncio.get_event_loop().run_until_complete( + pair, timeframe, data = self.loop.run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair)) logger.info(f"Downloaded data for {pair} with length {len(data)}.") @@ -1263,7 +1276,7 @@ class Exchange: results = await asyncio.gather(*input_coro, return_exceptions=True) for res in results: if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) + logger.warning(f"Async code raised an exception: {repr(res)}") if raise_: raise continue @@ -1317,27 +1330,32 @@ class Exchange: ) cached_pairs.append((pair, timeframe)) - results = asyncio.get_event_loop().run_until_complete( - asyncio.gather(*input_coroutines, return_exceptions=True)) - results_df = {} - # handle caching - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple (has 3 elements) - pair, timeframe, ticks = res - # keeping last candle time as last refreshed time of the pair - if ticks: - self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - ohlcv_df = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - results_df[(pair, timeframe)] = ohlcv_df - if cache: - self._klines[(pair, timeframe)] = ohlcv_df + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling + for input_coro in chunks(input_coroutines, 100): + async def gather_stuff(): + return await asyncio.gather(*input_coro, return_exceptions=True) + + results = self.loop.run_until_complete(gather_stuff()) + + # handle caching + for res in results: + if isinstance(res, Exception): + logger.warning(f"Async code raised an exception: {repr(res)}") + continue + # Deconstruct tuple (has 3 elements) + pair, timeframe, ticks = res + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + ohlcv_df = ohlcv_to_dataframe( + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + results_df[(pair, timeframe)] = ohlcv_df + if cache: + self._klines[(pair, timeframe)] = ohlcv_df + # Return cached klines for pair, timeframe in cached_pairs: results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False) @@ -1554,7 +1572,7 @@ class Exchange: if not self.exchange_has("fetchTrades"): raise OperationalException("This exchange does not support downloading Trades.") - return asyncio.get_event_loop().run_until_complete( + return self.loop.run_until_complete( self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a6d1b36b9..6f9acf930 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -126,6 +126,7 @@ class FreqtradeBot(LoggingMixin): self.rpc.cleanup() cleanup_db() + self.exchange.close() def startup(self) -> None: """ @@ -278,7 +279,8 @@ class FreqtradeBot(LoggingMixin): if order: logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.") self.update_trade_state(trade, order.order_id, - stoploss_order=order.ft_order_side == 'stoploss') + stoploss_order=order.ft_order_side == 'stoploss', + send_msg=False) trades: List[Trade] = Trade.get_open_trades_without_assigned_fees() for trade in trades: @@ -286,7 +288,7 @@ class FreqtradeBot(LoggingMixin): order = trade.select_order('buy', False) if order: logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") - self.update_trade_state(trade, order.order_id) + self.update_trade_state(trade, order.order_id, send_msg=False) def handle_insufficient_funds(self, trade: Trade): """ @@ -308,7 +310,7 @@ class FreqtradeBot(LoggingMixin): order = trade.select_order('buy', False) if order: logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") - self.update_trade_state(trade, order.order_id) + self.update_trade_state(trade, order.order_id, send_msg=False) def refind_lost_order(self, trade): """ @@ -578,10 +580,6 @@ class FreqtradeBot(LoggingMixin): ) trade.orders.append(order_obj) - # Update fees if order is closed - if order_status == 'closed': - self.update_trade_state(trade, order_id, order) - Trade.query.session.add(trade) Trade.commit() @@ -590,19 +588,25 @@ class FreqtradeBot(LoggingMixin): self._notify_enter(trade, order_type) + # Update fees if order is closed + if order_status == 'closed': + self.update_trade_state(trade, order_id, order) + return True - def _notify_enter(self, trade: Trade, order_type: str) -> None: + def _notify_enter(self, trade: Trade, order_type: Optional[str] = None, + fill: bool = False) -> None: """ Sends rpc notification when a buy occurred. """ msg = { 'trade_id': trade.id, - 'type': RPCMessageType.BUY, + 'type': RPCMessageType.BUY_FILL if fill else RPCMessageType.BUY, 'buy_tag': trade.buy_tag, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, - 'limit': trade.open_rate, + 'limit': trade.open_rate, # Deprecated (?) + 'open_rate': trade.open_rate, 'order_type': order_type, 'stake_amount': trade.stake_amount, 'stake_currency': self.config['stake_currency'], @@ -641,22 +645,6 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_enter_fill(self, trade: Trade) -> None: - msg = { - 'trade_id': trade.id, - 'type': RPCMessageType.BUY_FILL, - 'buy_tag': trade.buy_tag, - 'exchange': self.exchange.name.capitalize(), - 'pair': trade.pair, - 'open_rate': trade.open_rate, - 'stake_amount': trade.stake_amount, - 'stake_currency': self.config['stake_currency'], - 'fiat_currency': self.config.get('fiat_display_currency', None), - 'amount': trade.amount, - 'open_date': trade.open_date, - } - self.rpc.send_msg(msg) - # # SELL / exit positions / close trades logic and methods # @@ -679,7 +667,7 @@ class FreqtradeBot(LoggingMixin): trades_closed += 1 except DependencyException as exception: - logger.warning('Unable to sell trade %s: %s', trade.pair, exception) + logger.warning(f'Unable to sell trade {trade.pair}: {exception}') # Updating wallets if any trade occurred if trades_closed: @@ -923,8 +911,12 @@ class FreqtradeBot(LoggingMixin): if max_timeouts > 0 and canceled_count >= max_timeouts: logger.warning(f'Emergencyselling trade {trade}, as the sell order ' f'timed out {max_timeouts} times.') - self.execute_trade_exit(trade, order.get('price'), sell_reason=SellCheckTuple( - sell_type=SellType.EMERGENCY_SELL)) + try: + self.execute_trade_exit( + trade, order.get('price'), + sell_reason=SellCheckTuple(sell_type=SellType.EMERGENCY_SELL)) + except DependencyException as exception: + logger.warning(f'Unable to emergency sell trade {trade.pair}: {exception}') def cancel_all_open_orders(self) -> None: """ @@ -1154,16 +1146,16 @@ class FreqtradeBot(LoggingMixin): trade.sell_order_status = '' trade.close_rate_requested = limit trade.sell_reason = exit_tag or sell_reason.sell_reason - # In case of market sell orders the order can be closed immediately - if order.get('status', 'unknown') in ('closed', 'expired'): - self.update_trade_state(trade, trade.open_order_id, order) - Trade.commit() # Lock pair for one candle to prevent immediate re-buys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') self._notify_exit(trade, order_type) + # In case of market sell orders the order can be closed immediately + if order.get('status', 'unknown') in ('closed', 'expired'): + self.update_trade_state(trade, trade.open_order_id, order) + Trade.commit() return True @@ -1260,13 +1252,14 @@ class FreqtradeBot(LoggingMixin): # def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, - stoploss_order: bool = False) -> bool: + stoploss_order: bool = False, send_msg: bool = True) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. :param trade: Trade object of the trade we're analyzing :param order_id: Order-id of the order we're analyzing :param action_order: Already acquired order object + :param send_msg: Send notification - should always be True except in "recovery" methods :return: True if order has been cancelled without being filled partially, False otherwise """ if not order_id: @@ -1306,13 +1299,13 @@ class FreqtradeBot(LoggingMixin): # Updating wallets when order is closed if not trade.is_open: - if not stoploss_order and not trade.open_order_id: + if send_msg and not stoploss_order and not trade.open_order_id: self._notify_exit(trade, '', True) self.handle_protections(trade.pair) self.wallets.update() - elif not trade.open_order_id: + elif send_msg and not trade.open_order_id: # Buy fill - self._notify_enter_fill(trade) + self._notify_enter(trade, fill=True) return False diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 58607dcd3..78e6fb002 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -246,6 +246,9 @@ class Backtesting: Helper function to convert a processed dataframes into lists for performance reasons. Used by backtest() - so keep this optimized for performance. + + :param processed: a processed dictionary with format {pair, data}, which gets cleared to + optimize memory usage! """ # Every change to this headers list must evaluate further usages of the resulting tuple # and eventually change the constants for indexes at the top @@ -254,7 +257,8 @@ class Backtesting: self.progress.init_step(BacktestState.CONVERT, len(processed)) # Create dict with data - for pair, pair_data in processed.items(): + for pair in processed.keys(): + pair_data = processed[pair] self.check_abort() self.progress.increment() if not pair_data.empty: @@ -283,6 +287,9 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) data[pair] = df_analyzed[headers].values.tolist() + + # Do not hold on to old data to reduce memory usage + processed[pair] = pair_data = None return data def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, @@ -342,10 +349,7 @@ class Backtesting: # use Open rate if open_rate > calculated sell rate return sell_row[OPEN_IDX] - # Use the maximum between close_rate and low as we - # cannot sell outside of a candle. - # Applies when a new ROI setting comes in place and the whole candle is above that. - return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) + return close_rate else: # This should not be reached... @@ -366,6 +370,17 @@ class Backtesting: trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) + # call the custom exit price,with default value as previous closerate + current_profit = trade.calc_profit_ratio(closerate) + if sell.sell_type in (SellType.SELL_SIGNAL, SellType.CUSTOM_SELL): + # Custom exit pricing only for sell-signals + closerate = strategy_safe_wrapper(self.strategy.custom_exit_price, + default_retval=closerate)( + pair=trade.pair, trade=trade, + current_time=sell_row[DATE_IDX], + proposed_rate=closerate, current_profit=current_profit) + # Use the maximum between close_rate and low as we cannot sell outside of a candle. + closerate = min(max(closerate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) # Confirm trade exit: time_in_force = self.strategy.order_time_in_force['sell'] @@ -424,13 +439,21 @@ class Backtesting: stake_amount = self.wallets.get_trade_stake_amount(pair, None) except DependencyException: return None + # let's call the custom entry price, using the open price as default price + propose_rate = strategy_safe_wrapper(self.strategy.custom_entry_price, + default_retval=row[OPEN_IDX])( + pair=pair, current_time=row[DATE_IDX].to_pydatetime(), + proposed_rate=row[OPEN_IDX]) # default value is the open rate - min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) or 0 + # Move rate to within the candle's low/high rate + propose_rate = min(max(propose_rate, row[LOW_IDX]), row[HIGH_IDX]) + + min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, propose_rate, -0.05) or 0 max_stake_amount = self.wallets.get_available_stake_amount() stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount, default_retval=stake_amount)( - pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX], + pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=propose_rate, proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount) stake_amount = self.wallets.validate_stake_amount(pair, stake_amount, min_stake_amount) @@ -441,7 +464,7 @@ class Backtesting: time_in_force = self.strategy.order_time_in_force['sell'] # Confirm trade entry: if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX], + pair=pair, order_type=order_type, amount=stake_amount, rate=propose_rate, time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()): return None @@ -450,10 +473,10 @@ class Backtesting: has_buy_tag = len(row) >= BUY_TAG_IDX + 1 trade = LocalTrade( pair=pair, - open_rate=row[OPEN_IDX], + open_rate=propose_rate, open_date=row[DATE_IDX].to_pydatetime(), stake_amount=stake_amount, - amount=round(stake_amount / row[OPEN_IDX], 8), + amount=round(stake_amount / propose_rate, 8), fee_open=self.fee, fee_close=self.fee, is_open=True, @@ -503,7 +526,8 @@ class Backtesting: Of course try to not have ugly code. By some accessor are sometime slower than functions. Avoid extensive logging in this method and functions it calls. - :param processed: a processed dictionary with format {pair, data} + :param processed: a processed dictionary with format {pair, data}, which gets cleared to + optimize memory usage! :param start_date: backtesting timerange start datetime :param end_date: backtesting timerange end datetime :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited diff --git a/freqtrade/optimize/bt_progress.py b/freqtrade/optimize/bt_progress.py index d295956c7..c3b105915 100644 --- a/freqtrade/optimize/bt_progress.py +++ b/freqtrade/optimize/bt_progress.py @@ -12,7 +12,7 @@ class BTProgress: def init_step(self, action: BacktestState, max_steps: float): self._action = action self._max_steps = max_steps - self._proress = 0 + self._progress = 0 def set_new_value(self, new_value: float): self._progress = new_value diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 2c7cc0ea7..58da7d0d5 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -422,6 +422,7 @@ class Hyperopt: self.backtesting.exchange.close() self.backtesting.exchange._api = None # type: ignore self.backtesting.exchange._api_async = None # type: ignore + self.backtesting.exchange.loop = None # type: ignore # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 6d44d56b1..c0888808f 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,7 +5,8 @@ from typing import Any, Dict, List import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (calculate_max_drawdown, combine_dataframes_with_mean, +from freqtrade.data.btanalysis import (analyze_trade_parallelism, calculate_max_drawdown, + calculate_underwater, combine_dataframes_with_mean, create_cum_profit, extract_trades_of_period, load_trades) from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider @@ -185,6 +186,48 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame, return fig +def add_underwater(fig, row, trades: pd.DataFrame) -> make_subplots: + """ + Add underwater plot + """ + try: + underwater = calculate_underwater(trades, value_col="profit_abs") + + underwater = go.Scatter( + x=underwater['date'], + y=underwater['drawdown'], + name="Underwater Plot", + fill='tozeroy', + fillcolor='#cc362b', + line={'color': '#cc362b'}, + ) + fig.add_trace(underwater, row, 1) + except ValueError: + logger.warning("No trades found - not plotting underwater plot") + return fig + + +def add_parallelism(fig, row, trades: pd.DataFrame, timeframe: str) -> make_subplots: + """ + Add Chart showing trade parallelism + """ + try: + result = analyze_trade_parallelism(trades, timeframe) + + drawdown = go.Scatter( + x=result.index, + y=result['open_trades'], + name="Parallel trades", + fill='tozeroy', + fillcolor='#242222', + line={'color': '#242222'}, + ) + fig.add_trace(drawdown, row, 1) + except ValueError: + logger.warning("No trades found - not plotting Parallelism.") + return fig + + def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: """ Add trades to "fig" @@ -460,7 +503,12 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame], trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure: # Combine close-values for all pairs, rename columns to "pair" - df_comb = combine_dataframes_with_mean(data, "close") + try: + df_comb = combine_dataframes_with_mean(data, "close") + except ValueError: + raise OperationalException( + "No data found. Please make sure that data is available for " + "the timerange and pairs selected.") # Trim trades to available OHLCV data trades = extract_trades_of_period(df_comb, trades, date_index=True) @@ -477,20 +525,30 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame], name='Avg close price', ) - fig = make_subplots(rows=3, cols=1, shared_xaxes=True, - row_width=[1, 1, 1], + fig = make_subplots(rows=5, cols=1, shared_xaxes=True, + row_heights=[1, 1, 1, 0.5, 1], vertical_spacing=0.05, - subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"]) + subplot_titles=[ + "AVG Close Price", + "Combined Profit", + "Profit per pair", + "Parallelism", + "Underwater", + ]) fig['layout'].update(title="Freqtrade Profit plot") fig['layout']['yaxis1'].update(title='Price') fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}') fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}') + fig['layout']['yaxis4'].update(title='Trade count') + fig['layout']['yaxis5'].update(title='Underwater Plot') fig['layout']['xaxis']['rangeslider'].update(visible=False) fig.update_layout(modebar_add=["v1hovermode", "toggleSpikeLines"]) fig.add_trace(avgclose, 1, 1) fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') fig = add_max_drawdown(fig, 2, trades, df_comb, timeframe) + fig = add_parallelism(fig, 4, trades, timeframe) + fig = add_underwater(fig, 5, trades) for pair in pairs: profit_col = f'cum_profit_{pair}' diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 671b6362b..d3196d3ae 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -68,14 +68,14 @@ class PerformanceFilter(IPairList): # - then pair name alphametically sorted_df = list_df.merge(performance, on='pair', how='left')\ .fillna(0).sort_values(by=['count', 'pair'], ascending=True)\ - .sort_values(by=['profit'], ascending=False) + .sort_values(by=['profit_ratio'], ascending=False) if self._min_profit is not None: - removed = sorted_df[sorted_df['profit'] < self._min_profit] + removed = sorted_df[sorted_df['profit_ratio'] < self._min_profit] for _, row in removed.iterrows(): self.log_once( - f"Removing pair {row['pair']} since {row['profit']} is " + f"Removing pair {row['pair']} since {row['profit_ratio']} is " f"below {self._min_profit}", logger.info) - sorted_df = sorted_df[sorted_df['profit'] >= self._min_profit] + sorted_df = sorted_df[sorted_df['profit_ratio'] >= self._min_profit] pairlist = sorted_df['pair'].tolist() diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 4d3dd29e3..663bba49b 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -5,6 +5,7 @@ import logging import random from typing import Any, Dict, List +from freqtrade.enums import RunMode from freqtrade.plugins.pairlist.IPairList import IPairList @@ -18,7 +19,15 @@ class ShuffleFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - self._seed = pairlistconfig.get('seed') + # Apply seed in backtesting mode to get comparable results, + # but not in live modes to get a non-repeating order of pairs during live modes. + if config.get('runmode') in (RunMode.LIVE, RunMode.DRY_RUN): + self._seed = None + logger.info("Live mode detected, not applying seed.") + else: + self._seed = pairlistconfig.get('seed') + logger.info(f"Backtesting mode detected, applying seed value: {self._seed}") + self._random = random.Random(self._seed) @property diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 9383e5d06..20b899c5f 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional import arrow import numpy as np -from cachetools.ttl import TTLCache +from cachetools import TTLCache from pandas import DataFrame from freqtrade.exceptions import OperationalException diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 0ffc8a8c8..83116ebac 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -8,7 +8,7 @@ from functools import partial from typing import Any, Dict, List import arrow -from cachetools.ttl import TTLCache +from cachetools import TTLCache from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 3e5a002ff..314056fbb 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -6,7 +6,7 @@ from copy import deepcopy from typing import Any, Dict, List, Optional import arrow -from cachetools.ttl import TTLCache +from cachetools import TTLCache from pandas import DataFrame from freqtrade.exceptions import OperationalException diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index face79729..1e79dc743 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -2,13 +2,14 @@ PairList manager class """ import logging -from copy import deepcopy +from functools import partial from typing import Dict, List from cachetools import TTLCache, cached from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException +from freqtrade.mixins import LoggingMixin from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import PairListResolver @@ -17,7 +18,7 @@ from freqtrade.resolvers import PairListResolver logger = logging.getLogger(__name__) -class PairListManager(): +class PairListManager(LoggingMixin): def __init__(self, exchange, config: dict) -> None: self._exchange = exchange @@ -41,6 +42,9 @@ class PairListManager(): if not self._pairlist_handlers: raise OperationalException("No Pairlist Handlers defined") + refresh_period = config.get('pairlist_refresh_period', 3600) + LoggingMixin.__init__(self, logger, refresh_period) + @property def whitelist(self) -> List[str]: """The current whitelist""" @@ -108,9 +112,10 @@ class PairListManager(): except ValueError as err: logger.error(f"Pair blacklist contains an invalid Wildcard: {err}") return [] - for pair in deepcopy(pairlist): + log_once = partial(self.log_once, logmethod=logmethod) + for pair in pairlist.copy(): if pair in blacklist: - logmethod(f"Pair {pair} in your blacklist. Removing it from whitelist...") + log_once(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py index edbc39772..d110134d7 100644 --- a/freqtrade/rpc/api_server/api_backtest.py +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -33,6 +33,9 @@ async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: Bac if settings[setting] is not None: btconfig[setting] = settings[setting] + # Force dry-run for backtesting + btconfig['dry_run'] = True + # Start backtesting # Initialize backtesting object def run_backtest(): diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index d0e772848..10f181bb6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,10 +1,10 @@ from datetime import date, datetime -from enum import Enum from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.enums import OrderTypeValues class Ping(BaseModel): @@ -126,20 +126,12 @@ class Daily(BaseModel): class UnfilledTimeout(BaseModel): - buy: int - sell: int - unit: str + buy: Optional[int] + sell: Optional[int] + unit: Optional[str] exit_timeout_count: Optional[int] -class OrderTypeValues(Enum): - limit = 'limit' - market = 'market' - - class Config: - use_enum_values = True - - class OrderTypes(BaseModel): buy: OrderTypeValues sell: OrderTypeValues @@ -153,6 +145,7 @@ class OrderTypes(BaseModel): class ShowConfig(BaseModel): version: str + strategy_version: Optional[str] api_version: float dry_run: bool stake_currency: str @@ -167,7 +160,7 @@ class ShowConfig(BaseModel): trailing_stop_positive_offset: Optional[float] trailing_only_offset_is_reached: Optional[bool] unfilledtimeout: UnfilledTimeout - order_types: OrderTypes + order_types: Optional[OrderTypes] use_custom_stoploss: Optional[bool] timeframe: Optional[str] timeframe_ms: int diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 65b6941e2..1c1ff39df 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -3,7 +3,7 @@ from copy import deepcopy from pathlib import Path from typing import List, Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException from freqtrade import __version__ @@ -30,7 +30,8 @@ logger = logging.getLogger(__name__) # Pre-1.1, no version was provided # Version increments should happen in "small" steps (1.1, 1.12, ...) unless big changes happen. # 1.11: forcebuy and forcesell accept ordertype -API_VERSION = 1.11 +# 1.12: add blacklist delete endpoint +API_VERSION = 1.12 # Public API, requires no auth. router_public = APIRouter() @@ -121,9 +122,11 @@ def edge(rpc: RPC = Depends(get_rpc)): @router.get('/show_config', response_model=ShowConfig, tags=['info']) def show_config(rpc: Optional[RPC] = Depends(get_rpc_optional), config=Depends(get_config)): state = '' + strategy_version = None if rpc: state = rpc._freqtrade.state - resp = RPC._rpc_show_config(config, state) + strategy_version = rpc._freqtrade.strategy.version() + resp = RPC._rpc_show_config(config, state, strategy_version) resp['api_version'] = API_VERSION return resp @@ -155,6 +158,13 @@ def blacklist_post(payload: BlacklistPayload, rpc: RPC = Depends(get_rpc)): return rpc._rpc_blacklist(payload.blacklist) +@router.delete('/blacklist', response_model=BlacklistResponse, tags=['info', 'pairlist']) +def blacklist_delete(pairs_to_delete: List[str] = Query([]), rpc: RPC = Depends(get_rpc)): + """Provide a list of pairs to delete from the blacklist""" + + return rpc._rpc_blacklist_delete(pairs_to_delete) + + @router.get('/whitelist', response_model=WhitelistResponse, tags=['info', 'pairlist']) def whitelist(rpc: RPC = Depends(get_rpc)): return rpc._rpc_whitelist() diff --git a/freqtrade/rpc/api_server/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py index 79af659c7..a79c1a5fc 100644 --- a/freqtrade/rpc/api_server/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server/uvicorn_threaded.py @@ -47,7 +47,7 @@ class UvicornServer(uvicorn.Server): else: asyncio.set_event_loop(uvloop.new_event_loop()) try: - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() except RuntimeError: # When running in a thread, we'll not have an eventloop yet. loop = asyncio.new_event_loop() diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index f4e82261e..ef9689d0a 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -7,7 +7,7 @@ import datetime import logging from typing import Dict, List -from cachetools.ttl import TTLCache +from cachetools import TTLCache from pycoingecko import CoinGeckoAPI from requests.exceptions import RequestException diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c21890b7d..3328af30b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -98,7 +98,8 @@ class RPC: self._fiat_converter = CryptoToFiatConverter() @staticmethod - def _rpc_show_config(config, botstate: Union[State, str]) -> Dict[str, Any]: + def _rpc_show_config(config, botstate: Union[State, str], + strategy_version: Optional[str] = None) -> Dict[str, Any]: """ Return a dict of config options. Explicitly does NOT return the full config to avoid leakage of sensitive @@ -106,6 +107,7 @@ class RPC: """ val = { 'version': __version__, + 'strategy_version': strategy_version, 'dry_run': config['dry_run'], 'stake_currency': config['stake_currency'], 'stake_currency_decimals': decimals_per_coin(config['stake_currency']), @@ -858,6 +860,20 @@ class RPC: } return res + def _rpc_blacklist_delete(self, delete: List[str]) -> Dict: + """ Removes pairs from currently active blacklist """ + errors = {} + for pair in delete: + if pair in self._freqtrade.pairlists.blacklist: + self._freqtrade.pairlists.blacklist.remove(pair) + else: + errors[pair] = { + 'error_msg': f"Pair {pair} is not in the current blacklist." + } + resp = self._rpc_blacklist() + resp['errors'] = errors + return resp + def _rpc_blacklist(self, add: List[str] = None) -> Dict: """ Returns the currently active blacklist""" errors = {} diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 8085ece94..9f62b9e23 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -60,6 +60,10 @@ class RPCManager: } """ logger.info('Sending rpc message: %s', msg) + if 'pair' in msg: + msg.update({ + 'base_currency': self._rpc._freqtrade.exchange.get_pair_base_currency(msg['pair']) + }) for mod in self.registered_modules: logger.debug('Forwarding message to rpc.%s', mod.name) try: diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e6624f94d..61a0b1f65 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -111,9 +111,9 @@ class Telegram(RPCHandler): r'/daily$', r'/daily \d+$', r'/profit$', r'/profit \d+', r'/stats$', r'/count$', r'/locks$', r'/balance$', r'/stopbuy$', r'/reload_config$', r'/show_config$', - r'/logs$', r'/whitelist$', r'/blacklist$', r'/edge$', + r'/logs$', r'/whitelist$', r'/blacklist$', r'/bl_delete$', r'/weekly$', r'/weekly \d+$', r'/monthly$', r'/monthly \d+$', - r'/forcebuy$', r'/help$', r'/version$'] + r'/forcebuy$', r'/edge$', r'/help$', r'/version$'] # Create keys for generation valid_keys_print = [k.replace('$', '') for k in valid_keys] @@ -170,6 +170,7 @@ class Telegram(RPCHandler): CommandHandler('stopbuy', self._stopbuy), CommandHandler('whitelist', self._whitelist), CommandHandler('blacklist', self._blacklist), + CommandHandler(['blacklist_delete', 'bl_delete'], self._blacklist_delete), CommandHandler('logs', self._logs), CommandHandler('edge', self._edge), CommandHandler('help', self._help), @@ -198,8 +199,8 @@ class Telegram(RPCHandler): self._updater.start_polling( bootstrap_retries=-1, - timeout=30, - read_latency=60, + timeout=20, + read_latency=60, # Assumed transmission latency drop_pending_updates=True, ) logger.info( @@ -212,6 +213,7 @@ class Telegram(RPCHandler): Stops all running telegram threads. :return: None """ + # This can take up to `timeout` from the call to `start_polling`. self._updater.stop() def _format_buy_msg(self, msg: Dict[str, Any]) -> str: @@ -1162,22 +1164,28 @@ class Telegram(RPCHandler): Handler for /blacklist Shows the currently active blacklist """ - try: + self.send_blacklist_msg(self._rpc._rpc_blacklist(context.args)) - blacklist = self._rpc._rpc_blacklist(context.args) - errmsgs = [] - for pair, error in blacklist['errors'].items(): - errmsgs.append(f"Error adding `{pair}` to blacklist: `{error['error_msg']}`") - if errmsgs: - self._send_msg('\n'.join(errmsgs)) + def send_blacklist_msg(self, blacklist: Dict): + errmsgs = [] + for pair, error in blacklist['errors'].items(): + errmsgs.append(f"Error adding `{pair}` to blacklist: `{error['error_msg']}`") + if errmsgs: + self._send_msg('\n'.join(errmsgs)) - message = f"Blacklist contains {blacklist['length']} pairs\n" - message += f"`{', '.join(blacklist['blacklist'])}`" + message = f"Blacklist contains {blacklist['length']} pairs\n" + message += f"`{', '.join(blacklist['blacklist'])}`" - logger.debug(message) - self._send_msg(message) - except RPCException as e: - self._send_msg(str(e)) + logger.debug(message) + self._send_msg(message) + + @authorized_only + def _blacklist_delete(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /bl_delete + Deletes pair(s) from current blacklist + """ + self.send_blacklist_msg(self._rpc._rpc_blacklist_delete(context.args or [])) @authorized_only def _logs(self, update: Update, context: CallbackContext) -> None: @@ -1258,6 +1266,8 @@ class Telegram(RPCHandler): "*/whitelist:* `Show current whitelist` \n" "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " "to the blacklist.` \n" + "*/blacklist_delete [pairs]| /bl_delete [pairs]:* " + "`Delete pair / pattern from blacklist. Will reset on reload_conf.` \n" "*/reload_config:* `Reload configuration file` \n" "*/unlock :* `Unlock this Pair (or this lock id if it's numeric)`\n" @@ -1305,7 +1315,12 @@ class Telegram(RPCHandler): :param update: message update :return: None """ - self._send_msg('*Version:* `{}`'.format(__version__)) + strategy_version = self._rpc._freqtrade.strategy.version() + version_string = f'*Version:* `{__version__}`' + if strategy_version is not None: + version_string += f', *Strategy version: * `{strategy_version}`' + + self._send_msg(version_string) @authorized_only def _show_config(self, update: Update, context: CallbackContext) -> None: diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index b4c55649e..58b75769e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -2,6 +2,7 @@ This module manages webhook communication """ import logging +import time from typing import Any, Dict from requests import RequestException, post @@ -28,12 +29,9 @@ class Webhook(RPCHandler): super().__init__(rpc, config) self._url = self._config['webhook']['url'] - self._format = self._config['webhook'].get('format', 'form') - - if self._format != 'form' and self._format != 'json': - raise NotImplementedError('Unknown webhook format `{}`, possible values are ' - '`form` (default) and `json`'.format(self._format)) + self._retries = self._config['webhook'].get('retries', 0) + self._retry_delay = self._config['webhook'].get('retry_delay', 0.1) def cleanup(self) -> None: """ @@ -77,13 +75,30 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - try: - if self._format == 'form': - post(self._url, data=payload) - elif self._format == 'json': - post(self._url, json=payload) - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) + success = False + attempts = 0 + while not success and attempts <= self._retries: + if attempts: + if self._retry_delay: + time.sleep(self._retry_delay) + logger.info("Retrying webhook...") - except RequestException as exc: - logger.warning("Could not call webhook url. Exception: %s", exc) + attempts += 1 + + try: + if self._format == 'form': + response = post(self._url, data=payload) + elif self._format == 'json': + response = post(self._url, json=payload) + elif self._format == 'raw': + response = post(self._url, data=payload['data'], + headers={'Content-Type': 'text/plain'}) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + + # Throw a RequestException if the post was not successful + response.raise_for_status() + success = True + + except RequestException as exc: + logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d4b496ed0..59cd48dd2 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -394,6 +394,12 @@ class IStrategy(ABC, HyperStrategyMixin): """ return [] + def version(self) -> Optional[str]: + """ + Returns version of the strategy. + """ + return None + ### # END - Intended to be overridden by strategy ### @@ -697,23 +703,21 @@ class IStrategy(ABC, HyperStrategyMixin): custom_reason = custom_reason[:CUSTOM_SELL_MAX_LENGTH] else: custom_reason = None - # TODO: return here if sell-signal should be favored over ROI + if sell_signal in (SellType.CUSTOM_SELL, SellType.SELL_SIGNAL): + logger.debug(f"{trade.pair} - Sell signal received. " + f"sell_type=SellType.{sell_signal.name}" + + (f", custom_reason={custom_reason}" if custom_reason else "")) + return SellCheckTuple(sell_type=sell_signal, sell_reason=custom_reason) # Start evaluations # Sequence: - # ROI (if not stoploss) # Sell-signal + # ROI (if not stoploss) # Stoploss if roi_reached and stoplossflag.sell_type != SellType.STOP_LOSS: logger.debug(f"{trade.pair} - Required profit reached. sell_type=SellType.ROI") return SellCheckTuple(sell_type=SellType.ROI) - if sell_signal != SellType.NONE: - logger.debug(f"{trade.pair} - Sell signal received. " - f"sell_type=SellType.{sell_signal.name}" + - (f", custom_reason={custom_reason}" if custom_reason else "")) - return SellCheckTuple(sell_type=sell_signal, sell_reason=custom_reason) - if stoplossflag.sell_flag: logger.debug(f"{trade.pair} - Stoploss hit. sell_type={stoplossflag.sell_type}") diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index d10847099..98a39ea2d 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -260,8 +260,8 @@ class Wallets: if self._log: logger.info( f"Adjusted stake amount for pair {pair} is more than 30% bigger than " - f"the desired stake ({stake_amount} * 1.3 > {max_stake_amount}), " - f"ignoring trade." + f"the desired stake amount of ({stake_amount:.8f} * 1.3 = " + f"{stake_amount * 1.3:.8f}) < {min_stake_amount}), ignoring trade." ) return 0 stake_amount = min_stake_amount diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 5c0de86ff..66f718af0 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -85,9 +85,12 @@ class Worker: # Log state transition if state != old_state: - self.freqtrade.notify_status(f'{state.name.lower()}') - logger.info(f"Changing state to: {state.name}") + if old_state != State.RELOAD_CONFIG: + self.freqtrade.notify_status(f'{state.name.lower()}') + + logger.info( + f"Changing state{f' from {old_state.name}' if old_state else ''} to: {state.name}") if state == State.RUNNING: self.freqtrade.startup() @@ -113,8 +116,12 @@ class Worker: if self._heartbeat_interval: now = time.time() if (now - self._heartbeat_msg) > self._heartbeat_interval: + version = __version__ + strategy_version = self.freqtrade.strategy.version() + if (strategy_version is not None): + version += ', strategy_version: ' + strategy_version logger.info(f"Bot heartbeat. PID={getpid()}, " - f"version='{__version__}', state='{state.name}'") + f"version='{version}', state='{state.name}'") self._heartbeat_msg = now return state diff --git a/mkdocs.yml b/mkdocs.yml index b9c053324..fb1b80ebf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,8 +81,10 @@ markdown_extensions: - pymdownx.snippets: base_path: docs check_paths: true - - pymdownx.tabbed - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true - pymdownx.tasklist: custom_checkbox: true + - pymdownx.tilde - mdx_truly_sane_lists diff --git a/pyproject.toml b/pyproject.toml index f0637d8c6..ad32bad4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ exclude = ''' line_length = 100 multi_line_output=0 lines_after_imports=2 +skip_glob = ["**/.env*", "**/env/*", "**/.venv/*", "**/docs/*"] [build-system] requires = ["setuptools >= 46.4.0", "wheel"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 4c06e657b..b2fad4e03 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ coveralls==3.3.1 flake8==4.0.1 flake8-tidy-imports==4.5.0 -mypy==0.910 +mypy==0.930 pytest==6.2.5 pytest-asyncio==0.16.0 pytest-cov==3.0.0 @@ -14,16 +14,16 @@ pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.10.1 # For datetime mocking -time-machine==2.4.0 +time-machine==2.5.0 # Convert jupyter notebooks to markdown documents nbconvert==6.3.0 # mypy types -types-cachetools==4.2.5 +types-cachetools==4.2.7 types-filelock==3.2.1 -types-requests==2.26.0 -types-tabulate==0.8.3 +types-requests==2.26.3 +types-tabulate==0.8.4 # Extensions to datetime library -types-python-dateutil==2.8.2 \ No newline at end of file +types-python-dateutil==2.8.4 \ No newline at end of file diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index a3da8f0be..b785e73e9 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,10 +2,9 @@ -r requirements.txt # Required for hyperopt -scipy==1.7.2 -scikit-learn==1.0.1 +scipy==1.7.3 +scikit-learn==1.0.2 scikit-optimize==0.9.0 -filelock==3.4.0 +filelock==3.4.2 joblib==1.1.0 -psutil==5.8.0 progressbar2==3.55.0 diff --git a/requirements-plot.txt b/requirements-plot.txt index 488ef73d6..990edc3c8 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.4.0 +plotly==5.5.0 diff --git a/requirements.txt b/requirements.txt index a5af330af..bdc0da750 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,25 @@ -numpy==1.21.4 -pandas==1.3.4 +numpy==1.21.5; python_version <= '3.7' +numpy==1.22.0; python_version > '3.7' +pandas==1.3.5 pandas-ta==0.3.14b -ccxt==1.61.92 +ccxt==1.66.20 # Pin cryptography for now due to rust build errors with piwheels -cryptography==36.0.0 +cryptography==36.0.1 aiohttp==3.8.1 -SQLAlchemy==1.4.27 -python-telegram-bot==13.8.1 +SQLAlchemy==1.4.29 +python-telegram-bot==13.9 arrow==1.2.1 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 -jsonschema==4.2.1 -TA-Lib==0.4.21 +jsonschema==4.3.3 +TA-Lib==0.4.23 technical==1.3.0 tabulate==0.8.9 pycoingecko==2.2.0 jinja2==3.0.3 -tables==3.6.1 +tables==3.7.0 blosc==1.10.6 # find first, C search in arrays @@ -31,16 +32,16 @@ python-rapidjson==1.5 sdnotify==0.3.2 # API Server -fastapi==0.70.0 -uvicorn==0.15.0 +fastapi==0.70.1 +uvicorn==0.16.0 pyjwt==2.3.0 -aiofiles==0.7.0 -psutil==5.8.0 +aiofiles==0.8.0 +psutil==5.9.0 # Support for colorized terminal output colorama==0.4.4 # Building config files interactively questionary==1.10.0 -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.24 # Extensions to datetime library python-dateutil==2.8.2 diff --git a/setup.cfg b/setup.cfg index b311c94da..c5c7f2f25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Operating System :: MacOS Operating System :: Unix Topic :: Office/Business :: Financial :: Investment diff --git a/setup.sh b/setup.sh index 16ccde0df..c642a654d 100755 --- a/setup.sh +++ b/setup.sh @@ -25,7 +25,7 @@ function check_installed_python() { exit 2 fi - for v in 9 8 7 + for v in 9 10 8 7 do PYTHON="python3.${v}" which $PYTHON @@ -36,7 +36,7 @@ function check_installed_python() { fi done - echo "No usable python found. Please make sure to have python3.7 or newer installed" + echo "No usable python found. Please make sure to have python3.7 or newer installed." exit 1 } @@ -205,7 +205,7 @@ function config() { } function install() { - + echo_block "Installing mandatory dependencies" if [ "$(uname -s)" == "Darwin" ]; then @@ -219,7 +219,7 @@ function install() { install_redhat else echo "This script does not support your OS." - echo "If you have Python version 3.7 - 3.9, pip, virtualenv, ta-lib you can continue." + echo "If you have Python version 3.7 - 3.10, pip, virtualenv, ta-lib you can continue." echo "Wait 10 seconds to continue the next install steps or use ctrl+c to interrupt this shell." sleep 10 fi diff --git a/tests/conftest.py b/tests/conftest.py index 3ce064ee3..9cd1332d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ import logging import re from copy import deepcopy from datetime import datetime, timedelta -from functools import reduce from pathlib import Path from unittest.mock import MagicMock, Mock, PropertyMock @@ -50,17 +49,23 @@ def pytest_configure(config): def log_has(line, logs): - # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar') - # and we want to match line against foobar in the tuple - return reduce(lambda a, b: a or b, - filter(lambda x: x[2] == line, logs.record_tuples), - False) + """Check if line is found on some caplog's message.""" + return any(line == message for message in logs.messages) def log_has_re(line, logs): - return reduce(lambda a, b: a or b, - filter(lambda x: re.match(line, x[2]), logs.record_tuples), - False) + """Check if line matches some caplog's message.""" + return any(re.match(line, message) for message in logs.messages) + + +def num_log_has(line, logs): + """Check how many times line is found in caplog's messages.""" + return sum(line == message for message in logs.messages) + + +def num_log_has_re(line, logs): + """Check how many times line matches caplog's messages.""" + return sum(bool(re.match(line, message)) for message in logs.messages) def get_args(args): diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 1dcd04a80..47f1b8849 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -11,10 +11,10 @@ from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_MID, BT_DATA_COLUMNS_OLD, analyze_trade_parallelism, calculate_csum, calculate_market_change, calculate_max_drawdown, - combine_dataframes_with_mean, create_cum_profit, - extract_trades_of_period, get_latest_backtest_filename, - get_latest_hyperopt_file, load_backtest_data, load_trades, - load_trades_from_db) + calculate_underwater, combine_dataframes_with_mean, + create_cum_profit, extract_trades_of_period, + get_latest_backtest_filename, get_latest_hyperopt_file, + load_backtest_data, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from tests.conftest import create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT @@ -234,6 +234,13 @@ def test_combine_dataframes_with_mean(testdatadir): assert "mean" in df.columns +def test_combine_dataframes_with_mean_no_data(testdatadir): + pairs = ["ETH/BTC", "ADA/BTC"] + data = load_data(datadir=testdatadir, pairs=pairs, timeframe='6m') + with pytest.raises(ValueError, match=r"No objects to concatenate"): + combine_dataframes_with_mean(data) + + def test_create_cum_profit(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) @@ -284,9 +291,16 @@ def test_calculate_max_drawdown(testdatadir): assert isinstance(lval, float) assert hdate == Timestamp('2018-01-24 14:25:00', tz='UTC') assert lowdate == Timestamp('2018-01-30 04:45:00', tz='UTC') + + underwater = calculate_underwater(bt_data) + assert isinstance(underwater, DataFrame) + with pytest.raises(ValueError, match='Trade dataframe empty.'): drawdown, hdate, lowdate, hval, lval = calculate_max_drawdown(DataFrame()) + with pytest.raises(ValueError, match='Trade dataframe empty.'): + calculate_underwater(DataFrame()) + def test_calculate_csum(testdatadir): filename = testdatadir / "backtest-result_test.json" diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 575a590e7..627e29444 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -311,7 +311,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: assert td != len(data['UNITTEST/BTC']) start_real = data['UNITTEST/BTC'].iloc[0, 0] assert log_has(f'Missing data at start for pair ' - f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', + f'UNITTEST/BTC at 5m, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog) # Make sure we start fresh - test missing data at end caplog.clear() @@ -326,7 +326,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: # Shift endtime with +5 - as last candle is dropped (partial candle) end_real = arrow.get(data['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) assert log_has(f'Missing data at end for pair ' - f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', + f'UNITTEST/BTC at 5m, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog) diff --git a/tests/exchange/test_bitpanda.py b/tests/exchange/test_bitpanda.py new file mode 100644 index 000000000..4bd168e7e --- /dev/null +++ b/tests/exchange/test_bitpanda.py @@ -0,0 +1,47 @@ +from datetime import datetime +from unittest.mock import MagicMock + +from tests.conftest import get_patched_exchange + + +def test_get_trades_for_order(default_conf, mocker): + exchange_name = 'bitpanda' + order_id = 'ABCD-ABCD' + since = datetime(2018, 5, 5, 0, 0, 0) + default_conf["dry_run"] = False + mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) + api_mock = MagicMock() + + api_mock.fetch_my_trades = MagicMock(return_value=[{'id': 'TTR67E-3PFBD-76IISV', + 'order': 'ABCD-ABCD', + 'info': {'pair': 'XLTCZBTC', + 'time': 1519860024.4388, + 'type': 'buy', + 'ordertype': 'limit', + 'price': '20.00000', + 'cost': '38.62000', + 'fee': '0.06179', + 'vol': '5', + 'id': 'ABCD-ABCD'}, + 'timestamp': 1519860024438, + 'datetime': '2018-02-28T23:20:24.438Z', + 'symbol': 'LTC/BTC', + 'type': 'limit', + 'side': 'buy', + 'price': 165.0, + 'amount': 0.2340606, + 'fee': {'cost': 0.06179, 'currency': 'BTC'} + }]) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + + orders = exchange.get_trades_for_order(order_id, 'LTC/BTC', since) + assert len(orders) == 1 + assert orders[0]['price'] == 165 + assert api_mock.fetch_my_trades.call_count == 1 + # since argument should be + assert isinstance(api_mock.fetch_my_trades.call_args[0][1], int) + assert api_mock.fetch_my_trades.call_args[0][0] == 'LTC/BTC' + # Same test twice, hardcoded number and doing the same calculation + assert api_mock.fetch_my_trades.call_args[0][1] == 1525478395000 + # bitpanda requires "to" argument. + assert 'to' in api_mock.fetch_my_trades.call_args[1]['params'] diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b642b3fa2..071f4e2b8 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -20,7 +20,7 @@ from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver -from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re +from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re, num_log_has_re # Make sure to always keep one exchange here which is NOT subclassed!! @@ -1026,6 +1026,12 @@ def test_create_dry_run_order_limit_fill(default_conf, mocker, side, startprice, assert order_closed['status'] == 'closed' assert order['fee'] + # Empty orderbook test + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'asks': [], 'bids': []}) + exchange._dry_run_open_orders[order['id']]['status'] = 'open' + order_closed = exchange.fetch_dry_run_order(order['id']) + @pytest.mark.parametrize("side,rate,amount,endprice", [ # spread is 25.263-25.266 @@ -1734,6 +1740,44 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ (arrow.utcnow().int_timestamp - 2000) * 1000) +@pytest.mark.asyncio +async def test__async_kucoin_get_candle_history(default_conf, mocker, caplog): + caplog.set_level(logging.INFO) + api_mock = MagicMock() + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.DDoSProtection( + "kucoin GET https://openapi-v2.kucoin.com/api/v1/market/candles?" + "symbol=ETH-BTC&type=5min&startAt=1640268735&endAt=1640418735" + "429 Too Many Requests" '{"code":"429000","msg":"Too Many Requests"}')) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kucoin") + + msg = "Kucoin 429 error, avoid triggering DDosProtection backoff delay" + assert not num_log_has_re(msg, caplog) + + for _ in range(3): + with pytest.raises(DDosProtection, match=r'429 Too Many Requests'): + await exchange._async_get_candle_history( + "ETH/BTC", "5m", (arrow.utcnow().int_timestamp - 2000) * 1000, count=3) + assert num_log_has_re(msg, caplog) == 3 + + caplog.clear() + # Test regular non-kucoin message + api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.DDoSProtection( + "kucoin GET https://openapi-v2.kucoin.com/api/v1/market/candles?" + "symbol=ETH-BTC&type=5min&startAt=1640268735&endAt=1640418735" + "429 Too Many Requests" '{"code":"2222222","msg":"Too Many Requests"}')) + + msg = r'_async_get_candle_history\(\) returned exception: .*' + msg2 = r'Applying DDosProtection backoff delay: .*' + with patch('freqtrade.exchange.common.asyncio.sleep', get_mock_coro(None)): + for _ in range(3): + with pytest.raises(DDosProtection, match=r'429 Too Many Requests'): + await exchange._async_get_candle_history( + "ETH/BTC", "5m", (arrow.utcnow().int_timestamp - 2000) * 1000, count=3) + # Expect the "returned exception" message 12 times (4 retries * 3 (loop)) + assert num_log_has_re(msg, caplog) == 12 + assert num_log_has_re(msg2, caplog) == 9 + + @pytest.mark.asyncio async def test__async_get_candle_history_empty(default_conf, mocker, caplog): """ Test empty exchange result """ @@ -1777,7 +1821,7 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): assert len(res) == 1 # Test that each is in list at least once as order is not guaranteed assert log_has("Error loading ETH/BTC. Result was [[]].", caplog) - assert log_has("Async code raised an exception: TypeError", caplog) + assert log_has("Async code raised an exception: TypeError()", caplog) def test_get_next_limit_in_list(): @@ -2942,39 +2986,49 @@ def test_extract_cost_curr_rate(mocker, default_conf, order, expected) -> None: assert ex.extract_cost_curr_rate(order) == expected -@pytest.mark.parametrize("order,expected", [ +@pytest.mark.parametrize("order,unknown_fee_rate,expected", [ # Using base-currency ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, - 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.1), + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, None, 0.1), ({'symbol': 'ETH/BTC', 'amount': 0.05, 'cost': 0.05, - 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.08), + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, None, 0.08), # Using quote currency ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, - 'fee': {'currency': 'BTC', 'cost': 0.005}}, 0.1), + 'fee': {'currency': 'BTC', 'cost': 0.005}}, None, 0.1), ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, - 'fee': {'currency': 'BTC', 'cost': 0.002, 'rate': None}}, 0.04), + 'fee': {'currency': 'BTC', 'cost': 0.002, 'rate': None}}, None, 0.04), # Using foreign currency ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, - 'fee': {'currency': 'NEO', 'cost': 0.0012}}, 0.001944), + 'fee': {'currency': 'NEO', 'cost': 0.0012}}, None, 0.001944), ({'symbol': 'ETH/BTC', 'amount': 2.21, 'cost': 0.02992561, - 'fee': {'currency': 'NEO', 'cost': 0.00027452}}, 0.00074305), + 'fee': {'currency': 'NEO', 'cost': 0.00027452}}, None, 0.00074305), # Rate included in return - return as is ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, - 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, 0.01), + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, None, 0.01), ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, - 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.005}}, 0.005), + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.005}}, None, 0.005), # 0.1% filled - no costs (kraken - #3431) ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.0, - 'fee': {'currency': 'BTC', 'cost': 0.0, 'rate': None}}, None), + 'fee': {'currency': 'BTC', 'cost': 0.0, 'rate': None}}, None, None), ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.0, - 'fee': {'currency': 'ETH', 'cost': 0.0, 'rate': None}}, 0.0), + 'fee': {'currency': 'ETH', 'cost': 0.0, 'rate': None}}, None, 0.0), ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.0, - 'fee': {'currency': 'NEO', 'cost': 0.0, 'rate': None}}, None), + 'fee': {'currency': 'NEO', 'cost': 0.0, 'rate': None}}, None, None), + # Invalid pair combination - POINT/BTC is not a pair + ({'symbol': 'POINT/BTC', 'amount': 0.04, 'cost': 0.5, + 'fee': {'currency': 'POINT', 'cost': 2.0, 'rate': None}}, None, None), + ({'symbol': 'POINT/BTC', 'amount': 0.04, 'cost': 0.5, + 'fee': {'currency': 'POINT', 'cost': 2.0, 'rate': None}}, 1, 4.0), + ({'symbol': 'POINT/BTC', 'amount': 0.04, 'cost': 0.5, + 'fee': {'currency': 'POINT', 'cost': 2.0, 'rate': None}}, 2, 8.0), ]) -def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None: +def test_calculate_fee_rate(mocker, default_conf, order, expected, unknown_fee_rate) -> None: mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'last': 0.081}) + if unknown_fee_rate: + default_conf['exchange']['unknown_fee_rate'] = unknown_fee_rate ex = get_patched_exchange(mocker, default_conf) + assert ex.calculate_fee_rate(order) == expected diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 775f15b87..f41b6101c 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -426,8 +426,6 @@ tc26 = BTContainer(data=[ # Test 27: Sell with signal sell in candle 3 (ROI at signal candle) # Stoploss at 10% (irrelevant), ROI at 5% (will trigger) - Wins over Sell-signal -# TODO: figure out if sell-signal should win over ROI -# Sell-signal wins over stoploss tc27 = BTContainer(data=[ # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], @@ -436,8 +434,8 @@ tc27 = BTContainer(data=[ [3, 5010, 5012, 4986, 5010, 6172, 0, 1], # sell-signal [4, 5010, 5251, 4855, 4995, 6172, 0, 0], # Triggers ROI, sell-signal acted on [5, 4995, 4995, 4950, 4950, 6172, 0, 0]], - stop_loss=-0.10, roi={"0": 0.05}, profit_perc=0.05, use_sell_signal=True, - trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=4)] + stop_loss=-0.10, roi={"0": 0.05}, profit_perc=0.002, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=4)] ) # Test 28: trailing_stop should raise so candle 3 causes a stoploss diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index f5e182c1d..6290c3c55 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument import random +from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import MagicMock, PropertyMock @@ -648,7 +649,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) result = backtesting.backtest( - processed=processed, + processed=deepcopy(processed), start_date=min_date, end_date=max_date, max_open_trades=10, @@ -887,7 +888,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) processed = backtesting.strategy.advise_all_indicators(data) min_date, max_date = get_timerange(processed) backtest_conf = { - 'processed': processed, + 'processed': deepcopy(processed), 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 3, @@ -909,7 +910,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count backtest_conf = { - 'processed': processed, + 'processed': deepcopy(processed), 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 1, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index a43c62376..a5a131e45 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -169,6 +169,7 @@ def test_start_no_hyperopt_allowed(mocker, hyperopt_conf, caplog) -> None: def test_start_no_data(mocker, hyperopt_conf) -> None: + hyperopt_conf['user_data_dir'] = Path("tests") patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame)) mocker.patch( @@ -189,6 +190,12 @@ def test_start_no_data(mocker, hyperopt_conf) -> None: with pytest.raises(OperationalException, match='No data found. Terminating.'): start_hyperopt(pargs) + # Cleanup since that failed hyperopt start leaves a lockfile. + try: + Path(Hyperopt.get_lock_filename(hyperopt_conf)).unlink() + except Exception: + pass + def test_start_filelock(mocker, hyperopt_conf, caplog) -> None: hyperopt_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(hyperopt_conf))) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 6333266aa..f7ff495ac 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1,5 +1,6 @@ # pragma pylint: disable=missing-docstring,C0103,protected-access +import logging import time from unittest.mock import MagicMock, PropertyMock @@ -7,13 +8,14 @@ import pytest import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from tests.conftest import (create_mock_trades, get_patched_exchange, get_patched_freqtradebot, - log_has, log_has_re) + log_has, log_has_re, num_log_has) @pytest.fixture(scope="function") @@ -216,6 +218,34 @@ def test_invalid_blacklist(mocker, markets, static_pl_conf, caplog): log_has_re(r"Pair blacklist contains an invalid Wildcard.*", caplog) +def test_remove_logs_for_pairs_already_in_blacklist(mocker, markets, static_pl_conf, caplog): + logger = logging.getLogger(__name__) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + markets=PropertyMock(return_value=markets), + ) + freqtrade.pairlists.refresh_pairlist() + whitelist = ['ETH/BTC', 'TKN/BTC'] + caplog.clear() + caplog.set_level(logging.INFO) + + # Ensure all except those in whitelist are removed. + assert set(whitelist) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist + # Ensure that log message wasn't generated. + assert not log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...', caplog) + + for _ in range(3): + new_whitelist = freqtrade.pairlists.verify_blacklist( + whitelist + ['BLK/BTC'], logger.warning) + # Ensure that the pair is removed from the white list, and properly logged. + assert set(whitelist) == set(new_whitelist) + assert num_log_has('Pair BLK/BTC in your blacklist. Removing it from whitelist...', + caplog) == 1 + + def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): mocker.patch.multiple( @@ -657,6 +687,22 @@ def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: assert log_has("PerformanceFilter is not available in this mode.", caplog) +def test_ShuffleFilter_init(mocker, whitelist_conf, caplog) -> None: + whitelist_conf['pairlists'] = [ + {"method": "StaticPairList"}, + {"method": "ShuffleFilter", "seed": 42} + ] + + exchange = get_patched_exchange(mocker, whitelist_conf) + PairListManager(exchange, whitelist_conf) + assert log_has("Backtesting mode detected, applying seed value: 42", caplog) + caplog.clear() + whitelist_conf['runmode'] = RunMode.DRY_RUN + PairListManager(exchange, whitelist_conf) + assert not log_has("Backtesting mode detected, applying seed value: 42", caplog) + assert log_has("Live mode detected, not applying seed.", caplog) + + @pytest.mark.usefixtures("init_persistence") def test_PerformanceFilter_lookback(mocker, whitelist_conf, fee, caplog) -> None: whitelist_conf['exchange']['pair_whitelist'].append('XRP/BTC') @@ -1089,33 +1135,34 @@ def test_pairlistmanager_no_pairlist(mocker, whitelist_conf): # Happy path: Descending order, all values filled ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], ['ETH/BTC', 'TKN/BTC'], - [{'pair': 'TKN/BTC', 'profit': 5, 'count': 3}, {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + [{'pair': 'TKN/BTC', 'profit_ratio': 0.05, 'count': 3}, + {'pair': 'ETH/BTC', 'profit_ratio': 0.04, 'count': 2}], ['TKN/BTC', 'ETH/BTC']), # Performance data outside allow list ignored ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], ['ETH/BTC', 'TKN/BTC'], - [{'pair': 'OTHER/BTC', 'profit': 5, 'count': 3}, - {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + [{'pair': 'OTHER/BTC', 'profit_ratio': 0.05, 'count': 3}, + {'pair': 'ETH/BTC', 'profit_ratio': 0.04, 'count': 2}], ['ETH/BTC', 'TKN/BTC']), # Partial performance data missing and sorted between positive and negative profit ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], - [{'pair': 'ETH/BTC', 'profit': -5, 'count': 100}, - {'pair': 'TKN/BTC', 'profit': 4, 'count': 2}], + [{'pair': 'ETH/BTC', 'profit_ratio': -0.05, 'count': 100}, + {'pair': 'TKN/BTC', 'profit_ratio': 0.04, 'count': 2}], ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']), # Tie in performance data broken by count (ascending) ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], - [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 101}, - {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 2}, - {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 100}], + [{'pair': 'LTC/BTC', 'profit_ratio': -0.0501, 'count': 101}, + {'pair': 'TKN/BTC', 'profit_ratio': -0.0501, 'count': 2}, + {'pair': 'ETH/BTC', 'profit_ratio': -0.0501, 'count': 100}], ['TKN/BTC', 'ETH/BTC', 'LTC/BTC']), # Tie in performance and count, broken by alphabetical sort ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], - [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 1}, - {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 1}, - {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 1}], + [{'pair': 'LTC/BTC', 'profit_ratio': -0.0501, 'count': 1}, + {'pair': 'TKN/BTC', 'profit_ratio': -0.0501, 'count': 1}, + {'pair': 'ETH/BTC', 'profit_ratio': -0.0501, 'count': 1}], ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']), ]) def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, overall_performance, diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index b6fe1c691..e86022a91 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -424,7 +424,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] in ('0:00:00', '0:00:01') + assert stats['avg_duration'] in ('0:00:00', '0:00:01', '0:00:02') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) @@ -435,7 +435,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] in ('0:00:00', '0:00:01') + assert stats['avg_duration'] in ('0:00:00', '0:00:01', '0:00:02') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) assert isnan(stats['profit_all_coin']) @@ -1225,6 +1225,16 @@ def test_rpc_blacklist(mocker, default_conf) -> None: assert 'errors' in ret assert isinstance(ret['errors'], dict) + ret = rpc._rpc_blacklist_delete(["DOGE/BTC", 'HOT/BTC']) + + assert 'StaticPairList' in ret['method'] + assert len(ret['blacklist']) == 2 + assert ret['blacklist'] == default_conf['exchange']['pair_blacklist'] + assert ret['blacklist'] == ['ETH/BTC', 'XRP/.*'] + assert ret['blacklist_expanded'] == ['ETH/BTC', 'XRP/BTC', 'XRP/USDT'] + assert 'errors' in ret + assert isinstance(ret['errors'], dict) + def test_rpc_edge_disabled(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 76372df55..d6a58322a 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -533,6 +533,7 @@ def test_api_show_config(botclient): assert rc.json()['timeframe_min'] == 5 assert rc.json()['state'] == 'running' assert rc.json()['bot_name'] == 'freqtrade' + assert rc.json()['strategy_version'] is None assert not rc.json()['trailing_stop'] assert 'bid_strategy' in rc.json() assert 'ask_strategy' in rc.json() @@ -954,6 +955,38 @@ def test_api_blacklist(botclient, mocker): "errors": {}, } + rc = client_delete(client, f"{BASE_URI}/blacklist?pairs_to_delete=DOGE/BTC") + assert_response(rc) + assert rc.json() == {"blacklist": ["HOT/BTC", "ETH/BTC", "XRP/.*"], + "blacklist_expanded": ["ETH/BTC", "XRP/BTC", "XRP/USDT"], + "length": 3, + "method": ["StaticPairList"], + "errors": {}, + } + + rc = client_delete(client, f"{BASE_URI}/blacklist?pairs_to_delete=NOTHING/BTC") + assert_response(rc) + assert rc.json() == {"blacklist": ["HOT/BTC", "ETH/BTC", "XRP/.*"], + "blacklist_expanded": ["ETH/BTC", "XRP/BTC", "XRP/USDT"], + "length": 3, + "method": ["StaticPairList"], + "errors": { + "NOTHING/BTC": { + "error_msg": "Pair NOTHING/BTC is not in the current blacklist." + } + }, + } + rc = client_delete( + client, + f"{BASE_URI}/blacklist?pairs_to_delete=HOT/BTC&pairs_to_delete=ETH/BTC") + assert_response(rc) + assert rc.json() == {"blacklist": ["XRP/.*"], + "blacklist_expanded": ["XRP/BTC", "XRP/USDT"], + "length": 1, + "method": ["StaticPairList"], + "errors": {}, + } + def test_api_whitelist(botclient): ftbot, client = botclient diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 6c32e59fc..7f7629e1d 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -98,7 +98,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['stats'], ['daily'], ['weekly'], ['monthly'], " "['count'], ['locks'], ['unlock', 'delete_locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], " - "['stopbuy'], ['whitelist'], ['blacklist'], " + "['stopbuy'], ['whitelist'], ['blacklist'], ['blacklist_delete', 'bl_delete'], " "['logs'], ['edge'], ['help'], ['version']" "]") @@ -584,7 +584,7 @@ def test_monthly_handle(default_conf, update, ticker, limit_buy_order, fee, assert 'Monthly Profit over the last 2 months:' in msg_mock.call_args_list[0][0][0] assert 'Month ' in msg_mock.call_args_list[0][0][0] today = datetime.utcnow().date() - current_month = f"{today.year}-{today.month} " + current_month = f"{today.year}-{today.month:02} " assert current_month in msg_mock.call_args_list[0][0][0] assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] @@ -937,7 +937,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 4 - last_msg = msg_mock.call_args_list[-1][0][0] + last_msg = msg_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -952,6 +952,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'profit_amount': 6.314e-05, 'profit_ratio': 0.0629778, 'stake_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': 'USD', 'buy_tag': ANY, 'sell_reason': SellType.FORCE_SELL.value, @@ -1001,7 +1002,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, assert msg_mock.call_count == 4 - last_msg = msg_mock.call_args_list[-1][0][0] + last_msg = msg_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -1016,6 +1017,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'profit_amount': -5.497e-05, 'profit_ratio': -0.05482878, 'stake_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': 'USD', 'buy_tag': ANY, 'sell_reason': SellType.FORCE_SELL.value, @@ -1055,7 +1057,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None # Called for each trade 2 times assert msg_mock.call_count == 8 - msg = msg_mock.call_args_list[1][0][0] + msg = msg_mock.call_args_list[0][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -1070,6 +1072,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'profit_amount': -4.09e-06, 'profit_ratio': -0.00408133, 'stake_currency': 'BTC', + 'base_currency': 'ETH', 'fiat_currency': 'USD', 'buy_tag': ANY, 'sell_reason': SellType.FORCE_SELL.value, @@ -1470,6 +1473,13 @@ def test_blacklist_static(default_conf, update, mocker) -> None: in msg_mock.call_args_list[0][0][0]) assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC", "XRP/.*"] + msg_mock.reset_mock() + context.args = ["DOGE/BTC"] + telegram._blacklist_delete(update=update, context=context) + assert msg_mock.call_count == 1 + assert ("Blacklist contains 3 pairs\n`HOT/BTC, ETH/BTC, XRP/.*`" + in msg_mock.call_args_list[0][0][0]) + def test_telegram_logs(default_conf, update, mocker) -> None: mocker.patch.multiple( @@ -1597,12 +1607,20 @@ def test_help_handle(default_conf, update, mocker) -> None: def test_version_handle(default_conf, update, mocker) -> None: - telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._version(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + freqtradebot.strategy.version = lambda: '1.1.1' + + telegram._version(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + assert '*Version:* `{}`'.format(__version__) in msg_mock.call_args_list[0][0][0] + assert '*Strategy version: * `1.1.1`' in msg_mock.call_args_list[0][0][0] + def test_show_config_handle(default_conf, update, mocker) -> None: diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 04e63a3be..17d1baca9 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -292,3 +292,15 @@ def test__send_msg_with_json_format(default_conf, mocker, caplog): webhook._send_msg(msg) assert post.call_args[1] == {'json': msg} + + +def test__send_msg_with_raw_format(default_conf, mocker, caplog): + default_conf["webhook"] = get_webhook_dict() + default_conf["webhook"]["format"] = "raw" + webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + msg = {'data': 'Hello'} + post = MagicMock() + mocker.patch("freqtrade.rpc.webhook.post", post) + webhook._send_msg(msg) + + assert post.call_args[1] == {'data': msg['data'], 'headers': {'Content-Type': 'text/plain'}} diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index e5dae5461..7306f5cab 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1904,7 +1904,7 @@ def test_handle_trade_roi(default_conf_usdt, ticker_usdt, limit_buy_order_usdt_o # we might just want to check if we are in a sell condition without # executing # if ROI is reached we must sell - patch_get_signal(freqtrade, value=(False, True, None, None)) + patch_get_signal(freqtrade, value=(False, False, None, None)) assert freqtrade.handle_trade(trade) assert log_has("ETH/USDT - Required profit reached. sell_type=SellType.ROI", caplog) @@ -2171,10 +2171,20 @@ def test_check_handle_timedout_sell_usercustom(default_conf_usdt, ticker_usdt, l assert open_trade.is_open is True assert freqtrade.strategy.check_sell_timeout.call_count == 1 - # 2nd canceled trade ... + # 2nd canceled trade - Fail execute sell caplog.clear() open_trade.open_order_id = 'order_id_2' mocker.patch('freqtrade.persistence.Trade.get_exit_order_count', return_value=1) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit', + side_effect=DependencyException) + freqtrade.check_handle_timedout() + assert log_has_re('Unable to emergency sell .*', caplog) + + et_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.execute_trade_exit') + caplog.clear() + + # 2nd canceled trade ... + open_trade.open_order_id = 'order_id_2' freqtrade.check_handle_timedout() assert log_has_re('Emergencyselling trade.*', caplog) assert et_mock.call_count == 1 @@ -2979,7 +2989,7 @@ def test_execute_trade_exit_market_order(default_conf_usdt, ticker_usdt, fee, assert trade.close_profit == 0.09451372 assert rpc_mock.call_count == 3 - last_msg = rpc_mock.call_args_list[-1][0][0] + last_msg = rpc_mock.call_args_list[-2][0][0] assert { 'type': RPCMessageType.SELL, 'trade_id': 1, @@ -3231,7 +3241,7 @@ def test_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usdt, assert freqtrade.handle_trade(trade) is False # Test if buy-signal is absent (should sell due to roi = true) - patch_get_signal(freqtrade, value=(False, True, None, None)) + patch_get_signal(freqtrade, value=(False, False, None, None)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.ROI.value @@ -3417,11 +3427,11 @@ def test_disable_ignore_roi_if_buy_signal(default_conf_usdt, limit_buy_order_usd trade = Trade.query.first() trade.update(limit_buy_order_usdt) # Sell due to min_roi_reached - patch_get_signal(freqtrade, value=(True, True, None, None)) + patch_get_signal(freqtrade, value=(True, False, None, None)) assert freqtrade.handle_trade(trade) is True # Test if buy-signal is absent - patch_get_signal(freqtrade, value=(False, True, None, None)) + patch_get_signal(freqtrade, value=(False, False, None, None)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.ROI.value diff --git a/tests/test_misc.py b/tests/test_misc.py index de3f368e9..21a00f3be 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -184,16 +184,18 @@ def test_render_template_fallback(mocker): assert 'if self.dp' in val -def test_parse_db_uri_for_logging() -> None: - postgresql_conn_uri = "postgresql+psycopg2://scott123:scott123@host/dbname" - mariadb_conn_uri = "mariadb+mariadbconnector://app_user:Password123!@127.0.0.1:3306/company" - mysql_conn_uri = "mysql+pymysql://user:pass@some_mariadb/dbname?charset=utf8mb4" - sqlite_conn_uri = "sqlite:////freqtrade/user_data/tradesv3.sqlite" - censored_pwd = "*****" +@pytest.mark.parametrize('conn_url,expected', [ + ("postgresql+psycopg2://scott123:scott123@host:1245/dbname", + "postgresql+psycopg2://scott123:*****@host:1245/dbname"), + ("postgresql+psycopg2://scott123:scott123@host.name.com/dbname", + "postgresql+psycopg2://scott123:*****@host.name.com/dbname"), + ("mariadb+mariadbconnector://app_user:Password123!@127.0.0.1:3306/company", + "mariadb+mariadbconnector://app_user:*****@127.0.0.1:3306/company"), + ("mysql+pymysql://user:pass@some_mariadb/dbname?charset=utf8mb4", + "mysql+pymysql://user:*****@some_mariadb/dbname?charset=utf8mb4"), + ("sqlite:////freqtrade/user_data/tradesv3.sqlite", + "sqlite:////freqtrade/user_data/tradesv3.sqlite"), +]) +def test_parse_db_uri_for_logging(conn_url, expected) -> None: - def get_pwd(x): return x.split(':')[2].split('@')[0] - - assert get_pwd(parse_db_uri_for_logging(postgresql_conn_uri)) == censored_pwd - assert get_pwd(parse_db_uri_for_logging(mariadb_conn_uri)) == censored_pwd - assert get_pwd(parse_db_uri_for_logging(mysql_conn_uri)) == censored_pwd - assert sqlite_conn_uri == parse_db_uri_for_logging(sqlite_conn_uri) + assert parse_db_uri_for_logging(conn_url) == expected diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 8a40f4a20..40a76d04e 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -336,15 +336,20 @@ def test_generate_profit_graph(testdatadir): assert fig.layout.yaxis3.title.text == "Profit BTC" figure = fig.layout.figure - assert len(figure.data) == 5 + assert len(figure.data) == 7 avgclose = find_trace_in_fig_data(figure.data, "Avg close price") assert isinstance(avgclose, go.Scatter) profit = find_trace_in_fig_data(figure.data, "Profit") assert isinstance(profit, go.Scatter) - profit = find_trace_in_fig_data(figure.data, "Max drawdown 10.45%") - assert isinstance(profit, go.Scatter) + drawdown = find_trace_in_fig_data(figure.data, "Max drawdown 10.45%") + assert isinstance(drawdown, go.Scatter) + parallel = find_trace_in_fig_data(figure.data, "Parallel trades") + assert isinstance(parallel, go.Scatter) + + underwater = find_trace_in_fig_data(figure.data, "Underwater Plot") + assert isinstance(underwater, go.Scatter) for pair in pairs: profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}") diff --git a/tests/test_worker.py b/tests/test_worker.py index c3773d296..ddca9525b 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -43,7 +43,7 @@ def test_worker_stopped(mocker, default_conf, caplog) -> None: worker.freqtrade.state = State.STOPPED state = worker._worker(old_state=State.RUNNING) assert state is State.STOPPED - assert log_has('Changing state to: STOPPED', caplog) + assert log_has('Changing state from RUNNING to: STOPPED', caplog) assert mock_throttle.call_count == 1